From 92d9bfe83fbef2070696b3dc4937e6309b0b97af Mon Sep 17 00:00:00 2001 From: pk-zipstack Date: Tue, 19 May 2026 16:14:50 +0530 Subject: [PATCH 1/4] UN-3332 [FIX] Validate single-pass exports against the default profile The Prompt Studio export validation filtered prompt outputs by prompt.profile_manager, but single-pass execution stores outputs against the tool's default profile (see OutputManagerHelper.handle_prompt_output_update) with is_single_pass_extract=True. When the default profile differed from the prompt-level profile FK, the validation lookup missed the rows produced by the single-pass run and incorrectly rejected the export with "Prompt Studio project without prompts cannot be exported." Match the validation lookup to the mode the prompts were actually run in: the tool default profile + is_single_pass_extract=True when single-pass is enabled, the prompt's profile_manager + False otherwise. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../prompt_studio_registry_helper.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py index 4fee8c10bc..ea164b9b85 100644 --- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py +++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py @@ -319,10 +319,21 @@ def frame_export_json( prompt.profile_manager = default_llm_profile if not force_export: + # Single-pass execution stores outputs against the tool's + # default profile (see + # OutputManagerHelper.handle_prompt_output_update), not the + # prompt-level profile_manager FK. Match the lookup to the + # mode the prompts were actually run in so validation does + # not miss the rows created during single-pass. + if tool.single_pass_extraction_mode: + output_profile = default_llm_profile + else: + output_profile = prompt.profile_manager prompt_output = PromptStudioOutputManager.objects.filter( tool_id=tool.tool_id, prompt_id=prompt.prompt_id, - profile_manager=prompt.profile_manager, + profile_manager=output_profile, + is_single_pass_extract=tool.single_pass_extraction_mode, ).all() if not prompt_output: invalidated_outputs.append(prompt.prompt_key) From c7affd4b67f3f89066ef5f2fd0cbd379a2f4c364 Mon Sep 17 00:00:00 2001 From: pk-zipstack Date: Mon, 25 May 2026 12:06:23 +0530 Subject: [PATCH 2/4] UN-3332 [TEST] Pin export validation profile lookup for single-pass mode Regression test for frame_export_json: - single-pass on: filter uses default profile + is_single_pass_extract=True - single-pass off: filter uses prompt.profile_manager + is_single_pass_extract=False - force_export=True bypasses the PromptStudioOutputManager lookup Follows the sys.modules-stub pattern from test_build_index_payload.py to avoid pulling in Django app loading; skips cleanly if imports drift. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../tests/__init__.py | 0 .../tests/test_frame_export_json.py | 318 ++++++++++++++++++ 2 files changed, 318 insertions(+) create mode 100644 backend/prompt_studio/prompt_studio_registry_v2/tests/__init__.py create mode 100644 backend/prompt_studio/prompt_studio_registry_v2/tests/test_frame_export_json.py diff --git a/backend/prompt_studio/prompt_studio_registry_v2/tests/__init__.py b/backend/prompt_studio/prompt_studio_registry_v2/tests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/backend/prompt_studio/prompt_studio_registry_v2/tests/test_frame_export_json.py b/backend/prompt_studio/prompt_studio_registry_v2/tests/test_frame_export_json.py new file mode 100644 index 0000000000..8bcc72abbe --- /dev/null +++ b/backend/prompt_studio/prompt_studio_registry_v2/tests/test_frame_export_json.py @@ -0,0 +1,318 @@ +"""Regression tests for ``PromptStudioRegistryHelper.frame_export_json``. + +Pins the UN-3332 fix: when ``tool.single_pass_extraction_mode`` is True, +single-pass execution stores ``PromptStudioOutputManager`` rows under the +tool's *default* profile with ``is_single_pass_extract=True`` (see +``OutputManagerHelper.handle_prompt_output_update``). The export +validator must therefore look up rows by that same (profile, mode) tuple +— previously it filtered by ``prompt.profile_manager`` (the prompt-card +FK, frozen at prompt-creation time), which silently missed rows whenever +the default profile and the prompt-level profile diverged. The result +was a misleading "project without prompts cannot be exported" error +after a successful single-pass run. + +Mirrors the ``test_build_index_payload`` approach: the backend test +environment has no ``pytest-django`` and the helper has a heavy +Django-coupled import surface, so every collaborator is stubbed on +``sys.modules`` before the helper is imported. +""" + +from __future__ import annotations + +import sys +import types +from typing import Any +from unittest.mock import MagicMock, patch + +import pytest + + +def _install(name: str, attrs: dict[str, Any] | None = None) -> types.ModuleType: + """Install (or replace) a fake module into ``sys.modules``.""" + mod = types.ModuleType(name) + if attrs: + for key, value in attrs.items(): + setattr(mod, key, value) + sys.modules[name] = mod + return mod + + +def _install_package(name: str) -> types.ModuleType: + """Install a fake package (only if it is not already loaded).""" + if name in sys.modules: + return sys.modules[name] + mod = types.ModuleType(name) + mod.__path__ = [] # type: ignore[attr-defined] + sys.modules[name] = mod + return mod + + +try: + # Configure a minimal Django settings module so that ``settings`` + # attribute access inside ``frame_export_json`` (PLATFORM_POSTAMBLE, + # WORD_CONFIDENCE_POSTAMBLE) does not raise ``ImproperlyConfigured``. + # Safe to call repeatedly — ``configure`` is idempotent for our use + # because we check ``configured`` first. + try: + from django.conf import settings as _dj_settings + + if not _dj_settings.configured: + _dj_settings.configure( + PLATFORM_POSTAMBLE="", + WORD_CONFIDENCE_POSTAMBLE="", + ) + except Exception: + pass + + _install_package("account_v2") + _install("account_v2.models", {"User": MagicMock(name="User")}) + + _install_package("adapter_processor_v2") + _install( + "adapter_processor_v2.models", + {"AdapterInstance": MagicMock(name="AdapterInstance")}, + ) + + _install("plugins", {"get_plugin": MagicMock(return_value=None)}) + + _install_package("prompt_studio") + _install( + "prompt_studio.lookup_utils", + {"validate_lookups_for_export": MagicMock(return_value=({}, None))}, + ) + + _install_package("prompt_studio.prompt_profile_manager_v2") + _install( + "prompt_studio.prompt_profile_manager_v2.models", + {"ProfileManager": MagicMock(name="ProfileManager")}, + ) + + _install_package("prompt_studio.prompt_studio_core_v2") + _install( + "prompt_studio.prompt_studio_core_v2.models", + {"CustomTool": MagicMock(name="CustomTool")}, + ) + _install( + "prompt_studio.prompt_studio_core_v2.prompt_studio_helper", + {"PromptStudioHelper": MagicMock(name="PromptStudioHelper")}, + ) + + _install_package("prompt_studio.prompt_studio_output_manager_v2") + _install( + "prompt_studio.prompt_studio_output_manager_v2.models", + {"PromptStudioOutputManager": MagicMock(name="PromptStudioOutputManager")}, + ) + + _install_package("prompt_studio.prompt_studio_v2") + _install( + "prompt_studio.prompt_studio_v2.models", + {"ToolStudioPrompt": MagicMock(name="ToolStudioPrompt")}, + ) + + _install_package("unstract") + _install_package("unstract.tool_registry") + _install( + "unstract.tool_registry.dto", + { + "Properties": MagicMock(name="Properties"), + "Spec": MagicMock(name="Spec"), + "Tool": MagicMock(name="Tool"), + }, + ) + + # Sibling modules of the helper — both define Django Model / + # ModelSerializer classes that require ``INSTALLED_APPS`` at import + # time. Stub them so the helper's ``from .models import ...`` and + # ``from .serializers import ...`` resolve without booting Django. + _install( + "prompt_studio.prompt_studio_registry_v2.models", + {"PromptStudioRegistry": MagicMock(name="PromptStudioRegistry")}, + ) + _install( + "prompt_studio.prompt_studio_registry_v2.serializers", + {"PromptStudioRegistrySerializer": MagicMock(name="PromptStudioRegistrySerializer")}, + ) + + from prompt_studio.prompt_studio_registry_v2 import ( # noqa: E402 + prompt_studio_registry_helper as _psrh_mod, + ) + + PromptStudioRegistryHelper = _psrh_mod.PromptStudioRegistryHelper + _IMPORT_ERROR: str | None = None +except Exception as exc: # pragma: no cover — environment guard + _IMPORT_ERROR = ( + f"prompt_studio_registry_helper could not be imported in this " + f"environment: {type(exc).__name__}: {exc}" + ) + PromptStudioRegistryHelper = None # type: ignore[assignment] + _psrh_mod = None # type: ignore[assignment] + + +pytestmark = pytest.mark.skipif( + _IMPORT_ERROR is not None, reason=_IMPORT_ERROR or "" +) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _make_profile(name: str) -> MagicMock: + """Build a ProfileManager mock with the attributes frame_export_json + accesses on both the default and prompt-level profiles.""" + profile = MagicMock(name=f"ProfileManager[{name}]") + profile.profile_id = f"profile-{name}" + profile.llm.id = f"llm-{name}" + profile.vector_store.id = f"vdb-{name}" + profile.embedding_model.id = f"emb-{name}" + profile.embedding_model.adapter_id = f"adapter-{name}|suffix" + profile.x2text.id = f"x2t-{name}" + profile.chunk_size = 512 + profile.chunk_overlap = 64 + profile.retrieval_strategy = "simple" + profile.similarity_top_k = 3 + profile.section = "all" + profile.reindex = False + return profile + + +def _make_tool(*, single_pass: bool) -> MagicMock: + tool = MagicMock(name="CustomTool") + tool.tool_id = "tool-1" + tool.tool_name = "test-tool" + tool.description = "desc" + tool.author = "author" + tool.prompt_grammer = None + tool.summarize_prompt = "" + tool.summarize_as_source = False + tool.preamble = "" + tool.postamble = "" + tool.enable_challenge = False + tool.challenge_llm = None + tool.single_pass_extraction_mode = single_pass + tool.enable_highlight = False + tool.enable_word_confidence = False + return tool + + +def _make_prompt(*, profile: MagicMock) -> MagicMock: + prompt = MagicMock(name="ToolStudioPrompt") + prompt.prompt_id = "prompt-1" + prompt.prompt_key = "key" + prompt.prompt = "what is X?" + prompt.prompt_type = "LLM" # any non-NOTES value + prompt.active = True + prompt.required = False + prompt.enforce_type = "text" + prompt.profile_manager = profile + prompt.enable_postprocessing_webhook = False + prompt.postprocessing_webhook_url = "" + return prompt + + +def _run_export(*, tool: MagicMock, prompt: MagicMock, force_export: bool = False): + """Invoke ``frame_export_json`` with a patched + ``PromptStudioOutputManager.objects.filter`` and return the captured + filter call along with the result (or raised exception). + """ + # The filter chain is ``Model.objects.filter(...).all()`` — return a + # truthy list so the prompt is treated as "run". + filter_call = MagicMock(name="filter") + filter_call.return_value.all.return_value = [object()] + + objects = MagicMock(name="objects") + objects.filter = filter_call + + raised: Exception | None = None + result: Any = None + with patch.object(_psrh_mod.PromptStudioOutputManager, "objects", objects): + try: + result = PromptStudioRegistryHelper.frame_export_json( + tool=tool, prompts=[prompt], force_export=force_export, + ) + except Exception as exc: # surface to assertions + raised = exc + + return filter_call, result, raised + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestFrameExportJsonProfileLookup: + """Pin the UN-3332 fix: validation profile depends on single-pass mode.""" + + def test_single_pass_uses_default_profile_and_single_pass_flag(self) -> None: + """When single-pass is on, filter must use the tool's default + profile and ``is_single_pass_extract=True`` — NOT the prompt's + own profile_manager FK. + """ + default_profile = _make_profile("default") + prompt_profile = _make_profile("prompt") # the "wrong" one + tool = _make_tool(single_pass=True) + prompt = _make_prompt(profile=prompt_profile) + + with patch.object( + _psrh_mod.ProfileManager, + "get_default_llm_profile", + return_value=default_profile, + ): + filter_call, _result, raised = _run_export(tool=tool, prompt=prompt) + + assert raised is None, f"export failed unexpectedly: {raised!r}" + filter_call.assert_called_once() + kwargs = filter_call.call_args.kwargs + assert kwargs["profile_manager"] is default_profile, ( + "single-pass export must validate against the default profile" + ) + assert kwargs["is_single_pass_extract"] is True + assert kwargs["tool_id"] == tool.tool_id + assert kwargs["prompt_id"] == prompt.prompt_id + + def test_non_single_pass_uses_prompt_profile_and_normal_flag(self) -> None: + """When single-pass is off, filter must use the prompt's own + ``profile_manager`` and ``is_single_pass_extract=False``. + """ + default_profile = _make_profile("default") + prompt_profile = _make_profile("prompt") + tool = _make_tool(single_pass=False) + prompt = _make_prompt(profile=prompt_profile) + + with patch.object( + _psrh_mod.ProfileManager, + "get_default_llm_profile", + return_value=default_profile, + ): + filter_call, _result, raised = _run_export(tool=tool, prompt=prompt) + + assert raised is None, f"export failed unexpectedly: {raised!r}" + filter_call.assert_called_once() + kwargs = filter_call.call_args.kwargs + assert kwargs["profile_manager"] is prompt_profile, ( + "non-single-pass export must validate against the prompt's profile" + ) + assert kwargs["is_single_pass_extract"] is False + + def test_force_export_skips_output_lookup_entirely(self) -> None: + """``force_export=True`` bypasses validation: the filter must + never be called. + """ + default_profile = _make_profile("default") + prompt_profile = _make_profile("prompt") + tool = _make_tool(single_pass=True) + prompt = _make_prompt(profile=prompt_profile) + + with patch.object( + _psrh_mod.ProfileManager, + "get_default_llm_profile", + return_value=default_profile, + ): + filter_call, _result, raised = _run_export( + tool=tool, prompt=prompt, force_export=True, + ) + + assert raised is None, f"forced export failed: {raised!r}" + filter_call.assert_not_called() From 4ff2b3c1f9f3d9881a774c87a42d1d5f05a061e4 Mon Sep 17 00:00:00 2001 From: pk-zipstack Date: Mon, 25 May 2026 12:19:00 +0530 Subject: [PATCH 3/4] UN-3332 [FIX] Use default profile for per-prompt export JSON in single-pass mode MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Addresses two review-bot findings on top of the initial validation fix: 1. Greptile: per-prompt export entries (vector_db / embedding / llm / x2text / chunk_size / chunk_overlap / retrieval_strategy / similarity_top_k / section / reindex) still read from prompt.profile_manager even when single-pass mode was on. Same root cause as the validation bug — if the user added a second profile and set it as default before running, the exported tool embedded the stale prompt-level profile's settings instead of the default profile that was actually executed. Lifts the output_profile resolution out of the validation block so it drives both the lookup and the per-prompt payload, eliminating the divergence. 2. CodeRabbit: the test module's import-time `except Exception` masked genuine helper bugs (NameError, TypeError at module load, etc.) behind a "skipped suite" — defeating regression coverage. Narrow to ImportError / ModuleNotFoundError / AttributeError so only true environment/dependency failures skip; everything else surfaces. Tests extended to assert per-prompt JSON content (llm, vector_db, embedding, etc.) matches the expected profile for each mode. Verified the new assertions correctly fail on pre-fix per-prompt logic. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../prompt_studio_registry_helper.py | 49 ++++++++-------- .../tests/test_frame_export_json.py | 56 +++++++++++++++---- 2 files changed, 70 insertions(+), 35 deletions(-) diff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py index ea164b9b85..3b6267ed54 100644 --- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py +++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py @@ -318,17 +318,20 @@ def frame_export_json( if not prompt.profile_manager: prompt.profile_manager = default_llm_profile + # Single-pass execution always runs prompts via the tool's + # default profile (see + # OutputManagerHelper.handle_prompt_output_update). Both + # validation lookup AND the per-prompt export payload must + # therefore source their settings from the default profile in + # single-pass mode — otherwise the exported tool would embed + # the stale prompt-level profile and the validation lookup + # would miss the rows actually produced by the run. + if tool.single_pass_extraction_mode: + output_profile = default_llm_profile + else: + output_profile = prompt.profile_manager + if not force_export: - # Single-pass execution stores outputs against the tool's - # default profile (see - # OutputManagerHelper.handle_prompt_output_update), not the - # prompt-level profile_manager FK. Match the lookup to the - # mode the prompts were actually run in so validation does - # not miss the rows created during single-pass. - if tool.single_pass_extraction_mode: - output_profile = default_llm_profile - else: - output_profile = prompt.profile_manager prompt_output = PromptStudioOutputManager.objects.filter( tool_id=tool.tool_id, prompt_id=prompt.prompt_id, @@ -339,35 +342,31 @@ def frame_export_json( invalidated_outputs.append(prompt.prompt_key) continue - vector_db = str(prompt.profile_manager.vector_store.id) - embedding_model = str(prompt.profile_manager.embedding_model.id) - llm = str(prompt.profile_manager.llm.id) - x2text = str(prompt.profile_manager.x2text.id) - adapter_id = str(prompt.profile_manager.embedding_model.adapter_id) + vector_db = str(output_profile.vector_store.id) + embedding_model = str(output_profile.embedding_model.id) + llm = str(output_profile.llm.id) + x2text = str(output_profile.x2text.id) + adapter_id = str(output_profile.embedding_model.adapter_id) embedding_suffix = adapter_id.split("|")[0] output[JsonSchemaKey.PROMPT] = prompt.prompt output[JsonSchemaKey.ACTIVE] = prompt.active output[JsonSchemaKey.REQUIRED] = prompt.required - output[JsonSchemaKey.CHUNK_SIZE] = prompt.profile_manager.chunk_size + output[JsonSchemaKey.CHUNK_SIZE] = output_profile.chunk_size output[JsonSchemaKey.VECTOR_DB] = vector_db output[JsonSchemaKey.EMBEDDING] = embedding_model output[JsonSchemaKey.X2TEXT_ADAPTER] = x2text - output[JsonSchemaKey.CHUNK_OVERLAP] = prompt.profile_manager.chunk_overlap + output[JsonSchemaKey.CHUNK_OVERLAP] = output_profile.chunk_overlap output[JsonSchemaKey.LLM] = llm output[JsonSchemaKey.PREAMBLE] = tool.preamble output[JsonSchemaKey.POSTAMBLE] = tool.postamble output[JsonSchemaKey.GRAMMAR] = grammar_list output[JsonSchemaKey.TYPE] = prompt.enforce_type output[JsonSchemaKey.NAME] = prompt.prompt_key - output[JsonSchemaKey.RETRIEVAL_STRATEGY] = ( - prompt.profile_manager.retrieval_strategy - ) - output[JsonSchemaKey.SIMILARITY_TOP_K] = ( - prompt.profile_manager.similarity_top_k - ) - output[JsonSchemaKey.SECTION] = prompt.profile_manager.section - output[JsonSchemaKey.REINDEX] = prompt.profile_manager.reindex + output[JsonSchemaKey.RETRIEVAL_STRATEGY] = output_profile.retrieval_strategy + output[JsonSchemaKey.SIMILARITY_TOP_K] = output_profile.similarity_top_k + output[JsonSchemaKey.SECTION] = output_profile.section + output[JsonSchemaKey.REINDEX] = output_profile.reindex output[JsonSchemaKey.EMBEDDING_SUFFIX] = embedding_suffix # Webhook postprocessing settings output[JsonSchemaKey.ENABLE_POSTPROCESSING_WEBHOOK] = ( diff --git a/backend/prompt_studio/prompt_studio_registry_v2/tests/test_frame_export_json.py b/backend/prompt_studio/prompt_studio_registry_v2/tests/test_frame_export_json.py index 8bcc72abbe..442cfa42a7 100644 --- a/backend/prompt_studio/prompt_studio_registry_v2/tests/test_frame_export_json.py +++ b/backend/prompt_studio/prompt_studio_registry_v2/tests/test_frame_export_json.py @@ -139,7 +139,11 @@ def _install_package(name: str) -> types.ModuleType: PromptStudioRegistryHelper = _psrh_mod.PromptStudioRegistryHelper _IMPORT_ERROR: str | None = None -except Exception as exc: # pragma: no cover — environment guard +# Only swallow import/dependency failures here (stub drift, missing +# transitive module). Genuine bugs inside the helper (NameError, +# SyntaxError, TypeError at module load, etc.) must surface as real test +# failures, not silently skip the suite. +except (ImportError, ModuleNotFoundError, AttributeError) as exc: _IMPORT_ERROR = ( f"prompt_studio_registry_helper could not be imported in this " f"environment: {type(exc).__name__}: {exc}" @@ -242,13 +246,40 @@ def _run_export(*, tool: MagicMock, prompt: MagicMock, force_export: bool = Fals # --------------------------------------------------------------------------- +def _per_prompt_output(result: Any) -> dict[str, Any]: + """Return the single per-prompt entry from a frame_export_json result.""" + outputs = result["outputs"] + assert len(outputs) == 1, f"expected exactly one per-prompt output, got {outputs!r}" + return outputs[0] + + +def _assert_output_uses_profile(output: dict[str, Any], profile: MagicMock) -> None: + """Assert the per-prompt export entry was assembled from ``profile``. + + Covers the fields the exported tool consumes when single-pass is + disabled at runtime — these were previously hardcoded to + ``prompt.profile_manager`` regardless of single-pass mode. + """ + assert output["llm"] == profile.llm.id + assert output["vector-db"] == profile.vector_store.id + assert output["embedding"] == profile.embedding_model.id + assert output["x2text_adapter"] == profile.x2text.id + assert output["chunk-size"] == profile.chunk_size + assert output["chunk-overlap"] == profile.chunk_overlap + assert output["retrieval-strategy"] == profile.retrieval_strategy + assert output["similarity-top-k"] == profile.similarity_top_k + assert output["section"] == profile.section + assert output["reindex"] == profile.reindex + + class TestFrameExportJsonProfileLookup: """Pin the UN-3332 fix: validation profile depends on single-pass mode.""" def test_single_pass_uses_default_profile_and_single_pass_flag(self) -> None: - """When single-pass is on, filter must use the tool's default - profile and ``is_single_pass_extract=True`` — NOT the prompt's - own profile_manager FK. + """When single-pass is on, BOTH the validation filter and the + per-prompt export entry must use the tool's default profile (with + ``is_single_pass_extract=True``) — NOT the prompt's own + ``profile_manager`` FK. """ default_profile = _make_profile("default") prompt_profile = _make_profile("prompt") # the "wrong" one @@ -260,7 +291,7 @@ def test_single_pass_uses_default_profile_and_single_pass_flag(self) -> None: "get_default_llm_profile", return_value=default_profile, ): - filter_call, _result, raised = _run_export(tool=tool, prompt=prompt) + filter_call, result, raised = _run_export(tool=tool, prompt=prompt) assert raised is None, f"export failed unexpectedly: {raised!r}" filter_call.assert_called_once() @@ -271,10 +302,12 @@ def test_single_pass_uses_default_profile_and_single_pass_flag(self) -> None: assert kwargs["is_single_pass_extract"] is True assert kwargs["tool_id"] == tool.tool_id assert kwargs["prompt_id"] == prompt.prompt_id + _assert_output_uses_profile(_per_prompt_output(result), default_profile) def test_non_single_pass_uses_prompt_profile_and_normal_flag(self) -> None: - """When single-pass is off, filter must use the prompt's own - ``profile_manager`` and ``is_single_pass_extract=False``. + """When single-pass is off, BOTH the validation filter and the + per-prompt export entry must use the prompt's own + ``profile_manager`` (with ``is_single_pass_extract=False``). """ default_profile = _make_profile("default") prompt_profile = _make_profile("prompt") @@ -286,7 +319,7 @@ def test_non_single_pass_uses_prompt_profile_and_normal_flag(self) -> None: "get_default_llm_profile", return_value=default_profile, ): - filter_call, _result, raised = _run_export(tool=tool, prompt=prompt) + filter_call, result, raised = _run_export(tool=tool, prompt=prompt) assert raised is None, f"export failed unexpectedly: {raised!r}" filter_call.assert_called_once() @@ -295,10 +328,12 @@ def test_non_single_pass_uses_prompt_profile_and_normal_flag(self) -> None: "non-single-pass export must validate against the prompt's profile" ) assert kwargs["is_single_pass_extract"] is False + _assert_output_uses_profile(_per_prompt_output(result), prompt_profile) def test_force_export_skips_output_lookup_entirely(self) -> None: """``force_export=True`` bypasses validation: the filter must - never be called. + never be called. Per-prompt JSON still follows single-pass mode + (default profile here, because ``single_pass=True``). """ default_profile = _make_profile("default") prompt_profile = _make_profile("prompt") @@ -310,9 +345,10 @@ def test_force_export_skips_output_lookup_entirely(self) -> None: "get_default_llm_profile", return_value=default_profile, ): - filter_call, _result, raised = _run_export( + filter_call, result, raised = _run_export( tool=tool, prompt=prompt, force_export=True, ) assert raised is None, f"forced export failed: {raised!r}" filter_call.assert_not_called() + _assert_output_uses_profile(_per_prompt_output(result), default_profile) From 356f795db61cc3b769386479784832ea1a2c96c5 Mon Sep 17 00:00:00 2001 From: pk-zipstack Date: Thu, 28 May 2026 09:58:55 +0530 Subject: [PATCH 4/4] UN-3332 [MISC] Tighten output_profile inline comment MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per review feedback — drop the implementation-history reasoning and keep just the factual rule: single-pass uses the default profile, everything else uses the prompt-level profile_manager. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../prompt_studio_registry_helper.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py index 3b6267ed54..b39e4133da 100644 --- a/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py +++ b/backend/prompt_studio/prompt_studio_registry_v2/prompt_studio_registry_helper.py @@ -318,14 +318,8 @@ def frame_export_json( if not prompt.profile_manager: prompt.profile_manager = default_llm_profile - # Single-pass execution always runs prompts via the tool's - # default profile (see - # OutputManagerHelper.handle_prompt_output_update). Both - # validation lookup AND the per-prompt export payload must - # therefore source their settings from the default profile in - # single-pass mode — otherwise the exported tool would embed - # the stale prompt-level profile and the validation lookup - # would miss the rows actually produced by the run. + # Single-pass runs every prompt via the tool's default profile; + # otherwise each prompt uses its own profile_manager. if tool.single_pass_extraction_mode: output_profile = default_llm_profile else: