Evaluation option matrix, API/responsive coverage, org-model API tests

.github/workflows/run-smoke.yml

@@ -225,5 +225,7 @@ jobs:
         uses: actions/upload-artifact@v7
         with:
           name: smoke-test-report
-          path: reports/smoke_report.html
+          path: |
+            reports/smoke_report.html
+            screenshots/FAIL_*.png
           retention-days: 30

locators/evaluations_locators.py

@@ -404,3 +404,14 @@ class EvaluationsLocators:
 
     # Report download
     DOWNLOAD_REPORT_BUTTON = "button:has-text('Download Report'), a:has-text('Download Report')"
+
+    # Evaluation matrix (mapped on prod 2026-11-05)
+    MODAL_EVALUATION_DOMAIN_SELECT = "select[name='evaluationDomain']"
+    WIZARD_ADD_INPUT_PROMPT_BUTTON = "button:has-text('Add Input Prompt')"
+    WIZARD_CUSTOM_TEST_INPUT = "textarea[name='testInput']"
+    WIZARD_CUSTOM_REFERENCE_OUTPUT = "textarea[name='referenceOutput']"
+    PLAYGROUND_INPUT = "textarea[name='inputPrompt']"
+    PLAYGROUND_SUBMIT = "button:has-text('Submit')"
+    PLAYGROUND_PASSED = "button:has-text('Passed')"
+    PLAYGROUND_GENERATING = "text=Generating"
+    PLAYGROUND_SAVE_NEXT = "button:has-text('Save and Test New Input')"

locators/prompt_libraries_locators.py

@@ -29,9 +29,8 @@ class PromptLibrariesLocators:
     UPLOAD_FILES_TOGGLE = "text=Upload files, text=Upload Files"
 
     # Known library names
-    LIBRARY_KCC_ENGLISH = "text=KCC) - English Queries"
+    LIBRARY_KCC_ENGLISH = "text=KCC English Queries"
     LIBRARY_PARIKSHA = "text=Microsoft: PARIKSHA"
     LIBRARY_PUBLIC_INTEREST_ENGLISH = "text=Public Interest English"
-    LIBRARY_PUBLIC_INTEREST_HINDI = "text=Public Interest Hindi"
-    LIBRARY_KCC_HINDI = "text=KCC): Hindi Queries"
+    LIBRARY_KCC_HINDI = "text=KCC Hindi Queries"
     LIBRARY_KISAN_CALL = "text=Kisan Call Centre"

pages/new_evaluation_page.py

@@ -191,7 +191,7 @@ def select_model_by_index(self, index: int = 1) -> None:
     EXCLUDED_MODEL_NAMES = {"New AI Model", "xAI: Grok 4.1 Fast"}
 
     def select_random_valid_model_and_version(self) -> str:
-        """Select a random model, excluding 'New AI Model' and deprecated 'xAI: Grok 4.1 Fast'.
+        """Select a random model, excluding 'New AI Model', deprecated 'xAI: Grok 4.1 Fast' and any Opus model (cost).
 
         Returns the selected model's display text (useful for logging/assertions).
         Falls back to index 1 if every option is excluded or the list is empty.
@@ -201,7 +201,7 @@ def select_random_valid_model_and_version(self) -> str:
         options = self.get_modal_model_options()
         valid = [
             i for i, (_, text) in enumerate(options)
-            if text.strip() not in self.EXCLUDED_MODEL_NAMES
+            if text.strip() not in self.EXCLUDED_MODEL_NAMES and "opus" not in text.lower()
         ]
         if not valid:
             self.select_model_by_index(1)
@@ -278,6 +278,12 @@ def select_evaluator_type_in_modal(self, eval_type: str = "technical") -> None:
         sel = value_map.get(eval_type.lower(), value_map["technical"])
         self.page.locator(sel).first.check()
 
+    def select_domain_in_modal(self, domain: str) -> None:
+        """Pick the step-2 'Evaluation Domain' (healthcare | agriculture | general)."""
+        self.page.locator(EvaluationsLocators.MODAL_EVALUATION_DOMAIN_SELECT).first.select_option(
+            domain.upper()
+        )
+
     # DOM `value`s are the backend enum (TECHNICAL_AUDIT/DOMAIN_AUDIT/
     # CULTURAL_AUDIT); map back to the human-readable label the UI shows
     # and tests assert against. See locators/evaluations_locators.py.
@@ -370,6 +376,7 @@ def start_evaluation_from_modal(
         eval_type: str = "technical",
         objective: str = "Automated test evaluation objective",
         name: str | None = None,
+        domain: str | None = None,
     ) -> NewEvaluationPage:
         """Complete both modal steps and land on the single-page wizard.
 
@@ -389,6 +396,8 @@ def start_evaluation_from_modal(
         self.select_evaluation_method(method)
         self.click_modal_next()
         self.select_evaluator_type_in_modal(eval_type)
+        if domain:
+            self.select_domain_in_modal(domain)
         self.fill_modal_objective(objective)
         self.click_modal_start()
         return self
@@ -531,6 +540,26 @@ def select_first_prompt_library(self) -> bool:
         self.page.wait_for_timeout(1_000)
         return True
 
+    def select_smallest_prompt_library(self) -> int | None:
+        """Pick the library with the fewest (non-zero) test cases on the current
+        table page and return that count. Big libraries take ~50 min to run."""
+        radio_sel = EvaluationsLocators.WIZARD_PROMPT_LIBRARY_RADIO
+        rows = self.page.locator(f"tr:has({radio_sel})")
+        best: tuple[int, int] | None = None
+        for i in range(rows.count()):
+            digits = [
+                int(c.replace(",", "").strip())
+                for c in rows.nth(i).locator("td").all_inner_texts()
+                if c.replace(",", "").strip().isdigit()
+            ]
+            if digits and digits[0] > 0 and (best is None or digits[0] < best[0]):
+                best = (digits[0], i)
+        if best is None:
+            return None
+        rows.nth(best[1]).locator(radio_sel).first.click(timeout=10_000)
+        self.page.wait_for_timeout(1_000)
+        return best[0]
+
     def configure_bulk_workspace_minimal(self, module: str = "hallucination") -> bool:
         """Configure the minimum required to enable Run Evaluation: check a
         module, pick a sub-module, choose the prompt-library source, and
@@ -545,6 +574,58 @@ def configure_bulk_workspace_minimal(self, module: str = "hallucination") -> boo
         self.select_first_prompt_library()
         return self.is_run_evaluation_button_enabled()
 
+    def select_one_submodule_in_each_module(self) -> int:
+        """Bulk with several modules shows one sub-module combobox per module;
+        pick the first option in each. Returns how many were set."""
+        boxes = self.page.locator("input[role='combobox']")
+        done = 0
+        for i in range(boxes.count()):
+            boxes.nth(i).click(timeout=10_000)
+            self.page.wait_for_timeout(500)
+            boxes.nth(i).press("Enter")
+            self.page.mouse.click(5, 5)
+            self.page.wait_for_timeout(500)
+            done += 1
+        return done
+
+    def add_custom_prompt(self, test_input: str, reference_output: str = "") -> None:
+        """'Add your own prompts' source: open the Add Input Prompt dialog,
+        fill it and save (the dialog's Next button is the save action)."""
+        self.page.locator(EvaluationsLocators.WIZARD_ADD_INPUT_PROMPT_BUTTON).first.click()
+        field = self.page.locator(EvaluationsLocators.WIZARD_CUSTOM_TEST_INPUT)
+        field.fill(test_input)
+        if reference_output:
+            self.page.locator(EvaluationsLocators.WIZARD_CUSTOM_REFERENCE_OUTPUT).fill(reference_output)
+        self.page.get_by_role("dialog").get_by_role("button", name="Next").click()
+        field.wait_for(state="hidden", timeout=10_000)
+
+    def submit_playground_test_case(self, prompt: str, timeout: int = 90_000) -> None:
+        """Playground: type a prompt, Submit, wait for the model to finish,
+        mark it Passed and Save. Passed is clickable while the output still
+        says 'Generating', so wait for that to clear first; only
+        'Save and Test New Input' persists the case."""
+        self.page.locator(EvaluationsLocators.PLAYGROUND_INPUT).fill(prompt)
+        self.page.locator(EvaluationsLocators.PLAYGROUND_SUBMIT).first.click()
+        generating = self.page.locator(EvaluationsLocators.PLAYGROUND_GENERATING).first
+        try:
+            generating.wait_for(state="visible", timeout=5_000)
+        except Exception:  # noqa: BLE001 - fast model: it may already be gone
+            pass
+        generating.wait_for(state="hidden", timeout=timeout)
+        self.page.locator(EvaluationsLocators.PLAYGROUND_PASSED).first.click()
+        self.page.locator(EvaluationsLocators.PLAYGROUND_SAVE_NEXT).first.click()
+        self.page.wait_for_timeout(2_000)
+
+    def click_finish_evaluation(self, recommendation: str = "Automated playground evaluation.") -> None:
+        """Finish opens an 'Evaluation Recommendation' dialog: the recommendation
+        is required and a second 'Finish Evaluation' inside the dialog confirms."""
+        self.page.locator(EvaluationsLocators.FINISH_EVALUATION_BUTTON).first.click()
+        dialog = self.page.get_by_role("dialog")
+        dialog.get_by_role("textbox").fill(recommendation)
+        dialog.get_by_role("button", name="Finish Evaluation").click()
+        dialog.wait_for(state="hidden", timeout=30_000)
+        self.page.wait_for_timeout(2_000)
+
     def is_run_evaluation_library_error_visible(self) -> bool:
         """Return True if 'Please select a prompt library…' error is shown."""
         return self.is_visible(
@@ -735,9 +816,30 @@ def check_module(self, module: str) -> None:
             # Fallback: the label itself may act as a toggle (custom checkbox)
             self.click(label_sel)
 
+    _MODULE_LABELS = {
+        "hallucination": "Hallucination and Misinformation",
+        "bias": "Bias and Fairness",
+        "privacy": "Privacy and Safety",
+    }
+
     def uncheck_module(self, module: str) -> None:
-        """Uncheck an evaluation module checkbox (same logic as check_module)."""
-        self.check_module(module)  # toggle — callers must know current state
+        """Uncheck a module checkbox if it is currently checked (plain click:
+        force-clicks double-toggle these custom checkboxes)."""
+        label = self._MODULE_LABELS[module.lower()]
+        cb = self.page.locator(f"input[type='checkbox']:near(:text('{label}'))").first
+        if cb.count() and cb.is_checked():
+            cb.click()
+            self.page.wait_for_timeout(300)
+
+    def set_modules(self, modules) -> None:
+        """Select exactly `modules`. Hallucination is pre-checked on load, so
+        check the wanted ones first and then clear the rest."""
+        wanted = [m.lower() for m in modules]
+        for m in wanted:
+            self.check_module(m)
+        for m in self._MODULE_LABELS:
+            if m not in wanted:
+                self.uncheck_module(m)
 
     def module_subcategory_dropdown_visible(self) -> bool:
         """Return True if a sub-category dropdown is visible (module is checked)."""

reports/TEST_REPORT.md

@@ -1,30 +1,46 @@
 # Parakh Test Framework — Test Report
 
-**Generated:** 2026-07-31 06:13 UTC  
-**Environment:** `staging` — https://dev.parakh.civicdataspace.in  
+**Generated:** 2026-11-05 19:57 UTC  
+**Environment:** `production` — https://parakh.civicdataspace.in  
 **Browser:** `chromium`  
-**Overall result:** ✅ PASSED
+**Overall result:** ❌ FAILED
 
 ## Summary
 
 | Metric | Value |
 | ------ | ----- |
-| Total tests | 2 |
+| Total tests | 1 |
 | Passed | 0 ✅ |
-| Failed | 0 ❌ |
-| Skipped | 2 ⏭️ |
+| Failed | 1 ❌ |
+| Skipped | 0 ⏭️ |
+| xfailed | 0 🔶 |
+| xpassed | 0 🔷 |
 | Errors | 0 💥 |
 | Pass rate | 0.0% |
-| Duration | 0.00s |
+| Duration | 47.78s |
 
 ## Results by Suite
 
-### E2E (2 tests — ✅ 0 / ❌ 0 / ⏭️ 2)
+### E2E (1 tests — ✅ 0 / ❌ 1 / ⏭️ 0 / 🔶 0 / 🔷 0)
 
 | Test | Result | Duration |
 | ---- | ------ | -------- |
-| `test_evaluator_cards_are_present[chromium]` | ⏭️ skipped | 0.00s |
-| `test_at_least_two_evaluators_listed[chromium]` | ⏭️ skipped | 0.00s |
+| `test_create_run_complete_and_report[chromium-bulk-technical-general-hallucination+bias+privacy-library]` | ❌ failed | 47.72s |
+
+## Failure Details
+
+### ❌ `tests/e2e/test_evaluation_matrix.py::test_create_run_complete_and_report[chromium-bulk-technical-general-hallucination+bias+privacy-library]`
+
+```
+tests/e2e/test_evaluation_matrix.py:134: in test_create_run_complete_and_report
+    audit = _wait_status(client, audit_id, ("PENDING_REVIEW", "COMPLETED"), BUDGET_S)
+            ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+tests/e2e/test_evaluation_matrix.py:61: in _wait_status
+    pytest.fail(f"audit {audit_id} ended {last} while waiting for {want}")
+E   Failed: audit 941 ended FAILED while waiting for ('PENDING_REVIEW', 'COMPLETED')
+```
+
+**Screenshot:** `/Users/home/CivicDataSpace/ParakhAI_test/screenshots/20261106_012700_FAIL_FAST_test_create_run_complete_and_report[chromium-bulk-technical-general-hallucination+bias+privacy-library].png`
 
 ---
 

tests/accessibility/test_accessibility_auth.py

@@ -443,9 +443,6 @@ def test_ux010_new_evaluation_dialog_aria_describedby_resolves(
         )
 
     @pytest.mark.auth
-    @pytest.mark.xfail(
-        reason="UX-011: Select fields in New Evaluation dialog have no accessible label — known bug"
-    )
     def test_ux011_new_evaluation_dialog_selects_have_labels(
         self, authenticated_page_fast: Page
     ):

tests/api/test_auditor_management_api.py

@@ -17,7 +17,7 @@
 
 import pytest
 
-from tests.data.test_data import TestGraphQL
+from tests.data.test_data import TestGraphQL, first_org_model_id
 
 pytestmark = [pytest.mark.api, pytest.mark.regression, pytest.mark.auth]
 
@@ -78,12 +78,10 @@ def test_ai_model_unknown_id_returns_null(self, authenticated_graphql_client):
             assert result["data"].get("aiModel") is None
 
     def test_ai_model_shape_with_valid_model(self, authenticated_graphql_client):
-        models_result = authenticated_graphql_client(TestGraphQL.QUERY_MY_MODELS)
-        models = ((models_result.get("data") or {}).get("myModels")) or []
-        if not models:
-            pytest.skip("No AI models available for this user")
+        model_id = first_org_model_id(authenticated_graphql_client)
+        if not model_id:
+            pytest.skip("No non-Opus AI models in the sandbox org")
 
-        model_id = models[0]["id"]
         result = authenticated_graphql_client(
             TestGraphQL.QUERY_AI_MODEL, variables={"modelId": str(model_id)}
         )

tests/api/test_evaluation_matrix_api.py

@@ -0,0 +1,64 @@
+"""Evaluation option matrix at the API layer + matrix definition self-check.
+
+- TestMatrixDefinition: pure data checks on tests/data/evaluation_matrix.py (no network).
+- TestEvaluationOptionRoundTrip: every evaluator x domain pair (9) is accepted by
+  updateAudit and read back unchanged (write; sandbox-gated; drafts are cancelled).
+"""
+
+from itertools import product
+
+import pytest
+
+from tests.data.evaluation_matrix import DOMAINS, EVALUATORS, _selfcheck, all_combos, combos_for
+from tests.data.test_data import TestGraphQL, first_org_model_id
+
+_AUDIT_Q = "query($id: ID!){ audit(auditId:$id){ id status auditType auditScope evaluationMode } }"
+
+
+@pytest.mark.smoke
+@pytest.mark.regression
+class TestMatrixDefinition:
+    def test_selfcheck_passes(self):
+        _selfcheck()
+
+    def test_full_space_is_135_unique_cases(self):
+        ids = [c.id for c in all_combos()]
+        assert len(ids) == len(set(ids)) == 135
+
+    def test_selection_is_opt_in(self):
+        assert combos_for("") == [] and len(combos_for("pairwise")) == 14
+
+
+@pytest.mark.api
+@pytest.mark.regression
+@pytest.mark.regression_write
+@pytest.mark.auth
+class TestEvaluationOptionRoundTrip:
+    @pytest.mark.parametrize("evaluator,domain", list(product(EVALUATORS, DOMAINS)))
+    def test_evaluator_and_scope_round_trip(
+        self, authenticated_graphql_client, sandbox_org, cleanup_evaluation, evaluator, domain
+    ):
+        client = authenticated_graphql_client
+        model_id = first_org_model_id(client)
+        if not model_id:
+            pytest.skip("No non-Opus org model available")
+        created = client(
+            TestGraphQL.MUTATION_CREATE_BLANK_AUDIT,
+            variables={"input": {"modelId": model_id}},
+            method="POST",
+        )
+        audit_id = str((((created.get("data") or {}).get("createBlankAudit") or {}).get("audit") or {}).get("id") or "")
+        assert audit_id, created
+        cleanup_evaluation.append(audit_id)
+
+        upd = client(
+            TestGraphQL.MUTATION_UPDATE_AUDIT,
+            variables={"input": {"auditId": audit_id, "auditType": evaluator, "auditScope": domain,
+                                 "evaluationMode": "bulk", "name": f"API matrix {evaluator}/{domain}"}},
+            method="POST",
+        )
+        assert ((upd.get("data") or {}).get("updateAudit") or {}).get("success"), upd
+
+        got = ((client(_AUDIT_Q, variables={"id": audit_id}).get("data") or {}).get("audit")) or {}
+        assert (got.get("auditType") or "").lower() == evaluator, got
+        assert (got.get("auditScope") or "").lower() == domain, got

tests/api/test_playground_api.py

@@ -10,7 +10,7 @@
 
 import pytest
 
-from tests.data.test_data import TestGraphQL, TestModelConstants
+from tests.data.test_data import TestGraphQL, TestModelConstants, first_org_model_id
 
 pytestmark = [pytest.mark.api, pytest.mark.regression, pytest.mark.auth]
 
@@ -245,9 +245,7 @@ class TestPlaygroundEvaluationCreate:
 
     @staticmethod
     def _first_model_id(client) -> str | None:
-        result = client(TestGraphQL.QUERY_MY_MODELS)
-        models = ((result.get("data") or {}).get("myModels")) or []
-        return str(models[0]["id"]) if models else None
+        return first_org_model_id(client)
 
     def test_create_blank_audit_returns_draft_audit(
         self, authenticated_graphql_client, sandbox_org, cleanup_evaluation
@@ -353,6 +351,8 @@ def test_run_audit_playground_returns_valid_status(
         resp = ((run_result.get("data") or {}).get("runAudit")) or {}
         if resp and resp.get("audit"):
             status = resp["audit"].get("status")
-            assert status in TestModelConstants.AUDIT_STATUSES, (
+            # A rejected run (e.g. no evaluation module selected) leaves the audit DRAFT.
+            allowed = TestModelConstants.AUDIT_STATUSES + ([] if resp.get("success") else ["DRAFT"])
+            assert status in allowed, (
                 f"runAudit returned unexpected status: {status!r}"
             )