diff --git a/src/fi/alk/harness/background_noise.py b/src/fi/alk/harness/background_noise.py new file mode 100644 index 00000000..9a908587 --- /dev/null +++ b/src/fi/alk/harness/background_noise.py @@ -0,0 +1,73 @@ +"""Choose the caller-side ambient noise a scenario should be heard through. + +A scenario that sets ``background_noise`` wants the agent to handle a caller phoning from somewhere +real: a car, a street, an office. The clip is chosen here and handed to the voice engine, which +mixes it under the simulated caller's audio. + +Two sources, in order. A run may point ``ALK_BACKGROUND_NOISE_CATALOG`` at a JSON file of clips +(each with an ``environment`` tag and a ``url`` or ``path``); the catalog stays a local file so its +asset locations are never committed here. When no catalog matches, a LiveKit builtin clip is used, +which needs no external asset and always works. +""" + +from __future__ import annotations + +import json +import os +from pathlib import Path + +# LiveKit ships these; they are the reliable default when no custom catalog is configured. +_BUILTIN_BY_ENVIRONMENT: dict[str, str] = { + "street": "CITY_AMBIENCE", + "transit": "CITY_AMBIENCE", + "vehicle": "CITY_AMBIENCE", + "outdoors": "FOREST_AMBIENCE", + "retail": "CROWDED_ROOM", + "office": "OFFICE_AMBIENCE", + "home": "OFFICE_AMBIENCE", +} +_DEFAULT_BUILTIN = "OFFICE_AMBIENCE" + + +def enabled() -> bool: + """Whether any scenario may be heard through background noise on this run. + + Off unless ``ALK_BACKGROUND_NOISE`` opts in, so a run needs no environment at all to be + silent. Continuous ambient audio under the caller competes with endpoint detection, and calls + carrying it end earlier and on fewer turns, so silence is the setting a run should fall into + rather than the one it has to ask for. Opting in still only permits noise: a scenario that + asked for none stays silent either way. + """ + return os.environ.get("ALK_BACKGROUND_NOISE", "0").strip().lower() in ( + "1", + "on", + "true", + "yes", + ) + + +def source_for(environment: str = "", seed: str = "") -> str: + """A background-noise source for a scenario. + + Returns a ``url``/``path`` from the configured catalog when one matches the environment, else the + name of a LiveKit builtin clip. The choice is deterministic in ``seed`` so the same scenario + hears the same place across runs. + """ + env = (environment or "").strip().lower() + catalog = os.environ.get("ALK_BACKGROUND_NOISE_CATALOG", "").strip() + if catalog and Path(catalog).is_file(): + try: + entries = json.loads(Path(catalog).read_text(encoding="utf-8")) + except (OSError, ValueError): + entries = [] + if isinstance(entries, list) and entries: + pool = [ + entry + for entry in entries + if str(entry.get("environment", "")).strip().lower() == env + ] or entries + chosen = pool[sum(ord(character) for character in (seed or env or "x")) % len(pool)] + located = str(chosen.get("url") or chosen.get("path") or "").strip() + if located: + return located + return _BUILTIN_BY_ENVIRONMENT.get(env, _DEFAULT_BUILTIN) diff --git a/src/fi/alk/harness/build.py b/src/fi/alk/harness/build.py index a6341e49..7a0db07b 100644 --- a/src/fi/alk/harness/build.py +++ b/src/fi/alk/harness/build.py @@ -26,6 +26,7 @@ permission_gate, provider_env, provisioning, + thinking_config, ) from .contract import AgentContract from .session import Stage @@ -202,6 +203,7 @@ def open_stage( options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(ask, allowed) + options.thinking = thinking_config() return Stage(options, name=SKILL), destination diff --git a/src/fi/alk/harness/config.py b/src/fi/alk/harness/config.py index 44e30634..2a5ebbe0 100644 --- a/src/fi/alk/harness/config.py +++ b/src/fi/alk/harness/config.py @@ -52,6 +52,24 @@ def chosen_model(model: str | None = None) -> str: return model or os.environ.get("ALK_HARNESS_MODEL", DEFAULT_MODEL) +def thinking_config() -> dict[str, Any]: + """How much the model may think, from ALK_HARNESS_THINKING. + + The Claude Code CLI defaults to adaptive thinking. In this harness the correctness of what a + stage produces is re-checked by code gates (a scenario is proved against the real world, a + contract is validated), so the model's private reasoning is spent on decisions the gates make + again anyway. Left unset, that reasoning was the majority of generated tokens and the majority + of wall time. Default to disabled for speed; ``adaptive`` restores the old behaviour, and an + integer sets an explicit budget for models that still honour one. + """ + setting = os.environ.get("ALK_HARNESS_THINKING", "disabled").strip().lower() + if setting in {"adaptive", "on", "auto"}: + return {"type": "adaptive", "display": "omitted"} + if setting.isdigit() and int(setting) > 0: + return {"type": "enabled", "budget_tokens": int(setting), "display": "omitted"} + return {"type": "disabled"} + + def provisioning(enabled: bool | None = None) -> bool: """Compatibility switch for callers selecting the legacy provisioning surface. @@ -75,10 +93,20 @@ def provider_env(model: str | None = None) -> dict[str, str]: Claude Code resolves the GCP project from ``GOOGLE_CLOUD_PROJECT``, the credential file, or the active gcloud configuration, in that order, so an unset project id is not an error here. """ + # Every model a session can reach is pinned to the same one. Naming only the main model + # leaves the sub-agent and fast-path settings to the CLI's own preference, and a suite written + # by twenty writers then runs on whatever that preference happens to be rather than on the + # model the run asked for. + chosen = chosen_model(model) env = { "CLAUDE_CODE_USE_VERTEX": "1", "CLOUD_ML_REGION": os.environ.get("CLOUD_ML_REGION", "global"), - "ANTHROPIC_MODEL": chosen_model(model), + "ANTHROPIC_MODEL": chosen, + "ANTHROPIC_DEFAULT_SONNET_MODEL": chosen, + "ANTHROPIC_DEFAULT_OPUS_MODEL": chosen, + "ANTHROPIC_DEFAULT_HAIKU_MODEL": chosen, + "ANTHROPIC_SMALL_FAST_MODEL": chosen, + "CLAUDE_CODE_SUBAGENT_MODEL": chosen, } for passthrough in ( "ANTHROPIC_VERTEX_PROJECT_ID", @@ -123,6 +151,7 @@ def read_only_session( options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(granted=allowed) + options.thinking = thinking_config() return options diff --git a/src/fi/alk/harness/data/persona_vocabulary.json b/src/fi/alk/harness/data/persona_vocabulary.json new file mode 100644 index 00000000..279b6729 --- /dev/null +++ b/src/fi/alk/harness/data/persona_vocabulary.json @@ -0,0 +1,111 @@ +{ + "GenderChoices": [ + "male", + "female" + ], + "AgeGroupChoices": [ + "18-25", + "25-32", + "32-40", + "40-50", + "50-60", + "60+" + ], + "LocationChoices": [ + "United States", + "Canada", + "United Kingdom", + "Australia", + "India" + ], + "ProfessionChoices": [ + "Student", + "Teacher", + "Engineer", + "Doctor", + "Nurse", + "Business Owner", + "Manager", + "Sales Representative", + "Customer Service", + "Technician", + "Consultant", + "Accountant", + "Marketing Professional", + "Retired", + "Homemaker", + "Freelancer", + "Other" + ], + "PersonalityChoices": [ + "Friendly and cooperative", + "Professional and formal", + "Cautious and skeptical", + "Impatient and direct", + "Detail-oriented", + "Easy-going", + "Anxious", + "Confident", + "Analytical", + "Emotional", + "Reserved", + "Talkative" + ], + "CommunicationStyleChoices": [ + "Direct and concise", + "Detailed and elaborate", + "Casual and friendly", + "Formal and polite", + "Technical", + "Simple and clear", + "Questioning", + "Assertive", + "Passive", + "Collaborative" + ], + "AccentChoices": [ + "American", + "Australian", + "Indian", + "Canadian", + "Neutral" + ], + "LanguageChoices": [ + "Arabic", + "Bulgarian", + "Chinese Simplified", + "Czech", + "Danish", + "Dutch", + "English", + "Finnish", + "French", + "German", + "Greek", + "Hindi", + "Hungarian", + "Indonesian", + "Italian", + "Japanese", + "Korean", + "Malay", + "Norwegian", + "Polish", + "Portuguese", + "Romanian", + "Russian", + "Slovak", + "Spanish", + "Swedish", + "Turkish", + "Ukrainian", + "Vietnamese" + ], + "ConversationSpeedChoices": [ + "0.5", + "0.75", + "1.0", + "1.25", + "1.5" + ] +} diff --git a/src/fi/alk/harness/persona_guides.py b/src/fi/alk/harness/persona_guides.py new file mode 100644 index 00000000..70cf780e --- /dev/null +++ b/src/fi/alk/harness/persona_guides.py @@ -0,0 +1,157 @@ +"""The persona values the platform understands. + +A persona field is only useful if the platform recognises what is in it: an accent it knows +selects a voice, a personality it knows attaches a sentence of behaviour guidance. A value +written in words of its own renders fine and then does nothing, which is how a suite ends up +with callers who all behave the same. + +The values are read from the platform's own model when it is mounted, and from the copy carried +with the harness when it is not, so a writer is always offered real ones. The behaviour guidance +itself lives with the prompt builder, next to the code that applies it. +""" + +from __future__ import annotations + +import ast +import json +import logging +import os +from functools import lru_cache +from pathlib import Path + +logger = logging.getLogger(__name__) + +# Where the platform's tables are mounted. Colon-separated so voice and chat guides can both be +# offered; the first file defining a table wins, so voice takes precedence when both are present. +# Where the platform's persona model is mounted, for the values it accepts. +VOCABULARY_ENV = "HARNESS_PERSONA_VOCABULARY" + +# The persona fields worth constraining, and the choice class each is drawn from. Only the ones +# that change behaviour or routing: a free-text occupation harms nothing, an accent nobody +# recognises silently loses the voice it was supposed to select. +FIELDS = { + "gender": "GenderChoices", + "age_group": "AgeGroupChoices", + "occupation": "ProfessionChoices", + "location": "LocationChoices", + "personality": "PersonalityChoices", + "communication_style": "CommunicationStyleChoices", + "accent": "AccentChoices", + "languages": "LanguageChoices", +} + +# Constrained because something downstream reads them. The rest are offered as vocabulary but a +# writer who needs a value outside them is not stopped: an unknown occupation costs nothing, +# an unknown accent costs the voice. +ENFORCED = ("personality", "communication_style", "accent", "languages") + + +@lru_cache(maxsize=1) +def vocabulary() -> dict[str, list[str]]: + """What the platform accepts for each persona field. + + Parsed out of the model's ``TextChoices`` classes for the same reason the guidance is read + rather than restated: the platform is the one that has to understand these values, so it is + the one that decides what they are. A persona written in words of its own renders fine, gets + no behaviour guidance, and cannot be grouped with anything on the platform afterwards. + """ + path = os.environ.get(VOCABULARY_ENV) or "" + if not path or not Path(path).exists(): + # No model mounted. Fall back to the copy carried with the harness so a writer is always + # offered real values: an empty vocabulary silently lets it invent an accent that selects + # no voice and a personality that attaches no guidance. + return _bundled_vocabulary() + try: + tree = ast.parse(Path(path).read_text(encoding="utf-8")) + except (OSError, SyntaxError): + logger.warning("persona vocabulary at %s is unreadable; using the bundled copy", path) + return _bundled_vocabulary() + + by_class: dict[str, list[str]] = {} + for node in ast.walk(tree): + if not isinstance(node, ast.ClassDef): + continue + values: list[str] = [] + for item in node.body: + if not isinstance(item, ast.Assign): + continue + try: + held = ast.literal_eval(item.value) + except ValueError: + continue + # ``NAME = "value", "Label"`` is the choices shape; a bare string is also accepted. + if isinstance(held, tuple) and held and isinstance(held[0], str): + values.append(held[0]) + elif isinstance(held, str): + values.append(held) + if values: + by_class[node.name] = values + + found = { + field: by_class[cls] for field, cls in FIELDS.items() if by_class.get(cls) + } + if not found: + # The file parsed but held none of the classes we key on, so it is the wrong file or the + # classes moved. Silently returning nothing would drop every persona constraint at once. + logger.warning( + "persona vocabulary at %s defines none of %s; using the bundled copy", + path, + ", ".join(sorted(set(FIELDS.values()))), + ) + return _bundled_vocabulary() + return found + + + +@lru_cache(maxsize=1) +def _bundled_vocabulary() -> dict[str, list[str]]: + """The platform's persona values, carried with the harness. + + Kept so the harness constrains personas out of the box. Languages come from the agent + definition's set rather than the persona dropdown's two, because nothing on the platform + enforces the dropdown and a caller is expected to speak more than English and Hindi. + """ + path = Path(__file__).parent / "data" / "persona_vocabulary.json" + try: + by_class = json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + logger.warning("bundled persona vocabulary is unreadable; personas stay unconstrained") + return {} + return { + field: list(by_class[cls]) + for field, cls in FIELDS.items() + if by_class.get(cls) + } + + +def offered(field: str) -> list[str]: + """The values this field accepts, or nothing if the platform's model was not readable.""" + return list(vocabulary().get(field, [])) + + +def unrecognised(persona: dict[str, object]) -> list[str]: + """Persona values the platform would not recognise, as sentences saying what to use instead. + + Only the fields something downstream actually reads, and only when the vocabulary was found: + a harness that cannot see the platform's model must not start refusing personas over it. + """ + known = vocabulary() + if not known: + return [] + problems: list[str] = [] + for field in ENFORCED: + allowed = known.get(field) or [] + if not allowed: + continue + held = persona.get(field) + values = held if isinstance(held, list) else ([held] if held else []) + lowered = {str(one).strip().lower() for one in allowed} + for one in values: + text = str(one).strip() + if text and text.lower() not in lowered: + problems.append( + f"persona {field} {text!r} is not one the platform knows, so it will not " + f"reach the call. Use one of: {', '.join(allowed)}. Anything else this " + "person is like belongs in persona.metadata." + ) + return problems diff --git a/src/fi/alk/harness/platform.py b/src/fi/alk/harness/platform.py index 3d8a4169..5fe4a4a9 100644 --- a/src/fi/alk/harness/platform.py +++ b/src/fi/alk/harness/platform.py @@ -275,7 +275,11 @@ def persona_of(scenario: Any) -> dict[str, Any]: "name": str(persona.get("name") or getattr(scenario, "name", "") or "caller")[ :255 ], - "scenario_key": str(getattr(scenario, "name", "") or "")[:255], + # The scenario's own key, not its folder name: the key is ASCII-sanitised and falls back + # to a digest, which the name does not, and this value travels as an HTTP header. + "scenario_key": str( + getattr(scenario, "scenario_key", "") or getattr(scenario, "name", "") or "" + )[:255], "scenario_name": display_scenario_name(scenario), "role": str(persona.get("role") or persona.get("occupation") or "")[:255], "situation": str(getattr(scenario, "instruction", "") or ""), diff --git a/src/fi/alk/harness/reception.py b/src/fi/alk/harness/reception.py index 266f9252..7e858380 100644 --- a/src/fi/alk/harness/reception.py +++ b/src/fi/alk/harness/reception.py @@ -25,6 +25,7 @@ gate_hooks, permission_gate, provider_env, + thinking_config, ) from .session import Stage from .sources import AgentSource, clone_github_repository, resolve, supported @@ -150,6 +151,7 @@ async def point_at_agent(args: dict[str, Any]) -> dict[str, Any]: options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(ask, allowed) + options.thinking = thinking_config() return Stage(options, name="reception"), found diff --git a/src/fi/alk/harness/run/call.py b/src/fi/alk/harness/run/call.py index 7d9932dd..afe1e52b 100644 --- a/src/fi/alk/harness/run/call.py +++ b/src/fi/alk/harness/run/call.py @@ -107,7 +107,12 @@ def main(argv: list[str] | None = None) -> int: # about how a simulated caller behaves is decided twice. os.environ["HARNESS_INSTRUCTION"] = instruction os.environ["HARNESS_SCENARIO"] = scenario.name - os.environ["HARNESS_OUTCOME"] = scenario.tests + # The caller prompt is a template the harness fills, never prose it composes, so + # the generated template travels to the call with everything else. + # The caller is never handed the grader's pass question. `tests` is written about + # the agent in the third person, so as an objective it reads as a rubric rather + # than a motive. What this person wants is already in the instruction. + os.environ.pop("HARNESS_OUTCOME", None) os.environ["HARNESS_PERSONA"] = json.dumps( scenario.persona.model_dump(exclude_none=True) if scenario.persona is not None diff --git a/src/fi/alk/harness/run/data/voices_by_language_and_gender.json b/src/fi/alk/harness/run/data/voices_by_language_and_gender.json new file mode 100644 index 00000000..5daedb1e --- /dev/null +++ b/src/fi/alk/harness/run/data/voices_by_language_and_gender.json @@ -0,0 +1,693 @@ +{ + "en": { + "female": [ + "6ccbfb76-1fc6-48f7-b71d-91ac6298247b", + "e07c00bc-4134-4eae-9ea4-1a55fb45746b", + "f786b574-daa5-4673-aa0c-cbe3e8534c02", + "9626c31c-bec5-4cca-baa8-f8ba9e84c8bc", + "f9836c6e-a0bd-460e-9d3c-f7299fa60f94", + "829ccd10-f8b3-43cd-b8a0-4aeaa81f3b30", + "ec1e269e-9ca0-402f-8a18-58e0e022355a", + "66c6b81c-ddb7-4892-bdd5-19b5a7be38e7", + "a7b8d8fa-f6e5-4908-900e-0c11d1d82519", + "999df508-4de5-40a7-8bd3-8c12f678c284", + "26403c37-80c1-4a1a-8692-540551ca2ae5", + "694f9389-aac1-45b6-b726-9d9369183238", + "248be419-c632-4f23-adf1-5324ed7dbf1d", + "bf0a246a-8642-498a-9950-80c35e9276b5", + "57dcab65-68ac-45a6-8480-6c4c52ec1cd1", + "78ab82d5-25be-4f7d-82b3-7ad64e5b85b2", + "03496517-369a-4db1-8236-3d3ae459ddf7", + "e8e5fffb-252c-436d-b842-8879b84445b6", + "b7d50908-b17c-442d-ad8d-810c63997ed9", + "32b3f3c5-7171-46aa-abe7-b598964aa793", + "00a77add-48d5-4ef6-8157-71e5437b282d", + "4af7c703-f2a9-45dd-a7fd-724cf7efc371", + "156fb8d2-335b-4950-9cb3-a2d33befec77", + "8d8ce8c9-44a4-46c4-b10f-9a927b99a853", + "c2ac25f9-ecc4-4f56-9095-651354df60c0", + "5c42302c-194b-4d0c-ba1a-8cb485c84ab9", + "3b554273-4299-48b9-9aaf-eefd438e3941", + "71a7ad14-091c-4e8e-a314-022ece01c121", + "e3827ec5-697a-4b7c-9704-1a23041bbc51", + "8f091740-3df1-4795-8bd9-dc62d88e5131", + "5abd2130-146a-41b1-bcdb-974ea8e19f56", + "91b4cf29-5166-44eb-8054-30d40ecc8081", + "f6ff7c0c-e396-40a9-a70b-f7607edb6937", + "11af83e2-23eb-452f-956e-7fee218ccb5c", + "e13cae5c-ec59-4f71-b0a6-266df3c9bb8e", + "a01c369f-6d2d-4185-bc20-b32c225eab70", + "7ea5e9c2-b719-4dc3-b870-5ba5f14d31d8", + "f8f5f1b2-f02d-4d8e-a40d-fd850a487b3d", + "a38e4e85-e815-43ab-acf1-907c4688dd6c", + "f31cc6a7-c1e8-4764-980c-60a361443dd1", + "21b81c14-f85b-436d-aff5-43f2e788ecf8", + "f6141af3-5f94-418c-80ed-a45d450e7e2e", + "8985388c-1332-4ce7-8d55-789628aa3df4", + "043cfc81-d69f-4bee-ae1e-7862cb358650", + "c8605446-247c-4d39-acd4-8f4c28aa363c", + "607167f6-9bf2-473c-accc-ac7b3b66b30b", + "cccc21e8-5bcf-4ff0-bc7f-be4e40afc544", + "55deba52-bc73-4481-ab69-9c8831c8a7c3", + "996a8b96-4804-46f0-8e05-3fd4ef1a87cd", + "bf991597-6c13-47e4-8411-91ec2de5c466", + "daf747c6-6bc2-4083-bd59-aa94dce23f5d", + "c9440d34-5641-427b-bbb7-80ef7462576d", + "56b87df1-594d-4135-992c-1112bb504c59", + "0c8ed86e-6c64-40f0-b252-b773911de6bb", + "f4e8781b-a420-4080-81cf-576331238efa", + "57b6bf63-c7a1-4ffc-8e10-23bf45152dd6", + "5e10a334-7fa5-46d4-a64b-5ae6185da3fd", + "761afc95-bef5-44dd-aa07-d3c678912e43", + "f9fc912e-52f0-448a-8bfa-47e9ca75f25a", + "2747b6cf-fa34-460c-97db-267566918881", + "af346552-54bf-4c2b-a4d4-9d2820f51b6c", + "d3e03deb-5439-4203-add1-ca9a7501eaa7", + "04bfd756-4fd4-42c2-9ccf-37f647c5bf54", + "ca566b43-944e-4474-b494-7d9f0695f307", + "4d3d2e9c-14e4-4802-a8d8-bd5268a73fde", + "8634bd27-0acf-4056-b014-4fea0385ed9e", + "b56c6aac-f35f-46f7-9361-e8f078cec72e", + "f0377496-2708-4cc9-b2f8-1b7fdb5e1a2a", + "0a9a5903-0a30-4d2e-b6b6-891f73d4b4e0", + "f6ce3444-478b-4ce4-982e-bcb72dffe7aa", + "0d2162c2-2fe9-40a7-b3c1-43eab576a64b", + "cb6a8744-41b0-4cdc-b643-fabeb545c6a9", + "e4d5f4c4-6601-4779-bee1-b3c14d629dc6", + "3d9b50f9-10c5-4026-9ae1-c4a698f67fc5", + "eb649460-7e23-43bc-ad20-0a7a2749b938", + "1f575487-6f3d-40e0-862a-814f55b5fb15", + "050f5a7a-9d2b-4b76-84e3-2d056a0a3eb0", + "6fbca103-0f7f-4e49-97ed-49a53b4f3534", + "87041166-c212-4838-9028-05d7437df750", + "9329fbdb-e285-4fba-95ec-592e15f14476", + "eef47c0d-cb49-4160-a4a0-6b97ed4c81e6", + "69092565-1c93-4a88-9f2c-ac8cddaf9f65", + "d6b0c62a-c7ff-477c-9a1f-eadd64b94360", + "80c81aee-b6ad-4d12-9af8-a9c79c2e141d", + "ca31ce53-ebf6-4e51-b87d-2f65d5d1f7f8", + "aef96ff9-4578-4b5d-9744-7fb347cbe4d4", + "643f5eee-459d-4b41-b4fc-0b8407139be6", + "dcc82bcd-647e-4478-955f-8232d5122f8b", + "045f0292-0731-4a4c-971d-64594fc2c35a", + "86600680-b836-41e1-9916-8475728dcc14", + "b5c1bab5-f036-481f-9295-4db6f06f6443", + "4b1e0bf9-53a0-4e9e-8664-ba1314dbcb38", + "e5a6cd18-d552-4192-9533-82a08cac8f23", + "ea93f57f-7c71-4d79-aeaa-0a39b150f6ca", + "63927f41-9616-4ac2-89cf-f3afa346e0ef", + "3308b492-50cc-417e-89dd-1f446c574546", + "320f7211-3dc3-4292-89b1-3661e8cac27c", + "a2364c9d-1fe3-4553-9eff-100c4fe5ffc8", + "48369ca9-0645-40de-9821-0d55e18a03c2", + "d6905573-8e91-4e32-b103-fd4d1205cd87", + "1ac31ebd-9113-405b-9d80-4a4bbbeea91c", + "3f38cbe2-ce6a-4051-b5dc-2b2ee20b9bc1", + "083de431-6b5c-4b18-a2dc-264eafa205f2", + "cec7cae1-ac8b-4a59-9eac-ec48366f37ae", + "8a1b8af0-c4f6-423f-a268-5507fd4aefdf", + "19e399df-5b30-4fba-9d1d-99434f993614", + "efc5488b-5429-4e72-aaa2-570981cf47d9", + "cc00e582-ed66-4004-8336-0175b85c85f6", + "3af40927-948e-429b-b92d-e2158f79fb9f", + "64b2a604-f0de-449f-9d90-255602357c05", + "c7c790c5-2bf4-47e4-bc83-5f43e61f3803", + "f4c1a0b2-669d-403f-b440-4b34b34856aa", + "cbaf8084-f009-4838-a096-07ee2e6612b1", + "c1b9a03e-747f-40ad-8e7b-18caf8aaac0b", + "e2d08065-b658-466b-ad52-cef8ee21d307", + "f762e181-ddc7-486e-9a48-636bd7e229d4", + "3ef78ba6-9aaa-46a2-b5b5-f9ded76a2370", + "a7a59115-2425-4192-844c-1e98ec7d6877", + "f39d8500-0d9b-4b8b-a080-38f5188f5892", + "1b4ea5fb-b1c0-43ee-a7be-4e315878c2b1", + "01eaafa9-308a-4276-a017-6ab0cf061b1f", + "03b1c65d-4b7f-4c09-91a8-e2f6f78cb2c9", + "4e41a434-85fc-4614-b203-af79ba44d473", + "09ed0318-2f4a-41b1-abe5-d11da7537c31", + "8918ddfe-2ad4-4cc8-a573-e020ca13f3f5", + "46788d8e-cdf9-4d5c-9125-094eb2e4d44c", + "f80e7298-93f5-46d0-86f2-b8f29cfc88bd", + "1242fb95-7ddd-44ac-8a05-9e8a22a6137d", + "02fe5732-a072-4767-83e3-a91d41d274ca", + "fb78f09f-f998-4061-ad51-d71f90388f0e", + "c2da2a3e-b0d6-46bf-a09a-68562617a50a", + "ba0add52-783c-4ec0-8b9c-7a6b60f99d1c", + "8843adfb-77d3-455a-86f9-de0651555ec6", + "5cc54223-ec0c-4c50-87e9-b9947264e1f4", + "57c63422-d911-4666-815b-0c332e4d7d6a", + "414da90b-16b3-4e88-86f5-3c3945e8fa4b", + "2d01710c-7c77-4cf1-b0d0-5902a25f6e17", + "a5def41e-2e73-433f-92f7-5f1d99fef05d", + "98c87826-dba2-44f4-b123-4c7e3c8a2647", + "62305e79-9d39-4643-b003-5e0b096fe4f4", + "5993c2c9-5d59-403e-b459-946c8b302086", + "30236d07-62d0-4c63-abf7-df46aa45e473", + "27c12970-3efb-4f39-a78a-2fbb7bddc941", + "134838f5-ce7e-4876-ac32-6367b99daf83" + ], + "male": [ + "228fca29-3a0a-435c-8728-5cb483251068", + "5ee9feff-1265-424a-9d7f-8e4d431a12c7", + "5cad89c9-d88a-4832-89fb-55f2f16d13d3", + "41468051-3a85-4b68-92ad-64add250d369", + "c961b81c-a935-4c17-bfb3-ba2239de8c2f", + "a167e0f3-df7e-4d52-a9c3-f949145efdab", + "79f8b5fb-2cc8-479a-80df-29f7a7cf1a3e", + "146485fd-8736-41c7-88a8-7cdd0da34d84", + "565510e8-6b45-45de-8758-13588fbaec73", + "98a34ef2-2140-4c28-9c71-663dc4dd7022", + "1463a4e1-56a1-4b41-b257-728d56e93605", + "ed81fd13-2016-4a49-8fe3-c0d2761695fc", + "34575e71-908f-4ab6-ab54-b08c95d6597d", + "00967b2f-88a6-4a31-8153-110a92134b9f", + "729651dc-c6c3-4ee5-97fa-350da1f88600", + "820a3788-2b37-4d21-847a-b65d8a68c99a", + "a0e99841-438c-4a64-b679-ae501e7d6091", + "c99d36f3-5ffd-4253-803a-535c1bc9c306", + "9fa83ce3-c3a8-4523-accc-173904582ced", + "d46abd1d-2d02-43e8-819f-51fb652c1c61", + "638efaaa-4d0c-442e-b701-3fae16aad012", + "e00d0e4c-a5c8-443f-a8a3-473eb9a62355", + "42b39f37-515f-4eee-8546-73e841679c1d", + "41534e16-2966-4c6b-9670-111411def906", + "1259b7e3-cb8a-43df-9446-30971a46b8b0", + "4df027cb-2920-4a1f-8c34-f21529d5c3fe", + "1fc31370-81b1-4588-9c1a-f93793c6e01d", + "87bc56aa-ab01-4baa-9071-77d497064686", + "f114a467-c40a-4db8-964d-aaba89cd08fa", + "701a96e1-7fdd-4a6c-a81e-a4a450403599", + "3e1ed423-17e5-4773-b87c-25b031106e41", + "da4a4eff-3b7e-4846-8f70-f075ff61222c", + "23e9e50a-4ea2-447b-b589-df90dbb848a2", + "ee7ea9f8-c0c1-498c-9279-764d6b56d189", + "97f4b8fb-f2fe-444b-bb9a-c109783a857a", + "4f7f1324-1853-48a6-b294-4e78e8036a83", + "7cf0e2b1-8daf-4fe4-89ad-f6039398f359", + "87748186-23bb-4158-a1eb-332911b0b708", + "13524ffb-a918-499a-ae97-c98c7c4408c4", + "7e19344f-9f17-47d7-a13a-4366ad06ebf3", + "3246e36c-ac8c-418d-83cd-4eaad5a3b887", + "2a4d065a-ac91-4203-a015-eb3fc3ee3365", + "40104aff-a015-4da1-9912-af950fbec99e", + "86e30c1d-714b-4074-a1f2-1cb6b552fb49", + "41f3c367-e0a8-4a85-89e0-c27bae9c9b6d", + "c45bc5ec-dc68-4feb-8829-6e6b2748095d", + "726d5ae5-055f-4c3d-8355-d9677de68937", + "96c64eb5-a945-448f-9710-980abe7a514c", + "39b376fc-488e-4d0c-8b37-e00b72059fdd", + "bbee10a8-4f08-4c5c-8282-e69299115055", + "0b32066b-2bcc-44b9-89ab-0223a09d1606", + "bfd3644b-d561-4b1c-a01f-d9af98cb67c0", + "5619d38c-cf51-4d8e-9575-48f61a280413", + "34d923aa-c3b5-4f21-aac7-2c1f12730d4b", + "5c43e078-5ba4-4e1f-9639-8d85a403f76a", + "d7862948-75c3-4c7c-ae28-2959fe166f49", + "6a176356-ada1-4b48-b2ae-3a3fdd485680", + "66f5935b-af2e-4ec9-bb3e-59112e9ddc93", + "ee8b13e7-98af-4b15-89d1-8d402be10c94", + "1cb5b8bc-77c9-4e7c-a251-da02348e2727", + "f24ae0b7-a3d2-4dd1-89df-959bdc4ab179", + "db69127a-dbaf-4fa9-b425-2fe67680c348", + "5fb68a42-0ed7-46fa-8a8f-ad4b332fbf6f", + "b134c304-d095-4d2b-a77a-914f5e8e84e7", + "74f42072-6245-4fe2-b5dc-3dc9b56fdbd0", + "373e661a-f0ef-4e34-a09e-183184a443e6", + "9301949d-b7cd-40d9-a246-5a4430992d6b", + "e39b9fc0-23f5-4616-962a-da99c8ccb1dc", + "01fd7d67-d2a0-4e4e-8c48-42611c71a926", + "df872fcd-da17-4b01-a49f-a80d7aaee95e", + "6cb8801d-259a-4bdc-978f-b45808d58cd3", + "efa653e5-314d-46ca-9f90-70ac7d6ca71e", + "afb19d1b-4044-4f34-a962-f4aef640a002", + "c58bda25-abd5-4c72-97a2-4dbe049b368d", + "f688c0a6-dddd-48ba-8246-c099d494a162", + "a924b0e6-9253-4711-8fc3-5cb8e0188c94", + "6fccb471-26f7-4f7a-93dd-542935db6c20", + "17488b72-f815-44d8-bdd9-869971c3ec06", + "59697755-8cfb-4ccf-9da4-f2201d06b067", + "b58b6b46-1a27-46ba-8648-bc203a5d394e", + "3bf35adc-bcc4-464b-b834-c90c88cf6492", + "9c8880b2-ccf9-4730-b805-cea23df247d7", + "5cf0e4d9-ca2b-4fd5-81fa-89db3b645539", + "c0f43c66-9f21-4034-b485-8f1d3340d759", + "2948c301-9211-4112-8f36-4c3fc836ef12", + "49808e4c-998a-40a8-b2ea-8ac8e8ce779e", + "7a8ae0b6-504a-49af-92d3-4e7e2eb84ca1", + "cd6256ef-2b2a-41f6-a8d8-c1307af5061f", + "3ccc4544-84f7-45e3-ae57-5c52b5a1fac6", + "18f8d87b-0da9-4efa-b504-4580e303f7db", + "fdf6303b-4cfa-4f8e-b7ae-acb398984cf9", + "ea7c252f-6cb1-45f5-8be9-b4f6ac282242", + "2d5b8c3a-116c-4741-acaf-ba4fa289eba2", + "356f4a89-d056-4e2e-8c73-865fa4d3af0a", + "23112795-d54e-4560-9568-791a87c30201", + "1628cfcd-a161-4e47-98ff-46bffa4ab290", + "a892d232-f705-40d7-bc8d-e368b295ec2a", + "3d83e30f-c31b-4f26-b442-7075feafa53a", + "87a983d8-3471-4c4b-9ade-f1d10a4110ac", + "b2222537-1561-4425-8c3c-e1aca96ad853", + "8cbfe3ab-8364-4e72-b606-93f749519c66", + "d2c66146-c1c8-4c3a-9870-38e5a6b72442", + "5319c0b1-3dd1-4c00-b721-bfd2ec88ef56", + "f4a3a8e4-694c-4c45-9ca0-27caf97901b5", + "ed82c17b-4704-4d34-be43-5d19065acdf1", + "bbc5d060-50e1-45a3-87ff-191b8cea3092", + "b9cf5ec3-eaa4-46a5-a5b2-b0d0f22395a2", + "4c2dcd38-5608-45ca-8f11-51c88208d01c", + "90c896fa-aaa1-41af-a612-5267636440a3", + "d709a7e8-9495-4247-aef0-01b3207d11bf", + "1ce291a1-0771-4732-a3f7-8cca29bf055f", + "dbfa416f-d5c3-4006-854b-235ef6bdf4fd", + "6776173b-fd72-460d-89b3-d85812ee518d", + "921034a2-aace-4ef7-87b1-b9bc455c9a15", + "c78dd7ae-6692-4c44-a2a2-834e365afe60", + "0834f3df-e650-4766-a20c-5a93a43aa6e3", + "4cf80313-54dc-4ca9-a17c-3e5b8f68a78c", + "8d7d11ff-d985-48a2-a737-1da0b6fedc8b", + "3f04e815-3260-4f50-8fd9-af9c657be4c2", + "9a0894a9-28f0-436e-9a1d-e92bccbce4dd", + "710feaa3-b550-42f3-b3eb-6f37f2a7cc0a", + "0d42f0f6-c019-4082-b250-1c16133d1c82", + "efd255c7-f030-43d3-b5d8-c7b72063be70", + "7edf9efb-58fc-46ba-a648-3a00a86b111b", + "92c41dd4-04aa-45de-8504-a92b40cb8818", + "2f22b9bc-b0eb-4cb6-b5ae-0c099a0fdfad", + "79bfcec0-720c-41f2-a33a-f12383e9627f", + "e2d48e7b-cd73-4c4c-bc1e-f232580e8709", + "c63361f8-d142-4c62-8da7-8f8149d973d6", + "9287676d-f0cc-423f-ac03-3b3c7242f091", + "da69d796-4603-4419-8a95-293bfc5679eb", + "f96dc0b1-7900-4894-a339-81fb46d515a7", + "c1c65fc2-528a-4dde-a2c4-f822785c2704", + "b1ce5126-4d08-42c3-adef-d3eb39e90c7a", + "adde00e9-c98f-42ae-a94d-fc9f92f11c76", + "9fb269e7-70fe-4cbe-aa3f-28bdb67e3e84", + "80713a53-e484-4f69-9852-7891096016ac", + "7c8ba972-4960-4c43-bea0-8178e2205696", + "6fd4f468-0345-4f41-81d0-3f48ebc295e0", + "fd098a10-ba9e-445e-b144-be2a9f3dac02", + "c4e848dc-d4fd-4bc8-90ea-8525563ec0e5", + "b08c966e-2146-4592-99eb-3171a714a43c", + "a3a4fe2a-d402-41d1-be7d-28f71eda755f", + "9d2b4a7f-7ced-4fb8-b570-9ce21fb931c8", + "6b622a1d-906f-44af-b60c-7bef365bf124", + "10d17ae0-8f64-472a-be00-f00a98c729e0", + "8e14933d-ecd7-402b-9505-795130d69b35", + "7b2c0a2e-3dd3-4a44-b16b-26ecd8134279", + "79b8126f-c5d9-4a73-8585-ba5e1a077ed6", + "725d43d6-1196-480e-bd87-728ae5eff9e1", + "63426c82-a0c9-4f23-a175-50eb64c95ec1", + "61001bc6-9064-40a4-b8b2-29178e0fa558", + "5c7b66c2-3b58-464d-8a12-093410a269c5", + "3d79b1fd-daaa-439c-bff3-903dc18e7684", + "911b8b22-887f-4caf-bf87-85d834c08708" + ] + }, + "es": { + "female": [ + "5c5ad5e7-1020-476b-8b91-fdcbe9cc313c", + "cefcb124-080b-4655-b31f-932f3ee743de", + "c0c374aa-09be-42d9-9828-4d2d7df86962", + "d4db5fb9-f44b-4bd1-85fa-192e0f0d75f9", + "e9f0368b-3662-4a01-b037-e13ca5203c74", + "727f663b-0e90-4031-90f2-558b7334425b" + ], + "male": [ + "15d0c2e2-8d29-44c3-be23-d585d5f154a1", + "2695b6b5-5543-4be1-96d9-3967fb5e7fec", + "b5aa8098-49ef-475d-89b0-c9262ecf33fd", + "b042270c-d46f-4d4f-8fb0-7dd7c5fe5615" + ] + }, + "hi": { + "female": [ + "faf0731e-dfb9-4cfc-8119-259a79b27e12", + "95d51f79-c397-46f9-b49a-23763d3eaa2d", + "28ca2041-5dda-42df-8123-f58ea9c3da00", + "9cebb910-d4b7-4a4a-85a4-12c79137724c", + "bec003e2-3cb3-429c-8468-206a393c67ad", + "f91ab3e6-5071-4e15-b016-cde6f2bcd222", + "209d9a43-03eb-40d8-a7b7-51a6d54c052f", + "56e35e2d-6eb6-4226-ab8b-9776515a7094" + ], + "male": [ + "fd2ada67-c2d9-4afe-b474-6386b87d8fc3", + "be79f378-47fe-4f9c-b92b-f02cefa62ccf", + "9b953e7b-86a8-42f0-b625-1434fb15392b", + "bdab08ad-4137-4548-b9db-6142854c7525", + "393dd459-f8d8-4c3e-a86b-ec43a1113d0b", + "791d5162-d5eb-40f0-8189-f19db44611d8" + ] + }, + "de": { + "female": [ + "b9de4a89-2257-424b-94c2-db18ba68c81a", + "4ab1ff51-476d-42bb-8019-4d315f7c0c05", + "38aabb6a-f52b-4fb0-a3d1-988518f4dc06", + "3f4ade23-6eb4-4279-ab05-6a144947c4d5", + "1ade29fc-6b82-4607-9e70-361720139b12", + "6d4b1416-8d54-4d94-a788-8a802c086544" + ], + "male": [ + "384b625b-da5d-49e8-a76d-a2855d4f31eb", + "e00dd3df-19e7-4cd4-827a-7ff6687b6954", + "afa425cf-5489-4a09-8a3f-d3cb1f82150d", + "db229dfe-f5de-4be4-91fd-7b077c158578", + "b7187e84-fe22-4344-ba4a-bc013fcb533e", + "2be00b67-d53f-4eb5-89e7-96c224d56fbc" + ] + }, + "fr": { + "female": [ + "a8a1eb38-5f15-4c1d-8722-7ac0f329727d", + "65b25c5d-ff07-4687-a04c-da2f43ef6fa9", + "8832a0b5-47b2-4751-bb22-6a8e2149303d", + "6c64b57a-bc65-48e4-bff4-12dbe85606cd" + ], + "male": [ + "0418348a-0ca2-4e90-9986-800fb8b3bbc0", + "5c3c89e5-535f-43ef-b14d-f8ffe148c1f0", + "ab7c61f5-3daa-47dd-a23b-4ac0aac5f5c3", + "56df0456-8f47-4f7a-ac26-40c2f9797104" + ] + }, + "it": { + "female": [ + "d718e944-b313-4998-b011-d1cc078d4ef3", + "d609f27f-f1a4-410f-85bb-10037b4fba99", + "0e21713a-5e9a-428a-bed4-90d410b87f13", + "36d94908-c5b9-4014-b521-e69aee5bead0" + ], + "male": [ + "408daed0-c597-4c27-aae8-fa0497d644bf", + "e019ed7e-6079-4467-bc7f-b599a5dccf6f", + "79693aee-1207-4771-a01e-20c393c89e6f", + "029c3c7a-b6d9-44f0-814b-200d849830ff", + "88b329db-85d7-47cc-a5c5-98225a756721" + ] + }, + "pl": { + "male": [ + "3d335974-4c4a-400a-84dc-ebf4b73aada6", + "2a3503b2-b6b6-4534-a224-e8c0679cec4a", + "887149a8-4616-42ad-b2ce-c3819176f45d" + ], + "female": [ + "dcf62f33-7cff-4f20-85b2-2efaa68cbc32", + "575a5d29-1fdc-4d4e-9afa-5a9a71759864", + "ea7b5eee-39d9-40b0-b241-1910cbca9c62" + ] + }, + "ru": { + "female": [ + "064b17af-d36b-4bfb-b003-be07dba1b649", + "642014de-c0e3-4133-adc0-36b5309c23e6", + "779673f3-895f-4935-b6b5-b031dc78b319", + "9ed9f7e7-3ef6-4773-9dd3-ffcb479ca1f0" + ], + "male": [ + "888b7df4-e165-4852-bfec-0ab2b96aaa46" + ] + }, + "pt": { + "female": [ + "1cf751f6-8749-43ab-98bd-230dd633abdb", + "700d1ee3-a641-4018-ba6e-899dcadc9e2b", + "d4b44b9a-82bc-4b65-b456-763fce4c52f9", + "f39bf583-3b3d-402f-9ffb-6179d9ec3e35" + ], + "male": [ + "6a360542-a117-4ed5-9e09-e8bf9b05eabb", + "fbee0e7d-a83a-4082-bad1-13c70f86da4e" + ] + }, + "ja": { + "female": [ + "c7eafe22-8b71-40cd-850b-c5a3bbd8f8d2", + "59d4fd2f-f5eb-4410-8105-58db7661144f", + "2b568345-1d48-4047-b25f-7baccf842eb0", + "31c55968-a9f4-4115-8831-3a16952179c8" + ], + "male": [ + "e8a863c6-22c7-4671-86ca-91cacffc038d", + "6b92f628-be90-497c-8f4c-3b035002df71", + "b8e1169c-f16a-4064-a6e0-95054169e553" + ] + }, + "ko": { + "female": [ + "304fdbd8-65e6-40d6-ab78-f9d18b9efdf9", + "15628352-2ede-4f1b-89e6-ceda0c983fbc" + ] + }, + "zh": { + "female": [ + "7a5d4663-88ae-47b7-808e-8f9b9ee4127b", + "bf32f849-7bc9-4b91-8c62-954588efcc30", + "f9a4b3a6-b44b-469f-90e3-c8e19bd30e99", + "a53c3509-ec3f-425c-a223-977f5f7424dd" + ], + "male": [ + "eda5bbff-1ff1-4886-8ef1-4e69a77640a0", + "c59c247b-6aa9-4ab6-91f9-9eabea7dc69e", + "653b9445-ae0c-4312-a3ce-375504cff31e", + "16212f18-4955-4be9-a6cd-2196ce2c11d1" + ] + }, + "tr": { + "female": [ + "fa7bfcdc-603c-4bf1-a600-a371400d2f8c", + "bb2347fe-69e9-4810-873f-ffd759fe8420", + "0f95596c-09c4-4418-99fe-5c107e0713c0" + ], + "male": [ + "39f753ef-b0eb-41cd-aa53-2f3c284f948f", + "c1cfee3d-532d-47f8-8dd2-8e5b2b66bf1d", + "5a31e4fb-f823-4359-aa91-82c0ae9a991c", + "91e91d74-8eb4-43cd-97d3-7466c21db00d" + ] + }, + "sv": { + "female": [ + "f852eb8d-a177-48cd-bf63-7e4dcab61a36", + "6c6b05bf-ae5f-4013-82ab-7348e99ffdb2", + "00510a15-4216-4fdc-a0ab-05d74cd9f795" + ], + "male": [ + "0caedb75-417f-4e36-9b64-c21354cb94c8", + "32a806e8-894e-41ad-a4d5-6d9154d7b1e6" + ] + }, + "nl": { + "male": [ + "af482421-80f4-4379-b00c-a118def29cde", + "4b250449-c635-4b63-bd1d-b654b12ffcd4" + ], + "female": [ + "0eb213fe-4658-45bc-9442-33a48b24b133", + "ac317dac-1b8f-434f-b198-a490e2a4914d" + ] + }, + "no": { + "male": [ + "d6dca1b6-cdd8-4e9c-823c-e03979261740" + ] + }, + "te": { + "male": [ + "38bded0a-3ab4-42d1-8e47-2e0b6b10ced9" + ], + "female": [ + "07bc462a-c644-49f1-baf7-82d5599131be" + ] + }, + "kn": { + "female": [ + "7c6219d2-e8d2-462c-89d8-7ecba7c75d65" + ], + "male": [ + "6baae46d-1226-45b5-a976-c7f9b797aae2" + ] + }, + "fi": { + "male": [ + "ae1a833b-0d95-4b7f-8d05-d6418c6f8049" + ], + "female": [ + "65c34eec-42c9-4a75-a8bd-b676fb847b72" + ] + }, + "mr": { + "male": [ + "f227bc18-3704-47fe-b759-8c78a450fdfa" + ], + "female": [ + "5c32dce6-936a-4892-b131-bafe474afe5f" + ] + }, + "da": { + "female": [ + "c323c793-41f9-47b8-99dc-9b44b0440b84" + ] + }, + "bn": { + "female": [ + "59ba7dee-8f9a-432f-a6c0-ffb33666b654" + ], + "male": [ + "2ba861ea-7cdc-43d1-8608-4045b5a41de5" + ] + }, + "sk": { + "male": [ + "ca590fdc-df56-4d2e-94a4-ef5b423c7ddf" + ], + "female": [ + "abf68668-6549-462c-8426-1fa7b466b91d" + ] + }, + "uk": { + "male": [ + "05ffab9c-d380-4909-8375-cd12f59238c3" + ] + }, + "el": { + "female": [ + "50849023-76e9-46c7-af52-9ec39888a165" + ], + "male": [ + "b45eba5b-2215-4da7-9c7c-121c95ed7b81" + ] + }, + "ta": { + "male": [ + "d2870b91-1b4c-47ab-81a8-3718d8e9c222" + ], + "female": [ + "7f98e662-142d-41ba-89a2-12452640ce6d" + ] + }, + "vi": { + "male": [ + "0e58d60a-2f1a-4252-81bd-3db6af45fb41" + ], + "female": [ + "b8cd71e3-bc14-4538-a530-d6314731c036" + ] + }, + "id": { + "male": [ + "a053f6bc-7df4-40de-96d4-de026bc47ce8" + ], + "female": [ + "b441c4fd-4910-4c55-ae56-f0291057e2cc" + ] + }, + "ro": { + "female": [ + "34acfaee-c556-41ee-a5f6-c687fb20357c" + ], + "male": [ + "3f64ef99-d87b-4b51-b217-df7351f7886a" + ] + }, + "ka": { + "male": [ + "dbebd077-80cb-4bcf-b43b-4552f96341bb" + ], + "female": [ + "0bfbea6c-2f8f-4f86-b411-aa2316561e36" + ] + }, + "ml": { + "female": [ + "b426013c-002b-4e89-8874-8cd20b68373a" + ] + }, + "ms": { + "male": [ + "8281db18-6ac5-47bb-91a8-ce23a1f1d951" + ], + "female": [ + "83604597-55fa-4ccc-8357-730b313f353f" + ] + }, + "he": { + "male": [ + "3e32f3c5-9ac0-4192-9994-87fdb277120f" + ] + }, + "bg": { + "female": [ + "fcbecbcc-0cef-4615-8b5a-712fe1b39dd0" + ], + "male": [ + "d132064c-b931-4a80-bf0d-02a331ec4572" + ] + }, + "th": { + "male": [ + "5de076e9-7b28-4442-b279-e7d80d573505" + ], + "female": [ + "ccc7bb22-dcd0-42e4-822e-0731b950972f" + ] + }, + "hu": { + "female": [ + "e97c3b37-1aa5-46af-afb7-9545086aaa92" + ], + "male": [ + "36e0c00b-1bfd-4ad7-a0e8-928d4cadca00" + ] + }, + "pa": { + "female": [ + "991c62ce-631f-48b0-8060-2a0ebecbd15b" + ], + "male": [ + "8bacd442-a107-4ec1-b6f1-2fcb3f6f4d56" + ] + }, + "cs": { + "female": [ + "bdc4a3ce-2e22-4398-8cd6-76b7160d2298" + ], + "male": [ + "89266bab-6e15-455d-8654-e18c440b0656" + ] + }, + "tl": { + "male": [ + "c4cbcb7d-d9fa-4eac-b547-46831718ef58" + ], + "female": [ + "9261664a-c3d0-4200-9038-5466bcf3a09c" + ] + }, + "ar": { + "female": [ + "6304c635-6681-4f9e-85b6-a97f4d26461a" + ], + "male": [ + "e3087ad8-7018-4154-9a87-11577f916cd4" + ] + }, + "gu": { + "female": [ + "4590a461-bc68-4a50-8d14-ac04f5923d22" + ], + "male": [ + "91925fe5-42ee-4ebe-96c1-c84b12a85a32" + ] + }, + "hr": { + "male": [ + "a1a16724-b1f3-4b27-9e47-8a175115e93c" + ], + "female": [ + "2a2624ad-bd06-4563-81fd-0519742e25d2" + ] + } +} diff --git a/src/fi/alk/harness/run/models.py b/src/fi/alk/harness/run/models.py index 3134685e..38563f85 100644 --- a/src/fi/alk/harness/run/models.py +++ b/src/fi/alk/harness/run/models.py @@ -30,9 +30,10 @@ # this is the setting worth revisiting first once the target can be handed to ALK. AGENT = "claude-sonnet-4-6" USER = "claude-sonnet-4-6" -# Kept separate and stronger. A judged sub-goal is the one place a cheap wrong answer is -# expensive: it decides a pass, it runs once per scenario, and nobody re-reads it. -JUDGE = "claude-opus-4-7" +# One model for every role. A judged sub-goal was kept on a stronger model, but a run that mixes +# tiers is slower and harder to reason about, and the checks that decide a pass are code rather +# than judgement wherever they can be. Override with ALK_JUDGE_MODEL when a run needs it. +JUDGE = "claude-sonnet-4-6" def for_roles(override: str | None = None) -> dict[str, str]: diff --git a/src/fi/alk/harness/run/sdk_voice.py b/src/fi/alk/harness/run/sdk_voice.py index 30c0b004..05d4604f 100644 --- a/src/fi/alk/harness/run/sdk_voice.py +++ b/src/fi/alk/harness/run/sdk_voice.py @@ -8,9 +8,11 @@ from __future__ import annotations import argparse +import logging import asyncio import json import os +from functools import lru_cache from pathlib import Path from fi import simulate @@ -25,6 +27,8 @@ ) from fi.simulate.runtime.runner import SimulationRunner +logger = logging.getLogger(__name__) + def _required(name: str) -> str: value = os.environ.get(name, "").strip() @@ -41,14 +45,415 @@ def _json_env(name: str, default): return parsed +# Language names a persona may carry, to the codes Deepgram STT expects. Unrecognised values that +# already look like a code are passed through; everything else falls back to English. +# Language names and region codes to the code the providers expect. Ported from the platform +# so a persona resolves to the same language here as it does there. Anything unrecognised +# falls back to English, which is what the platform does too. +_LANGUAGE_CODES: dict[str, str] = { + "ar": "ar", + "ar-sa": "ar", + "arabic": "ar", + "bg": "bg", + "bulgarian": "bg", + "ca": "ca", + "catalan": "ca", + "chinese": "zh", + "chinese simplified": "zh", + "chinese traditional": "zh-TW", + "chinese (cantonese, traditional)": "zh-HK", + "chinese (mandarin, simplified)": "zh", + "chinese (mandarin, traditional)": "zh-TW", + "cs": "cs", + "czech": "cs", + "da": "da", + "da-dk": "da", + "danish": "da", + "de": "de", + "de-ch": "de-CH", + "dutch": "nl", + "el": "el", + "en": "en-US", + "en-au": "en-AU", + "en-gb": "en-GB", + "en-in": "en-IN", + "en-nz": "en-NZ", + "en-us": "en-US", + "english": "en-US", + "es": "es", + "es-419": "es-419", + "estonian": "et", + "et": "et", + "fi": "fi", + "finnish": "fi", + "flemish": "nl-BE", + "fr": "fr", + "fr-ca": "fr-CA", + "french": "fr", + "german": "de", + "greek": "el", + "hi": "hi", + "hindi": "hi", + "hu": "hu", + "hungarian": "hu", + "id": "id", + "indonesian": "id", + "it": "it", + "italian": "it", + "ja": "ja", + "japanese": "ja", + "ko": "ko", + "ko-kr": "ko", + "korean": "ko", + "latvian": "lv", + "lithuanian": "lt", + "lt": "lt", + "lv": "lv", + "malay": "ms", + "ms": "ms", + "nl": "nl", + "nl-be": "nl-BE", + "no": "no", + "norwegian": "no", + "pl": "pl", + "polish": "pl", + "portuguese": "pt", + "pt": "pt", + "pt-br": "pt-BR", + "pt-pt": "pt-PT", + "ro": "ro", + "romanian": "ro", + "ru": "ru", + "russian": "ru", + "sk": "sk", + "slovak": "sk", + "spanish": "es", + "sv": "sv", + "sv-se": "sv", + "swedish": "sv", + "th": "th", + "th-th": "th", + "thai": "th", + "tr": "tr", + "turkish": "tr", + "uk": "uk", + "ukrainian": "uk", + "vi": "vi", + "vietnamese": "vi", + "zh": "zh", + "zh-cn": "zh", + "zh-hans": "zh", + "zh-hant": "zh-TW", + "zh-hk": "zh-HK", + "zh-tw": "zh-TW", +} + + +def _normalised_language(raw: str) -> str: + """The code the providers expect, from a language name or a region code. + + Ported from the platform: lowercase, strip, exact lookup, and anything unrecognised becomes + English rather than failing the call. + """ + return _LANGUAGE_CODES.get((raw or "").strip().lower(), "en-US") + + +# Languages we transcribe with Deepgram's multilingual model rather than a single language code. +# The platform sends Arabic to Azure, which we do not have, so it joins Spanish on the model that +# does cover it. Deliberate divergence: we only ever use providers we hold keys for. +_MULTILINGUAL_STT = ("ar", "es") + + +def _transcriber_for(language: str) -> tuple[str, str, str]: + """The (provider, model, language) a persona's language needs for speech to text. + + Deepgram throughout, because Deepgram and Cartesia are the only providers configured. A + language Deepgram serves better multilingually is sent to that model instead of its own code. + """ + code = (language or "").lower() + if code.split("-", 1)[0] in _MULTILINGUAL_STT: + return ("deepgram", "nova-3", "multi") + return ("deepgram", "nova-3", language or "en-US") + + +def _persona_stt_language() -> str: + """The STT language for this call's caller, from the persona's languages. + + An explicit SIMULATOR_STT_LANGUAGE always wins. Otherwise the persona's first language is used, + so a caller who speaks Hindi is transcribed as Hindi rather than forced to English. + """ + override = os.environ.get("SIMULATOR_STT_LANGUAGE", "").strip() + if override: + return override + raw = os.environ.get("HARNESS_PERSONA", "").strip() + if raw: + try: + languages = (json.loads(raw) or {}).get("languages") or [] + except ValueError: + languages = [] + if isinstance(languages, list) and languages: + first = str(languages[0]).strip().lower() + if first in _LANGUAGE_CODES: + return _LANGUAGE_CODES[first] + if 2 <= len(first) <= 5 and first.replace("-", "").isalpha(): + return first + return "en" + + +# Cartesia voice selection: a persona's accent (or, failing that, language) chooses a catalog +# language bucket, and gender chooses within it, so a caller sounds like the accent the scenario +# wrote across dozens of languages rather than the handful of English voices Deepgram aura ships. +# Accent wins over language; both accept ISO codes and demonyms. Runs only when a Cartesia key is +# present; otherwise the Deepgram aura path below is used unchanged. +_CARTESIA_SUPPORTED_LANGS = frozenset( + { + "en", + "es", + "hi", + "de", + "fr", + "it", + "pl", + "ru", + "pt", + "ja", + "ko", + "zh", + "tr", + "sv", + "nl", + "no", + "te", + "kn", + "fi", + "mr", + "da", + "bn", + "sk", + "uk", + "el", + "ta", + "vi", + "id", + "ro", + "ka", + "ml", + "ms", + "he", + "bg", + "th", + "hu", + "pa", + "cs", + "tl", + "ar", + "gu", + "hr", + } +) +_CARTESIA_ACCENT_TO_LANG: dict[str, str] = { + "spanish": "es", + "south american": "es", + "indian": "hi", + "german": "de", + "french": "fr", + "italian": "it", + "polish": "pl", + "russian": "ru", + "portuguese": "pt", + "brazilian": "pt", + "japanese": "ja", + "korean": "ko", + "chinese": "zh", + "mandarin": "zh", + "turkish": "tr", + "swedish": "sv", + "dutch": "nl", + "norwegian": "no", + "finnish": "fi", + "danish": "da", + "slovak": "sk", + "ukrainian": "uk", + "greek": "el", + "romanian": "ro", + "georgian": "ka", + "bulgarian": "bg", + "thai": "th", + "hungarian": "hu", + "czech": "cs", + "croatian": "hr", + "vietnamese": "vi", + "indonesian": "id", + "malay": "ms", + "malaysian": "ms", + "tagalog": "tl", + "filipino": "tl", + "arabic": "ar", + "hebrew": "he", + "israeli": "he", + "telugu": "te", + "kannada": "kn", + "marathi": "mr", + "bengali": "bn", + "tamil": "ta", + "malayalam": "ml", + "punjabi": "pa", + "gujarati": "gu", +} +_CARTESIA_LANGUAGE_TO_LANG: dict[str, str] = { + "english": "en", + "chinese simplified": "zh", + "chinese traditional": "zh", + "hinglish": "hi", + "spanish": "es", + "hindi": "hi", + "german": "de", + "french": "fr", + "italian": "it", + "polish": "pl", + "russian": "ru", + "portuguese": "pt", + "japanese": "ja", + "korean": "ko", + "chinese": "zh", + "mandarin": "zh", + "turkish": "tr", + "swedish": "sv", + "dutch": "nl", + "norwegian": "no", + "telugu": "te", + "kannada": "kn", + "finnish": "fi", + "marathi": "mr", + "danish": "da", + "bengali": "bn", + "slovak": "sk", + "ukrainian": "uk", + "greek": "el", + "tamil": "ta", + "vietnamese": "vi", + "indonesian": "id", + "romanian": "ro", + "georgian": "ka", + "malayalam": "ml", + "malay": "ms", + "hebrew": "he", + "bulgarian": "bg", + "thai": "th", + "hungarian": "hu", + "punjabi": "pa", + "czech": "cs", + "tagalog": "tl", + "filipino": "tl", + "arabic": "ar", + "gujarati": "gu", + "croatian": "hr", +} +_CARTESIA_DEFAULT_VOICE = "f786b574-daa5-4673-aa0c-cbe3e8534c02" + + +def _norm(value) -> str: + return str(value or "").strip().lower().replace("-", " ") + + +@lru_cache(maxsize=1) +def _cartesia_catalog() -> dict: + path = Path(__file__).parent / "data" / "voices_by_language_and_gender.json" + try: + return json.loads(path.read_text(encoding="utf-8")) + except (OSError, ValueError): + return {} + + +def _persona_language_name(persona: dict) -> str: + languages = persona.get("languages") + if isinstance(languages, list) and languages: + return _norm(languages[0]) + return _norm(persona.get("language")) + + +def _cartesia_lang_key(persona: dict) -> str: + """The catalog language bucket for a persona: accent wins, then language, else English.""" + accent = _norm(persona.get("accent")) + key = _CARTESIA_ACCENT_TO_LANG.get(accent) + if key in _CARTESIA_SUPPORTED_LANGS: + return key + language = _persona_language_name(persona) + key = _CARTESIA_LANGUAGE_TO_LANG.get(language) + if key in _CARTESIA_SUPPORTED_LANGS: + return key + if language in _CARTESIA_SUPPORTED_LANGS: + return language + return "en" + + +def _cartesia_voice_for(persona: dict) -> str: + """A stable Cartesia voice id for one caller, chosen by accent/language and gender. + + Deterministic by persona name so a caller keeps its voice across runs while a suite still + spreads voices. Falls back across gender and to English when a long-tail language lacks one. + """ + gender = _norm(persona.get("gender")) + if gender not in ("male", "female"): + gender = "female" + catalog = _cartesia_catalog() + key = _cartesia_lang_key(persona) + other = "male" if gender == "female" else "female" + voices = ( + (catalog.get(key) or {}).get(gender) + or (catalog.get(key) or {}).get(other) + or (catalog.get("en") or {}).get(gender) + or [] + ) + if not voices: + return _CARTESIA_DEFAULT_VOICE + index = sum(ord(character) for character in str(persona.get("name") or "")) % len( + voices + ) + return voices[index] + + +def _voice_providers() -> tuple[str, str]: + """The (stt, tts) providers for the caller. An explicit env override wins; otherwise Cartesia + when its key is present (richer, multi-language voices), else Deepgram aura.""" + keyed = bool(os.environ.get("CARTESIA_API_KEY", "").strip()) + default = "cartesia" if keyed else "deepgram" + stt = os.environ.get("SIMULATOR_STT_PROVIDER", "").strip() or default + tts = os.environ.get("SIMULATOR_TTS_PROVIDER", "").strip() or default + if tts == "deepgram" and not keyed and not os.environ.get("SIMULATOR_TTS_PROVIDER"): + # Deepgram aura is one voice, so every persona sounds the same and the accent, language + # and gender the scenario chose are silently dropped. The call still runs, which is why + # this has to be said out loud rather than left to whoever listens to the recording. + logger.warning( + "cartesia_key_missing_personas_share_one_voice", + extra={"tts": "deepgram/aura-asteria-en"}, + ) + return stt, tts + + def _simulator() -> simulate.SimulatorAgentDefinition: + # The caller's brain is fixed on Vertex Gemini and only the model name is configurable. Its + # voice is not: speech to text and text to speech follow the persona's language, because a + # caller who speaks Japanese cannot be transcribed as English. llm_provider = os.environ.get("SIMULATOR_LLM_PROVIDER", "google") - stt_provider = os.environ.get("SIMULATOR_STT_PROVIDER", "deepgram") - tts_provider = os.environ.get("SIMULATOR_TTS_PROVIDER", "deepgram") + language = _persona_stt_language() + stt_provider, stt_model, stt_language = _transcriber_for(language) + _, tts_provider = _voice_providers() + default_tts_voice = ( + _CARTESIA_DEFAULT_VOICE if tts_provider == "cartesia" else "aura-asteria-en" + ) defaults = { - "llm": {"google": "gemini-2.5-flash-lite", "openai": "gpt-4o-mini"}, - "stt": {"deepgram": "nova-2", "google": "chirp_2"}, - "tts": {"deepgram": "aura-asteria-en", "google": "en-US-Chirp3-HD-Aoede"}, + "llm": {"google": "gemini-2.5-flash", "openai": "gpt-4o-mini"}, + "stt": { + "deepgram": stt_model or "nova-3", + "cartesia": "ink-2", + "google": "chirp_2", + }, + "tts": { + "deepgram": "aura-asteria-en", + "cartesia": "sonic-3.5", + "google": "en-US-Chirp3-HD-Aoede", + }, } def model(kind: str, provider: str) -> str: @@ -67,32 +472,89 @@ def model(kind: str, provider: str) -> str: stt={ "provider": stt_provider, "model": model("stt", stt_provider), - "language": os.environ.get("SIMULATOR_STT_LANGUAGE", "en"), + "language": stt_language, }, tts={ "provider": tts_provider, "model": model("tts", tts_provider), - "voice": os.environ.get("SIMULATOR_TTS_VOICE", "aura-asteria-en"), + "voice": os.environ.get("SIMULATOR_TTS_VOICE", default_tts_voice), }, + # Written as separate numbered rules rather than one paragraph. These arrive late in a + # long prompt, and a rule buried mid-sentence there does not survive: a caller ignored the + # loop rule for four turns while it was the tail of a compound sentence. instructions=( - "Act as the customer described by the scenario. Speak naturally and briefly. " - "Use only the supplied facts and never invent account, address, payment, or " - "verification data. Do not volunteer private data: agree when asked whether a " - "verification code should be sent, and disclose the actual code only after the " - "agent says it was sent and explicitly asks you to read it. Answer repair questions " - "with the missing fact, not by restarting the request. Never repeat the same answer " - "more than twice. When the requested outcome is complete, thank the agent and end " - "the call." + "Act as the customer described by the scenario. Speak naturally and briefly.\n" + "These rules override anything else when they conflict:\n" + "1. Use ONLY the facts you were given. Never invent an account detail, address, " + "payment state, or verification code.\n" + "2. If the agent asks about something you were given no fact for, say plainly that " + "you do not know or cannot tell. Never guess, and never claim something happened on " + "your end when you were not told it did.\n" + "3. Do not volunteer private data. Agree when asked whether a verification code " + "should be sent, and read the code out only after the agent says it was sent and " + "asks you for it.\n" + "4. Answer a repair question with the missing fact, not by restarting your request.\n" + "5. STOP AFTER THREE. Count the agent's replies. If three of them say essentially " + "the same thing without the task moving forward, do not try a fifth time and do not " + "rephrase the same point again. Say once that this is not working and you will try " + "later, then end the call.\n" + "6. Otherwise let the agent finish. Say yes when it asks to proceed and wait for it " + "to confirm the outcome rather than hanging up early.\n" + "7. Once the outcome is confirmed, thank the agent and end the call." ), allow_interruptions=True, ) +# Deepgram aura encodes the speaker in the model name, so a persona's accent selects a voice by +# choosing the aura model. Only English accents aura actually ships are mapped; anything else keeps +# the default so a caller never loses a voice to an accent the provider cannot render. +_AURA_BY_ACCENT: dict[str, dict[str, list[str]]] = { + "american": { + "female": ["aura-asteria-en", "aura-luna-en", "aura-hera-en", "aura-stella-en"], + "male": ["aura-orion-en", "aura-arcas-en", "aura-perseus-en", "aura-zeus-en"], + }, + "british": {"female": ["aura-athena-en"], "male": ["aura-helios-en"]}, + "irish": {"female": ["aura-athena-en"], "male": ["aura-angus-en"]}, + "australian": {"female": ["aura-athena-en"], "male": ["aura-helios-en"]}, +} + + +def _aura_voice_for(persona: dict) -> str: + """A stable aura voice for one caller, chosen by accent and gender. + + Callers who share an accent still differ: the voice within the accent's set is picked by the + persona name, so a suite varies without being random between runs of the same scenario. + """ + accent = str(persona.get("accent") or "").strip().lower() + gender = str(persona.get("gender") or "").strip().lower() + if gender not in ("male", "female"): + gender = "female" + bucket = next( + (voices for key, voices in _AURA_BY_ACCENT.items() if key in accent), + _AURA_BY_ACCENT["american"], + ) + voices = bucket.get(gender) or next(iter(bucket.values())) + index = sum(ord(character) for character in str(persona.get("name") or "")) % len( + voices + ) + return voices[index] + + def _scenario() -> simulate.Scenario: fixture = _json_env("HARNESS_FIXTURE", {}) persona = _json_env("HARNESS_PERSONA", {"name": "customer"}) persona = dict(persona) if isinstance(persona, dict) else {"name": "customer"} persona["role"] = "customer" + # Give the caller a voice from its accent/language when none was set, so different callers + # sound different and match what the scenario wrote. Cartesia draws from the multi-language + # catalog; Deepgram falls back to the aura voices it ships. + if not persona.get("voice") and not persona.get("voice_id"): + tts_provider = _voice_providers()[1].lower() + if tts_provider == "cartesia": + persona["voice"] = _cartesia_voice_for(persona) + elif tts_provider == "deepgram": + persona["voice"] = _aura_voice_for(persona) metadata = dict(persona.get("metadata") or {}) if isinstance(fixture, dict) and fixture.get("phone"): # LiveKit exposes this as participant metadata/attributes. A target can @@ -117,10 +579,9 @@ def _scenario() -> simulate.Scenario: simulate.Persona( persona=persona, situation=_required("HARNESS_INSTRUCTION"), - outcome=os.environ.get( - "HARNESS_OUTCOME", - "Complete the requested task and close naturally.", - ), + # Empty by default: the instruction already says what this person wants, in + # their own words. A generic objective here only competes with it. + outcome=os.environ.get("HARNESS_OUTCOME", ""), knowledge=knowledge, behavior_policy={ "disclosure_policy": 0.72, diff --git a/src/fi/alk/harness/run/simulation.py b/src/fi/alk/harness/run/simulation.py index 7b94491e..d205ce6d 100644 --- a/src/fi/alk/harness/run/simulation.py +++ b/src/fi/alk/harness/run/simulation.py @@ -32,13 +32,16 @@ from dataclasses import asdict from datetime import UTC, datetime from pathlib import Path -from typing import Any +from typing import TYPE_CHECKING, Any from ..contract import AgentContract from ..scenario import Scenario from ..world.runtime import Call from .grade import Judgement, Result +if TYPE_CHECKING: + from .conversation import Exchange + RUNS = "runs" RUN = "run.json" RESULT = "result.json" @@ -501,6 +504,23 @@ def _calls_of(calls: Any) -> list[dict[str, Any]]: ] +def _said(line: str) -> Exchange: + """One transcript line as a turn, with its role read off rather than left in the text. + + The line arrives already labelled ("assistant: ..."). Keeping that label in the text made the + judge read ``agent: assistant: ...``, two speakers deep for every turn. + """ + from .conversation import Exchange + + role, _, text = line.partition(":") + named = role.strip().lower() + if named in ("assistant", "agent"): + return Exchange("agent", text.strip()) + if named in ("user", "customer"): + return Exchange("customer", text.strip()) + return Exchange("customer", line.strip()) + + async def _spoken_to( scenario: Scenario, contract: AgentContract, @@ -526,7 +546,7 @@ async def _spoken_to( from ..catalogue import load_catalogue from .call import place_the_call - from .conversation import Exchange, Transcript + from .conversation import Transcript from .evidence import measured, newest_report, spoken_times, tracks_in from .grade import ( checkpoints, @@ -570,7 +590,10 @@ def placed_once() -> tuple[int, dict[str, Any], str]: os.environ["HARNESS_VOICE_OUTPUT_ROOT"] = str(sdk_output.resolve()) os.environ["HARNESS_INSTRUCTION"] = instruction os.environ["HARNESS_SCENARIO"] = scenario.name - os.environ["HARNESS_OUTCOME"] = scenario.tests + # The caller is never handed the grader's pass question: `tests` is written about + # the agent in the third person, so as an objective it reads as a rubric rather + # than a motive. What this person wants is already in the instruction. + os.environ.pop("HARNESS_OUTCOME", None) os.environ["HARNESS_PERSONA"] = json.dumps( scenario.persona.model_dump(exclude_none=True) if scenario.persona is not None @@ -588,6 +611,29 @@ def placed_once() -> tuple[int, dict[str, Any], str]: os.environ["HARNESS_FIXTURE"] = json.dumps( scenario.fixture, ensure_ascii=False, default=str ) + # A scenario that asks to be heard through background noise selects a clip for the + # caller's environment; the voice engine mixes it under the caller. Cleared otherwise so + # a previous call's noise never leaks into a quiet one. + from ..background_noise import enabled as noise_enabled + + noisy = getattr(scenario, "background_noise", False) and noise_enabled() + if noisy: + from ..background_noise import source_for + + # The scenario names the place when it cares which one; otherwise the fixture + # says where the caller is, and failing that any noise will do. + environment = noisy if isinstance(noisy, str) else "" + if not environment and isinstance(scenario.fixture, dict): + environment = str( + scenario.fixture.get("environment") + or scenario.fixture.get("location") + or "" + ) + os.environ["HARNESS_BACKGROUND_NOISE"] = source_for( + environment, seed=scenario.name + ) + else: + os.environ.pop("HARNESS_BACKGROUND_NOISE", None) code = place_the_call( os.environ.get("HARNESS_VOICE_CASE", "2.1.2"), on_exchange=live_exchange if on_exchange else None, @@ -695,13 +741,7 @@ def placed_once() -> tuple[int, dict[str, Any], str]: # happened is that one check passed and the other was never asked, which reads as the agent # half-failing rather than as the suite not having looked. spoken_transcript = Transcript( - exchanges=[ - Exchange( - "agent" if line.lower().startswith("assistant") else "customer", line - ) - for line in spoken.splitlines() - if line.strip() - ], + exchanges=[_said(line) for line in spoken.splitlines() if line.strip()], calls=list(world.calls), ended=str((case.get("metadata") or {}).get("status") or "finished"), ) diff --git a/src/fi/alk/harness/run/tools.py b/src/fi/alk/harness/run/tools.py index 442512e6..93e2f238 100644 --- a/src/fi/alk/harness/run/tools.py +++ b/src/fi/alk/harness/run/tools.py @@ -270,7 +270,7 @@ async def list_scenarios(_args: dict[str, Any]) -> dict[str, Any]: ) ) lines.append( - f"{one.name}{mark}\n tests: {one.tests or one.use_case or '—'}\n" + f"{one.name}{mark}\n passes when: {one.tests or one.use_case or '—'}\n" f" settled by code: {', '.join(settled) or 'none'}\n" f" judged: {', '.join(judged) or 'none'}" ) @@ -464,7 +464,10 @@ def placed() -> tuple[LiveRun, str, list[str], str]: # how a simulated caller behaves is not decided in two places. os.environ["HARNESS_INSTRUCTION"] = instruction os.environ["HARNESS_SCENARIO"] = scenario.name - os.environ["HARNESS_OUTCOME"] = scenario.tests + # The caller is never handed the grader's pass question. `tests` is written about + # the agent in the third person, so as an objective it reads as a rubric rather + # than a motive. What this person wants is already in the instruction. + os.environ.pop("HARNESS_OUTCOME", None) os.environ["HARNESS_PERSONA"] = json.dumps( scenario.persona.model_dump(exclude_none=True) if scenario.persona is not None diff --git a/src/fi/alk/harness/scenario.py b/src/fi/alk/harness/scenario.py index 90428c6c..ef5956b8 100644 --- a/src/fi/alk/harness/scenario.py +++ b/src/fi/alk/harness/scenario.py @@ -13,14 +13,14 @@ from __future__ import annotations import ast +import hashlib import json -import random import re from collections import Counter from math import ceil -from typing import Any +from typing import Any, ClassVar -from pydantic import BaseModel, Field +from pydantic import BaseModel, Field, model_validator from .catalogue import Catalogue from .simulator import variables_in @@ -149,11 +149,35 @@ def format_persona(self) -> str: return "\n".join(parts) +def _slug(name: str) -> str: + """An ASCII key for ``name``, safe to send as a header value. + + Falls back to a digest rather than an empty string: an empty key would collapse every + scenario in a job onto one idempotency key on the receiving side. + """ + cleaned = re.sub(r"[^a-z0-9]+", "-", (name or "").strip().lower()).strip("-") + return cleaned or "scenario-" + hashlib.sha256(name.encode()).hexdigest()[:12] + + +def _decided_by(name: str) -> bool: + """Whether this scenario is noisy, decided by its name so a rerun decides the same.""" + return hashlib.sha256((name or "").encode()).digest()[0] % 2 == 0 + + class Scenario(BaseModel): """One test: what changes, what is asked, what a correct agent does, what must hold.""" name: str + # How this scenario is identified on the wire. Derived from ``name``, which is already unique + # across a suite and already a slug because it is the folder name. It ships as a header, so + # anything outside ASCII is dropped and an empty result falls back to a digest. + scenario_key: str = "" + # Assigned by the platform when the scenario is pre-allocated. Never written here. + scenario_id: str = "" use_case: str = "" + # What makes this row different from its siblings in the same use case. Coverage is counted + # on the pair, so a use case can carry many scenarios without any reading as a duplicate. + branch: str = "" tests: str = "" # What this scenario changes about the world after it is reset, as code: a file defining @@ -195,19 +219,34 @@ class Scenario(BaseModel): max_turns: int = 10 - # Whether this call happens somewhere noisy. Recorded per scenario rather than per run, so a - # suite covers both conditions and the same scenario stays comparable to itself across runs. - # Chosen at random when the writer does not say, because a suite where every call is quiet - # tests an agent nobody has: real callers phone from cars, kitchens and streets. - # - # Nothing consumes this yet. It is carried so the scenarios written from today are already - # answerable when the caller learns to add noise, rather than needing to be rewritten then. - background_noise: bool = Field(default_factory=lambda: random.choice((True, False))) + # Where this call is made from. A string names the place ("street", "vehicle", "retail"), and + # True asks for noise while leaving the place to the fixture. Left unset it is decided from + # the name, so a suite still covers both conditions but the same suite decides the same way + # twice; a coin flip here made a seeded run unreproducible. + background_noise: bool | str = "" + + # Slots the caller filled by the run rather than by the scenario. Listed so a template that + # uses one is not rejected as unfillable at write time. + RUNTIME_SLOTS: ClassVar[tuple[str, ...]] = ("channel", "situation") + + @model_validator(mode="after") + def _identify(self) -> "Scenario": + if not self.scenario_key: + self.scenario_key = _slug(self.name) + if self.background_noise == "": + self.background_noise = _decided_by(self.name) + return self def slots(self) -> dict[str, str]: """Every value this scenario offers the simulator prompt.""" persona = {"persona": self.persona.format_persona()} if self.persona else {} - return {"instruction": self.instruction, **self.variables, **persona} + runtime = {name: "" for name in self.RUNTIME_SLOTS} + return { + "instruction": self.instruction, + **runtime, + **self.variables, + **persona, + } def validate_scenario( @@ -232,6 +271,12 @@ def validate_scenario( missing := scenario.persona.missing_profile_fields() ): problems.append("persona is incomplete: " + ", ".join(missing)) + elif scenario.persona is not None: + # A persona in words of its own renders fine and then does nothing: no behaviour guidance + # attaches, and the accent it names selects no voice. + from .persona_guides import unrecognised + + problems.extend(unrecognised(scenario.persona.model_dump())) if not scenario.sub_goals: problems.append( "no sub_goals: nothing would be graded. Name the entries of the catalogue this " diff --git a/src/fi/alk/harness/scenario_tools.py b/src/fi/alk/harness/scenario_tools.py index e6249837..07112617 100644 --- a/src/fi/alk/harness/scenario_tools.py +++ b/src/fi/alk/harness/scenario_tools.py @@ -12,6 +12,7 @@ from __future__ import annotations import json +import os from pathlib import Path from typing import Any @@ -50,6 +51,41 @@ def _err(text: str) -> dict[str, Any]: return {"content": [{"type": "text", "text": text}], "is_error": True} +def parallel_suites() -> bool: + """Whether a suite is written by several writers at once. + + Off by default. Writing one scenario at a time is slower but is the path the base branch runs + on, and a suite that is written slowly is worth more than one that is not written at all. + Set HARNESS_PARALLEL_SCENARIOS=1 to fan out instead. + """ + return os.environ.get("HARNESS_PARALLEL_SCENARIOS", "").strip() == "1" + + +def persona_field(name: str) -> dict[str, Any]: + """The schema for one persona field, carrying the platform's own values where it has them. + + Offered as an enum so the values arrive right the first time. Without the platform's model + to read, it stays a plain string rather than an enum of nothing. + """ + from .persona_guides import offered + + allowed = offered(name) + return {"type": "string", "enum": allowed} if allowed else {"type": "string"} + + +def persona_vocabulary_note() -> str: + """A sentence about why the persona fields are constrained, when they are.""" + from .persona_guides import vocabulary + + if not vocabulary(): + return "" + return ( + " The listed values are the ones the platform understands: they carry behaviour " + "guidance into the call and select the caller's voice. Anything else about this person " + "goes in metadata, where it is free text." + ) + + def write_scenarios( scenarios: list[Scenario], destination: Path, catalogue: Catalogue | None = None ) -> Path: @@ -96,8 +132,14 @@ def accept_scenario( kept: list[Scenario], simulator_prompt: str = "", hard_constraints: list[str] | None = None, + persist: bool = True, ) -> dict[str, Any]: - """Validate one scenario, then prove it. A plain function so both halves are testable.""" + """Validate one scenario, then prove it. A plain function so both halves are testable. + + ``persist`` is off for a writer that shares the destination with siblings: writing the suite + removes every folder not in the writer's own list, so persisting here would delete whatever + the others have proved. Those writers keep their work in ``kept`` and the caller saves once. + """ try: scenario = Scenario.model_validate(payload) except Exception as invalid: @@ -139,7 +181,8 @@ def accept_scenario( # A proved scenario is already valuable work. Persist it immediately so a stopped model, # browser refresh, process restart, or later scenario failure cannot make the UI say none # were written. ``save_scenarios`` remains the suite-level diversity/finality gate. - write_scenarios(kept, world_root, catalogue) + if persist: + write_scenarios(kept, world_root, catalogue) return _ok( f"{scenario.name} {'replaced' if replaced else 'kept'}. All three gates pass: the world " "is ready for it, the reference solution passes its checks, and those checks fail when " @@ -168,17 +211,22 @@ def not_ready(kept: list[Scenario], wanted: int, catalogue: Catalogue) -> list[s # "cancel a pending order", which is neither what it tests nor distinguishable afterwards # from the scenario that really does test that. A use case is how coverage is counted, so a # duplicate quietly overstates it. - claimed: dict[str, list[str]] = {} + # Keyed on the pair, not the use case alone. A use case fans out into several branches and + # each is a separate test, so keying on the use case alone caps a suite at one scenario per + # use case — which is how a request for forty against fourteen use cases became unsaveable. + claimed: dict[tuple[str, str], list[str]] = {} for one in kept: case = (one.use_case or "").strip().lower() + branch = (one.branch or "").strip().lower() if case: - claimed.setdefault(case, []).append(one.name) - for case, names in claimed.items(): + claimed.setdefault((case, branch), []).append(one.name) + for (case, branch), names in claimed.items(): if len(names) > 1: + where = f"{case!r}" if not branch else f"{case!r} / {branch!r}" problems.append( - f"{' and '.join(names)} both claim the use case {case!r}. Give each the use case " - "it actually exercises, or drop the one that duplicates the other. Coverage is " - "counted by use case, so two scenarios sharing one hides a gap." + f"{' and '.join(names)} both claim {where}. Give each the branch it actually " + "exercises, or drop the one that duplicates the other. Coverage is counted by " + "use case and branch, so two scenarios sharing both hides a gap." ) # Sub-goals are shared so results roll up. A suite where every scenario invents its own is a @@ -193,16 +241,36 @@ def not_ready(kept: list[Scenario], wanted: int, catalogue: Catalogue) -> list[s def scenario_tools( - contract: AgentContract, world_root: Path, destination: Path, *, wanted: int + contract: AgentContract, + world_root: Path, + destination: Path, + *, + wanted: int, + can_save: bool = True, + start_from: list[Scenario] | None = None, ) -> tuple[Any, list[Scenario]]: - """A server for writing scenarios against one built environment.""" - kept: list[Scenario] = load_scenarios(destination) + """A server for writing scenarios against one built environment. + + ``can_save`` is what makes several writers safe at once. Saving rewrites the index and + removes any folder not in the saver's own list, so two writers saving concurrently delete + each other's work. A writer that only submits keeps its scenarios in ``kept``, and whoever + spawned it merges the lists and writes once. + + ``start_from`` seeds that list. A parallel writer starts empty rather than from disk, so it + is never counted as already having what a sibling wrote. + """ + kept: list[Scenario] = ( + list(start_from) if start_from is not None else load_scenarios(destination) + ) catalogue = load_catalogue(destination) simulator_prompt = load_simulator_prompt(destination) target = {"count": wanted} exploration = {"since_submit": 0} - scenario_required = ["name", "instruction", "solution", "sub_goals"] + # ``branch`` is required because coverage is counted on the use case and branch pair, and the + # merge drops a repeat of that pair. A writer that leaves it out gives every scenario in its + # slice the same pair, and all but the first are silently thrown away. + scenario_required = ["name", "branch", "instruction", "solution", "sub_goals"] if contract.conversational: scenario_required.append("persona") @@ -373,36 +441,71 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: "type": "string", "description": "Which of the agent's use cases this belongs to.", }, + "branch": { + "type": "string", + "description": "The condition that makes this scenario different from the " + "others in the same use case, in one line: what is true here that is not " + "true of its siblings.", + }, "tests": { "type": "string", "description": "One line: what this scenario is trying to find out.", }, + "background_noise": { + "type": "string", + "description": "Where the caller is phoning from: street, transit, vehicle, " + "outdoors, retail, office or home. Name it whenever the instruction implies " + "somewhere, a caller leaving a hotel or standing on a street is not in a " + "quiet room. Left out, it is decided from the scenario name.", + }, "instruction": { "type": "string", - "description": "The task, written to the person the agent is serving. For a " - "conversational agent this fills the simulator prompt's slot.", + "description": "What this person is trying to achieve, written to them. " + "State the objective first, in their own terms, so they pursue it rather " + "than narrate a situation: 'Get the cancellation fee refunded', not 'You " + "were charged a fee'. Then give them everything they need to hold the " + "conversation without inventing anything: the facts they know, the values " + "they can be asked for, and what they will only say once asked. Every value " + "real and read out of the world.\n" + "Write only what this person knows before the call starts. Never write what " + "the agent will do, in any phrasing: not what it will send, offer, ask for, " + "disclose or decide, and no closing line about what counts as done. Those " + "are the behaviours under test, and a person primed to expect them plays " + "along whether or not they happen, so the check passes on a conversation " + "that never earned it. Give them the value, the preference or the problem " + "they arrived with, and let the agent's handling of it be what is measured.\n" + "Test every sentence by asking whether this person could say it out loud. " + "They have never seen the agent's design, so a parenthetical explaining " + "where the agent should find a value fails that test just as much as a " + "sentence predicting what it will say. Worst of all is agreeing in advance " + "to something the agent has not done yet: that hands over a pass the " + "conversation never earned.", }, "persona": { "type": "object", "description": "Who the simulated person is, separate from the task. Use " "the established voice-scenario shape and only grounded, test-relevant " - "details. This fills the simulator prompt's persona slot.", + "details. This fills the simulator prompt's persona slot." + + persona_vocabulary_note(), "properties": { "name": {"type": "string"}, - "gender": {"type": "string"}, - "age_group": {"type": "string"}, - "occupation": {"type": "string"}, - "location": {"type": "string"}, - "personality": {"type": "string"}, - "communication_style": {"type": "string"}, + "gender": persona_field("gender"), + "age_group": persona_field("age_group"), + "occupation": persona_field("occupation"), + "location": persona_field("location"), + "personality": persona_field("personality"), + "communication_style": persona_field("communication_style"), "initial_message": { "type": "string", "description": "The caller's natural opening request, specific to " "this scenario. Do not use a generic greeting.", }, "keywords": {"type": "array", "items": {"type": "string"}}, - "languages": {"type": "array", "items": {"type": "string"}}, - "accent": {"type": "string"}, + "languages": { + "type": "array", + "items": persona_field("languages"), + }, + "accent": persona_field("accent"), "multilingual": {"type": "boolean"}, "metadata": {"type": "object"}, }, @@ -491,6 +594,7 @@ async def submit_scenario(args: dict[str, Any]) -> dict[str, Any]: kept=kept, simulator_prompt=simulator_prompt, hard_constraints=contract.hard_constraints, + persist=can_save, ) if not result.get("is_error"): exploration["since_submit"] = 0 @@ -612,6 +716,109 @@ async def drop_scenario(args: dict[str, Any]) -> dict[str, Any]: write_scenarios(kept, destination, catalogue) return _ok(f"{name} dropped. {len(kept)} left") + @tool( + "generate_suite", + "Write a whole suite at once by splitting it across the agent's use cases, one writer " + "per slice, several running at the same time, then reviewing what came back and " + "filling what it missed. Use this whenever somebody asks for a number of scenarios " + "rather than one in particular: writing twenty or fifty one at a time runs out of " + "turns long before it finishes.\n\n" + "Pass `slices` when you know how the suite should be divided, which you do once you " + "have looked at the world: give each use case a share in proportion to how much can " + "genuinely go wrong in it, and name the angle each slice should take. Without it the " + "work is divided evenly, which pads the thin use cases and under-covers the rich ones. " + "Everything produced clears the same three gates, and the suite is saved.", + schema( + { + "count": int, + "at_once": int, + "slices": { + "type": ["array", "null"], + "description": "How to divide the suite. One entry per writer.", + "items": { + "type": "object", + "properties": { + "use_case": { + "type": "string", + "description": "One of the agent's use cases, worded as the " + "contract words it.", + }, + "angle": { + "type": "string", + "description": "What this slice should look for: the ordinary " + "path, the branch that cannot be completed, the rule under " + "pressure, state that has to carry.", + }, + "count": { + "type": "integer", + "description": "How many scenarios this slice is worth, in " + "proportion to how much can genuinely go wrong in it.", + }, + "why": { + "type": "string", + "description": "Why it earns that share.", + }, + }, + "required": ["use_case", "count"], + }, + }, + }, + ["count"], + ), + ) + async def generate_suite(args: dict[str, Any]) -> dict[str, Any]: + from .scenarios import MOST_AT_ONCE, MOST_IN_ONE_GO, write_in_parallel + + asked = int(args.get("count") or 0) + if asked < 1: + return _err("say how many scenarios the suite should have") + cases = [one for one in contract.real_use_cases if one.strip()] + given = args.get("slices") or None + if not cases and not given: + return _err( + "this contract names no use cases, so there is nothing to split the work " + "across. Write them one at a time with submit_scenario, or fix the contract." + ) + + # A large ask is served a batch at a time. Spinning up a writer per scenario would put + # hundreds of model sessions on one machine, and the person waiting would see nothing + # for an hour. A batch they can read, and an offer of the rest, is the better trade. + count = min(asked, MOST_IN_ONE_GO) + at_once = max(1, min(int(args.get("at_once") or 0) or 4, MOST_AT_ONCE)) + + produced = await write_in_parallel( + contract, + out=destination, + wanted=count, + use_cases=cases, + slices=given, + at_once=at_once, + ) + # The suite is already on disk. The open session's own list has to be brought level with + # it, or a later save_scenarios here would write out the stale list and delete every + # folder the fan-out just produced. + kept[:] = produced + target["count"] = len(produced) + + by_case: dict[str, int] = {} + for one in produced: + name = one.use_case or "unassigned" + by_case[name] = by_case.get(name, 0) + 1 + lines = "\n".join(f" {n} x {case[:70]}" for case, n in sorted(by_case.items())) + said = ( + f"{len(produced)} scenarios across {len(by_case)} use cases, {at_once} writers at a " + f"time. Each cleared all three gates and the suite is saved.\n{lines}" + ) + if asked > count: + said += ( + f"\n\n{asked - count} of the {asked} asked for are still to write. " + f"{MOST_IN_ONE_GO} is as many as one pass does, so that the suite can be looked " + "at before more is spent on it. Show what came back, then ask whether to carry " + "on with the rest, change direction first, or stop here. Call generate_suite " + "again for the next batch once they have said." + ) + return _ok(said) + @tool( "save_scenarios", "Write the kept scenarios out. Every one has already been proved by submit_scenario, so " @@ -670,13 +877,21 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: fix_tool_tool, aim_for, drop_scenario, - save_scenarios, - ], + ] + # Only the session a person is talking to may fan out. A writer that is itself one slice + # of a fan-out calling this would split its own slice again, and so on. + + ( + [generate_suite, save_scenarios] + if can_save and parallel_suites() + else [save_scenarios] + if can_save + else [] + ), ) return server, kept -TOOL_NAMES = ( +_ALWAYS = ( "inspect_world", "inspect_scenario", "try_calls", @@ -692,6 +907,17 @@ async def save_scenarios(_args: dict[str, Any]) -> dict[str, Any]: ) +def tool_names() -> tuple[str, ...]: + """The tools a saving session publishes, which depends on how a suite is written.""" + if parallel_suites(): + return (*_ALWAYS[:-1], "generate_suite", "save_scenarios") + return _ALWAYS + + +# Kept as a name because callers import it; it reflects the surface for this process. +TOOL_NAMES = tool_names() + + def world_summary(world_root: Path) -> str: """What is in the built environment, for grounding the writer before it asks.""" world = restore(world_root) diff --git a/src/fi/alk/harness/scenarios.py b/src/fi/alk/harness/scenarios.py index 57a4a581..5832b7b2 100644 --- a/src/fi/alk/harness/scenarios.py +++ b/src/fi/alk/harness/scenarios.py @@ -10,11 +10,15 @@ from __future__ import annotations +import asyncio +import logging +import os +from dataclasses import dataclass from collections.abc import Callable from pathlib import Path from typing import Any -from claude_agent_sdk import ClaudeAgentOptions +from claude_agent_sdk import ClaudeAgentOptions, create_sdk_mcp_server, tool from .config import ( UNWANTED, @@ -24,21 +28,31 @@ load_skill, permission_gate, provider_env, + thinking_config, ) +from .catalogue import load_catalogue from .contract import AgentContract from .scenario import Scenario from .scenario_tools import ( + parallel_suites, SCENARIO_SERVER, TOOL_NAMES, load_scenarios, scenario_tools, world_summary, + write_scenarios, ) from .session import Stage -from .tools import qualified +from .tools import qualified, schema + +logger = logging.getLogger(__name__) SKILL = "write-scenarios" +# The review pass runs its own tool server, kept apart from the writers' one so a reviewer can +# only report gaps and never submit or save a scenario itself. +REVIEW_SERVER = "suite-review" + # Turns a scenario costs in practice: look at the world, rehearse the calls, submit, and often # one more to correct what a gate refused. @@ -100,6 +114,7 @@ def open_stage( options.disallowed_tools = list(UNWANTED) options.hooks = gate_hooks(allowed) options.can_use_tool = permission_gate(ask, allowed) + options.thinking = thinking_config() return Stage(options, name=SKILL), destination @@ -131,6 +146,13 @@ def opening(contract: AgentContract, wanted: int = 10, existing: int = 0) -> str "across several turns. If a proof says an intended check is vacuous or broken, repair " "that named sub-goal with add_sub_goal and resubmit. Never evade a gate by deleting a " "check for behavior the scenario still claims to test. Then save_scenarios." + + ( + "\n\nFor a suite rather than one scenario, say briefly how you are splitting it " + "across the agent's use cases and then write it with generate_suite in the same " + "turn: it runs a writer per use case at the same time and saves what they prove." + if parallel_suites() + else "" + ) ) @@ -139,6 +161,520 @@ def load(destination: Path) -> list[Scenario]: return load_scenarios(Path(destination)) +# What a suite costs, and what it is allowed to cost. +# +# Writers run as separate model sessions, so wall clock is roughly the number of scenarios +# divided by how many run at once. The two ceilings below exist for different reasons: one +# protects the machine, the other protects the person waiting. Asking for a thousand scenarios +# is a reasonable thing to want and an unreasonable thing to do in one go, so a large ask is +# served a batch at a time with the rest offered back. +AT_ONCE = 4 +MOST_AT_ONCE = int(os.environ.get("HARNESS_WRITERS_AT_ONCE") or 8) +MOST_IN_ONE_GO = int(os.environ.get("HARNESS_SUITE_BATCH") or 50) + +# How many times the suite is reviewed and topped up after the first pass. One is enough to +# catch a slice that came back short or a use case nobody covered; more turns it into a loop +# that keeps finding smaller things to say. +TOP_UP_ROUNDS = 1 + + +@dataclass(frozen=True) +class Slice: + """One writer's share of a suite: what to write, how much, and why it is worth writing.""" + + use_case: str + angle: str = "" + count: int = 1 + why: str = "" + + def named(self) -> str: + return f"{self.use_case}: {self.angle}" if self.angle else self.use_case + + +def even_slices(wanted: int, use_cases: list[str]) -> list[Slice]: + """The fallback split, when nobody said how the work should be divided. + + Evenly, with the remainder going to the ones named first, because a contract lists its + primary use cases before its marginal ones. It is a poor plan and it is meant to be: a use + case with one real branch gets the same share as one with six, so the first pads and the + second under-covers. It exists so a caller that supplies no plan still gets a suite. + """ + if not use_cases: + return [] + if wanted <= len(use_cases): + return [Slice(use_case=case, count=1) for case in use_cases[:wanted]] + each, extra = divmod(wanted, len(use_cases)) + return [ + Slice(use_case=case, count=each + (1 if i < extra else 0)) + for i, case in enumerate(use_cases) + ] + + +def planned(wanted: int, use_cases: list[str], given: list[dict] | None) -> list[Slice]: + """The split this suite will actually be written to. + + A plan supplied by the caller wins, because whoever is talking to the person has just read + the contract and the world and knows which use cases have something in them. Sizing every + use case identically is the thing that made suites pad in one place and under-cover in + another, and the plan is the only part of the process that knows the difference. + + Anything the plan leaves out is filled in evenly, and anything it over-asks for is trimmed, + so a plan can be rough without producing a suite nobody asked for. + """ + if not given: + return even_slices(wanted, use_cases) + + known = {case.strip().lower(): case for case in use_cases} + slices: list[Slice] = [] + for one in given: + if not isinstance(one, dict): + continue + case = str(one.get("use_case") or "").strip() + if not case: + continue + # Match the contract's own wording where the plan paraphrased it, so a slice is filed + # under a use case the coverage count recognises rather than a near-miss of one. + case = known.get(case.lower(), case) + try: + count = max(1, int(one.get("count") or 1)) + except (TypeError, ValueError): + count = 1 + slices.append( + Slice( + use_case=case, + angle=str(one.get("angle") or "").strip(), + count=count, + why=str(one.get("why") or "").strip(), + ) + ) + if not slices: + return even_slices(wanted, use_cases) + + # Trim from the end rather than scaling everything down: the plan put its most valuable + # slices first, and shaving one scenario off each is how a deliberate plan becomes an even + # one again. + total = sum(one.count for one in slices) + while total > wanted and slices: + last = slices[-1] + if last.count > 1: + slices[-1] = Slice(last.use_case, last.angle, last.count - 1, last.why) + else: + slices.pop() + total = sum(one.count for one in slices) + return slices + + +def callers_for(index: int, wanted: int) -> str: + """Which callers this slice should write, so the suite varies across slices as well as within. + + Instruction alone cannot do this. Each writer is blind to the others, so each independently + picks the safest value and the suite converges on it: measured across three suites, more + than half the callers came out "Professional and formal" and over three quarters American, + with nobody doing anything wrong. Worse, a slice writing a single scenario has nothing to + vary at all. + + So the spread is dealt out here, the same way the work is. Each slice is handed a different + starting point in the platform's own vocabularies and told to begin there. It is a + suggestion rather than a rule, because the caller still has to suit the scenario: a stolen + phone is not a cheerful call whatever this hands out. + """ + from .persona_guides import offered + + people = offered("personality") + accents = offered("accent") + if not people: + return "" + picks = [people[(index + step) % len(people)] for step in range(max(1, wanted))] + said = ( + "\n\nStart from these callers, and move off them only where the scenario calls for " + f"somebody else: {', '.join(picks)}." + ) + if accents: + # Spread several offered accents across this writer's callers rather than naming just one, + # so the suite does not collapse to a single default accent and the agent's speech handling + # is genuinely varied. + spread = [ + accents[(index + step) % len(accents)] + for step in range(min(len(accents), max(2, wanted))) + ] + said += ( + " Give your callers varied accents from the offered set, a different one per caller " + f"where it fits rather than defaulting everyone to the same accent: {', '.join(spread)}. " + "A suite where every caller sounds the same is a missed test of the agent's speech " + "handling, so do not make them all American unless a scenario truly requires it." + ) + return said + + +def brief_for( + contract: AgentContract, mine: Slice, siblings: list[Slice], callers: str +) -> str: + """What one writer is told: its share, what everyone else holds, and the bar. + + Written as a brief rather than a template because a writer that cannot see its siblings + will otherwise write what they are writing. Naming their angles is cheaper than discovering + the overlap at the merge and throwing the loser away. + """ + others = "\n".join(f" - {one.named()}" for one in siblings if one is not mine) + aim = f" {mine.use_case}" + if mine.angle: + aim += f"\n Angle: {mine.angle}" + if mine.why: + aim += f"\n Worth testing because: {mine.why}" + + return ( + f"Write {mine.count} scenario{'s' if mine.count != 1 else ''} for {contract.agent!r}, " + "all of them within this one slice:\n\n" + f"{aim}\n\n" + + ( + "The rest of the suite is being written at the same time by others, covering:\n" + f"{others}\n\nStay out of theirs. A scenario that strays is either a duplicate of " + "somebody else's or a gap in yours.\n\n" + if others + else "" + ) + + "Every scenario carries this use case verbatim in `use_case`, and its own one-line " + "`branch` saying what makes it different from the others you write here. Branches are " + "where the variety lives: the ordinary path, the branch that cannot be completed, the " + "rule under pressure, state that has to carry across turns, the same request against a " + "differently seeded world.\n\n" + "What each one has to be, before you submit it:\n" + " - every value real, read out of the world with inspect_world, never invented\n" + " - an instruction that is a circumstance the person is living through, not a script " + "of lines to say\n" + " - a setup that makes true whatever the instruction presumes, and a ready check that " + "proves it\n" + " - a solution worked out with try_calls first, so the gates are not where you find " + "out it cannot be passed\n" + " - sub-goals named from the shared catalogue, and checks that assert the right call " + "with the right arguments or the right end state, never that something merely happened\n" + " - a scenario a competent agent could plausibly fail. If any correct implementation " + "passes it for free, it teaches nothing and is not worth the run\n\n" + "Look at the world first, and read the sub-goals already defined. Submit each scenario " + "with submit_scenario and then stop: do not save, and do not ask what to do next. " + "Whoever asked for this collects the suite and writes it." + callers + ) + + +async def _write_slice( + contract: AgentContract, + mine: Slice, + siblings: list[Slice], + *, + index: int, + destination: Path, + on_event: Callable[..., Any] | None, + ask: Callable[..., Any] | None, +) -> list[Scenario]: + """One slice, written by its own session. Returns what it proved, unsaved.""" + server, kept = scenario_tools( + contract, + destination, + destination, + wanted=mine.count, + can_save=False, + start_from=[], + ) + logger.info("slice starting: %s (wants %s)", mine.named(), mine.count) + seen = 0 + + def watch(event: Any) -> None: + # Report as they land rather than at the end. A slice that proves its first scenario + # four minutes in is the difference between a run that looks alive and one that does not. + nonlocal seen + if len(kept) != seen: + seen = len(kept) + logger.info("slice %s proved %s of %s", mine.named(), seen, mine.count) + if on_event: + on_event(event) + + allowed = [ + qualified(SCENARIO_SERVER, name) for name in TOOL_NAMES if name != "save_scenarios" + ] + options = ClaudeAgentOptions( + system_prompt=( + f"{load_skill(SKILL)}\n\n## This agent\n\n{contract.brief(with_data=True)}" + f"\n\n## Its world\n\n{world_summary(destination)}" + f"\n\n## Your slice\n\nYou are writing only: {mine.named()}" + ), + allowed_tools=allowed, + mcp_servers={SCENARIO_SERVER: server}, + permission_mode="default", + cwd=str(destination.parent if destination.parent.exists() else Path.cwd()), + setting_sources=[], + max_turns=turns_for(mine.count), + model=chosen_model(), + env=provider_env(), + ) + options.disallowed_tools = list(UNWANTED) + options.hooks = gate_hooks(allowed) + options.can_use_tool = permission_gate(ask, allowed) + options.thinking = thinking_config() + stage = Stage(options, name=f"{SKILL}:{mine.named()[:40]}") + try: + async with stage: + await stage.say( + brief_for(contract, mine, siblings, callers_for(index, mine.count)), + on_event=watch, + ) + except Exception as broke: # noqa: BLE001 - one slice failing must not lose the others + logger.warning("slice %s failed after %s: %s", mine.named(), len(kept), broke) + if on_event: + on_event({"type": "slice_failed", "slice": mine.named(), "why": str(broke)[:300]}) + return list(kept) + logger.info("slice %s finished with %s of %s", mine.named(), len(kept), mine.count) + return list(kept) + + +def merged(written: list[list[Scenario]]) -> list[Scenario]: + """One suite out of several writers, with folder-name collisions renamed rather than dropped. + + Asking for twenty scenarios has to return twenty. Two scenarios may legitimately share a use + case and a branch and still test different things, so sharing them is not a reason to discard + one; an earlier version dropped those and quietly returned eighteen. + + The one collision that cannot be tolerated is the folder name, because the folder is where a + scenario lives on disk and the loser would overwrite the winner. Those are given a numbered + suffix instead of being thrown away, so nothing generated is ever lost. + """ + suite: list[Scenario] = [] + taken: set[str] = set() + for batch in written: + for one in batch: + if one.name in taken: + stem, suffix = one.name, 2 + while f"{stem}-{suffix}" in taken: + suffix += 1 + one = one.model_copy(update={"name": f"{stem}-{suffix}", "scenario_key": ""}) + logger.info("renamed a duplicate folder name to %s", one.name) + taken.add(one.name) + suite.append(one) + return suite + + +def _suite_summary(suite: list[Scenario]) -> str: + """The whole suite as a reviewer needs to see it: what each row claims to test.""" + return "\n".join( + f" {one.name} | use case: {one.use_case} | branch: {one.branch} | passes when: {one.tests}" + for one in suite + ) + + +async def gaps_in( + contract: AgentContract, + suite: list[Scenario], + *, + destination: Path, + wanted: int, + ask: Callable[..., Any] | None = None, +) -> list[Slice]: + """What the finished suite is missing, as slices that would fill it. + + Nobody looks at a suite written in parallel. Each writer sees its own slice and the merge + only removes collisions, so a use case that came back one short, or an obvious branch that + every writer assumed somebody else had, survives to the end and nobody notices. This is the + one pass that reads the suite as a whole. + """ + if not suite: + return [] + found: list[Slice] = [] + + @tool( + "submit_gaps", + "The gaps worth filling in this suite, as the slices that would fill them. Return " + "nothing when the suite covers what it should: a suite that is finished is a real " + "answer, and inventing work to report is worse than saying so.", + schema( + { + "gaps": { + "type": "array", + "description": "One entry per gap. Empty when the suite is covering what " + "it should.", + "items": { + "type": "object", + "properties": { + "use_case": {"type": "string"}, + "angle": { + "type": "string", + "description": "The scenario that is missing, in one line.", + }, + "why": {"type": "string"}, + }, + "required": ["use_case", "angle"], + }, + } + }, + ["gaps"], + ), + ) + async def submit_gaps(args: dict[str, Any]) -> dict[str, Any]: + for one in args.get("gaps") or []: + if not isinstance(one, dict): + continue + case = str(one.get("use_case") or "").strip() + if case: + found.append( + Slice( + use_case=case, + angle=str(one.get("angle") or "").strip(), + count=1, + why=str(one.get("why") or "").strip(), + ) + ) + return { + "content": [ + {"type": "text", "text": f"{len(found)} gap(s) recorded. Nothing else to do."} + ] + } + + server = create_sdk_mcp_server(name=REVIEW_SERVER, version="0.1.0", tools=[submit_gaps]) + allowed = [qualified(REVIEW_SERVER, "submit_gaps")] + options = ClaudeAgentOptions( + system_prompt=( + "You are reviewing a suite of tests somebody else wrote for an AI agent, in " + "parallel, each writer blind to the others. Your only job is to say what is " + "missing.\n\n" + "Look for: a use case of this agent that nothing covers; a use case covered only " + "on its ordinary path, where the branch that cannot be completed or the rule under " + "pressure is the interesting one; two rows that are the same test under different " + "names, leaving the branch one of them claimed uncovered.\n\n" + "Judge coverage of the agent, not of the plan. Do not ask for more of what is " + "already well covered, and do not report a gap you cannot name a scenario for. " + "A suite of the right size that covers what matters is finished, and saying so is " + f"the useful answer.\n\n## This agent\n\n{contract.brief()}" + ), + allowed_tools=allowed, + mcp_servers={REVIEW_SERVER: server}, + permission_mode="default", + cwd=str(destination.parent if destination.parent.exists() else Path.cwd()), + setting_sources=[], + max_turns=8, + model=chosen_model(), + env=provider_env(), + ) + options.disallowed_tools = list(UNWANTED) + options.hooks = gate_hooks(allowed) + options.can_use_tool = permission_gate(ask, allowed) + stage = Stage(options, name=f"{SKILL}:review") + try: + async with stage: + await stage.say( + f"This suite has {len(suite)} scenarios against a target of {wanted}:\n\n" + f"{_suite_summary(suite)}\n\n" + "Say what it is missing, then submit_gaps. Submit an empty list if it is " + "covering what it should." + ) + except Exception: # noqa: BLE001 - a review that fails leaves the suite as written + return [] + return found + + +async def write_in_parallel( + contract: AgentContract, + *, + out: Path | None = None, + wanted: int = 10, + use_cases: list[str] | None = None, + slices: list[dict] | None = None, + at_once: int = AT_ONCE, + rounds: int = TOP_UP_ROUNDS, + on_event: Callable[..., Any] | None = None, + ask: Callable[..., Any] | None = None, +) -> list[Scenario]: + """Write a suite with one session per slice, review it, fill what it missed, and save once. + + Sequentially, a suite costs roughly three turns a scenario against one budget, which is why + asking for forty stopped around twenty-five. Here the work is split into slices that run at + the same time, so the wall clock is the slowest slice rather than the sum of all of them. + + Saving stays here, once, for a reason: ``save_scenarios`` regenerates the index and deletes + any folder it does not know about, so letting the writers save would have each of them + remove the others' work. + """ + destination = out or artifact_dir(contract.agent) + cases = [case for case in (use_cases or contract.real_use_cases) if case.strip()] + if not cases and not slices: + # Nothing to partition on. One writer, the ordinary path, rather than no scenarios. + return await write(contract, out=destination, wanted=wanted, on_event=on_event, ask=ask) + + at_once = max(1, min(at_once or AT_ONCE, MOST_AT_ONCE)) + allocation = planned(wanted, cases, slices) + logger.info( + "writing %s scenarios across %s slices, %s at a time: %s", + wanted, + len(allocation), + at_once, + ", ".join(f"{one.named()} x{one.count}" for one in allocation), + ) + if on_event: + on_event( + { + "type": "planned", + "slices": [(one.named(), one.count) for one in allocation], + "at_once": at_once, + } + ) + + limit = asyncio.Semaphore(at_once) + + async def guarded(mine: Slice, siblings: list[Slice], index: int) -> list[Scenario]: + async with limit: + return await _write_slice( + contract, + mine, + siblings, + index=index, + destination=destination, + on_event=on_event, + ask=ask, + ) + + written = await asyncio.gather( + *(guarded(one, allocation, index) for index, one in enumerate(allocation)), + return_exceptions=False, + ) + suite = merged([load_scenarios(destination), *written]) + + # Read the whole thing and fill what nobody covered. Bounded, because a reviewer asked + # twice will always find something smaller to say. + for _ in range(max(0, rounds)): + if len(suite) >= wanted: + break + missing = await gaps_in( + contract, suite, destination=destination, wanted=wanted, ask=ask + ) + missing = missing[: max(0, wanted - len(suite))] + if not missing: + break + if on_event: + on_event({"type": "topping_up", "slices": [one.named() for one in missing]}) + logger.info( + "topping up %s of %s with %s more slices: %s", + len(suite), + wanted, + len(missing), + ", ".join(f"{one.named()} x{one.count}" for one in missing), + ) + more = await asyncio.gather( + *( + guarded(one, missing, len(allocation) + index) + for index, one in enumerate(missing) + ), + return_exceptions=False, + ) + before = len(suite) + suite = merged([suite, *more]) + allocation = [*allocation, *missing] + if len(suite) == before: + break + + write_scenarios(suite, destination, load_catalogue(destination)) + logger.info("suite saved: %s of %s asked for", len(suite), wanted) + if on_event: + on_event({"type": "saved", "kept": len(suite), "asked": wanted}) + return load(destination) + + async def write( contract: AgentContract, *, diff --git a/src/fi/alk/harness/skills/harness.md b/src/fi/alk/harness/skills/harness.md index dd3597d2..4f410ea7 100644 --- a/src/fi/alk/harness/skills/harness.md +++ b/src/fi/alk/harness/skills/harness.md @@ -1,40 +1,35 @@ # The harness -You are a harness that builds test suites for AI agents. +You build test suites for AI agents, working with a person in a conversation they can see all of. -Somebody has an agent — a support assistant, a voice ordering system, something that books or -cancels or looks things up — and no reliable way to know whether it works. Reading its +Somebody has an agent, a support assistant, a voice ordering system, something that books or +cancels or looks things up, and no reliable way to know whether it works. Reading its transcripts tells you what it said, not whether what it said was true. Your job is to produce something better: a real environment the agent's tools act on, a set of tests that are provably worth running, and results that can be trusted because they were settled by code rather than by opinion. -You work with a person, in a conversation. They can see everything you do. - -**You are this thing, so speak as it.** Your tools refuse you sometimes; that is the design, and -it is still you being refused. "Two scenarios ended up sharing a use case, fixing them" is what -happened. "The harness needs unique use cases" is the same event narrated from outside, and it -reads as blaming a system you are not part of. Never refer to the harness in the third person, -and never explain your own tooling's rules as though they were somebody else's requirements: say -what you are doing about it. - -Where a limit genuinely is not yours, say whose it is and what to do: a stage you cannot reach -from here, a credential nobody has set, an agent that cannot be run without editing it. Those are -facts about the situation, not deflections. +**Write as the one doing the work.** "Two scenarios ended up sharing a use case, fixing them" is +what happened. "The harness needs unique use cases" is the same event narrated from outside, as +though a system you were not part of had imposed it on you. Report what you did and what you are +doing about it, including when a tool refuses you. Where a limit is genuinely someone else's, say +whose and what to do: a stage you cannot reach from here, a credential nobody has set, an agent +that cannot be run without editing it. Those are facts about the situation, not deflections. ## What you produce, in order -Four stages. Each one produces something the next needs, and each is a conversation you can be -interrupted in, corrected in, and resumed in. +Each stage produces something the next needs, and each is a conversation you can be interrupted +in, corrected in, and resumed in. **1. Understand.** Read the agent's source and write down what is verifiably true about it: the tools it really has with their exact argument names and permitted values, the rules it obeys, what it depends on, its data, and what it is for. This is the contract, and everything afterwards is confined to it. -**2. Build the environment.** From that contract, build the world the agent acts in — a database, -a service, whatever its tools need — so that every call it makes resolves against something real -and gets a truthful answer, including a truthful refusal. Also written here: the prompt for the +**2. Build or provision the environment.** The world the agent acts in, so that every call it +makes resolves against something real and gets a truthful answer, including a truthful refusal. +Either build it from the contract, a database, a service, whatever its tools need, or provision +the runtime the agent already ships, when it ships one. Also written here: the prompt for the person the agent talks to, and the catalogue of named sub-goals the agent can be checked on. **3. Write the scenarios.** Each one changes the world a little, gives the person a task, and diff --git a/src/fi/alk/harness/skills/write-scenarios/SKILL.md b/src/fi/alk/harness/skills/write-scenarios/SKILL.md index 86ac6c86..6e49a593 100644 --- a/src/fi/alk/harness/skills/write-scenarios/SKILL.md +++ b/src/fi/alk/harness/skills/write-scenarios/SKILL.md @@ -19,9 +19,19 @@ afterwards. ``` name short identifier; it becomes this scenario's folder -use_case which of the agent's use cases this belongs to -tests one line: what this scenario is trying to find out -instruction the task, written to the person the agent is serving +use_case which of the agent's use cases this belongs to, copied from the contract + word for word. Not paraphrased, not shortened, not reworded to fit this + scenario: results are grouped by matching this string exactly, so a + rewording silently becomes a group of its own +branch what makes this one different from its siblings in that use case +tests one line: the condition this scenario passes on. It is shown to people as + "passes when", so write it to complete that phrase. Both this and branch are + read by whoever looks at results, so write them about the agent's behaviour + and never about how the scenario was built. "synthetic", "seeded", + "setup_code", "fixture" and the like name your own machinery, not anything + the agent did, and they are noise in a report +instruction what this person is trying to achieve, written to them, plus everything + they need to pursue it without inventing anything persona who that person is: identity, communication style, languages/accent and characteristics setup_code Python: def setup(world) — what this scenario changes first ready_code Python: def ready(world) — is the world ready for this scenario @@ -45,7 +55,7 @@ scenario that looks fine and measures nothing. | | What it is | What it must never contain | |---|---|---| -| **instruction** | what the person on the other side is living through | the answer, the checks, or facts they could not know | +| **instruction** | what the person on the other side is living through | the answer, the checks, facts they could not know, or anything the agent is expected to do | | **setup** | the world's condition | anything the person is supposed to say | | **checks** | the hidden pass or fail rules | anything the agent was told | @@ -109,6 +119,81 @@ not exist, and no lookup will ever find them. **Possessing and volunteering are separate.** Whether the person offers a value unprompted is the scenario's business. Whether they have it at all is not optional. +**Write the instruction as an objective, not a situation.** A caller who is told what happened +narrates it; a caller who is told what they want pursues it. Open with the goal in their own words +("Get put right"), not with the history that led to it ("You were charged +"), then give them the facts they hold, the values they can be asked for, and what +they will only say once asked for it. Every value read out of the world, never invented. + +**Never tell the caller what the agent will do.** This is the single most common way a scenario +silently stops measuring anything. The agent's moves are what the scenario is testing, so a caller +who has been told to expect them will play along whether or not they happen, and the check passes +on a conversation that never earned it. Write only what this person knows before the call starts. + +``` +BAD The agent will tell you about . Accept it and say yes when + asked to confirm. + (the scenario is testing whether the agent discloses . A caller + primed to accept it agrees even when the agent never says it, so the run + reports a pass for behaviour that did not occur) + +GOOD You want . You will accept if there is one, but + you want to know before you agree to anything. + (the caller's own position. If the agent discloses, they accept; if it does + not, they ask, and the transcript records which happened) +``` + +The same rule covers every phrasing of it: "the agent will send you ", "they will offer +you ", "they should transfer you". Give the person the value, the preference or the +problem they arrived with. What the agent does about it is the measurement, so it cannot also be +part of the brief. + +**The test that catches all of it: could this person say the sentence out loud?** The instruction +is read by someone who has never seen the agent's design and does not know how it works. So a +parenthetical explaining where the agent is supposed to find a value is not a smaller version of +the mistake, it is the same mistake in a quieter voice. + +``` +BAD Your : (the agent should find this from your ) + (the caller has no idea the agent has records, let alone which one. The note is + written for whoever reads the scenario, not for the person on the call, and it + tells them the mechanism that is being tested) + +GOOD Your is the same one you used last time. You do not remember the + exact address and would rather not look it up. + (now the caller has a reason to expect the agent to know, which is what makes + the agent's lookup worth testing, without being told the lookup exists) +``` + +Pre-agreeing to something the agent has not done yet is the most damaging form. "You have already + that the agent will " hands the agent a pass: the person confirms it +whether or not it happened. Write what they have done, never what they have done in response to an +action the agent has not taken. + +**Steps that happen outside the conversation need a state, not a response.** Some flows depend on +the person doing something the simulation cannot actually perform: following a link, checking +another device, reading a message. The temptation is to write the person's answer in advance, and +that is exactly the pass-handing form above, because the answer arrives whether or not the agent +ever asked. + +Give them a standing disposition instead, and let the agent's action trigger it: + +``` +BAD The agent will send you . Tell them you have + completed it when asked. + (the scenario is testing whether the agent sends it. This person confirms + completing it even in a run where nothing was ever sent) + +GOOD You have your with you and you are willing to follow anything you + are sent. You have not been sent anything yet. + (a state. If the agent sends it, this person can act on it and say so + truthfully. If the agent never does, they have nothing to confirm, and the + transcript shows the difference) +``` + +The closing sentence matters: stating what has **not** happened yet is what stops the person +assuming it has. + **Use persona deliberately.** An accent, personality or characteristic belongs in `persona` only when it changes the conversational risk being exercised. A rude customer is a different scenario from a polite one only if the agent must handle that difference. Persona never contains the @@ -157,6 +242,21 @@ rule under pressure, the state that has to carry, the same request against a dif world. Keep that plan concise and continue immediately unless the person explicitly asked to review it. + +**Pass your plan to it.** The tool takes the split as an argument, and you have just read the +world and know which use cases have +something in them; it is the part of this only you can do. Each slice names its use case, +the angle it should take, how many scenarios it is worth, and why. Left to itself the work is +divided evenly, which is how a use case with one real branch pads to three and one with six gets +three. + +A large request comes back a batch at a time rather than all at once, with the rest offered. When +that happens, show what came back and ask whether to carry on, change direction first, or stop. +Do not silently loop until the number is reached. + +Use `submit_scenario` for what it is good at: one scenario somebody asked for by name, a +replacement for one that came back wrong, or filling a specific gap in a suite that already +exists. Anything described as a number of scenarios is a suite. After inspecting the world, submit the first scenario in the same response. Then prove and save one scenario at a time. Never silently compose the whole suite before the next tool call: the UI must show progress, and already-proved work must survive a stopped or timed-out model turn. @@ -212,9 +312,7 @@ Every stance still obeys the bar above: a real person could bring it, a competen fail it, and the values are real. A stance chooses *what to look at*, never whether the scenario has to be honest. -Two rules keep this from turning into noise. **Each scenario carries one use case, and no two -scenarios carry the same one** — a duplicate is either the same test twice or one of them is -mislabelled, and it hides a gap while appearing to fill it. And a stance that produces nothing new +Two rules keep this from turning into noise. **Each scenario carries one use case and one branch, and no two scenarios carry the same pair**: a duplicate is either the same test twice or one of them is mislabelled, and it hides a gap while appearing to fill it. Several scenarios sharing a use case is normal and expected; that is what branches are for. What is not allowed is two rows that agree on both. And a stance that produces nothing new for a given agent produces nothing: an agent with no rules to bend does not need an adversarial scenario invented for it. @@ -376,10 +474,13 @@ hides the problem and everything built afterwards inherits it. 1. `inspect_world` with no table, then look at the ones that matter. Read the sub-goals already defined. 2. Read the agent's hard rules. Each one is a branch waiting to be written. -3. For each scenario: work out the solution, `try_calls` it with your `setup_code`, then +3. For a suite, say how you are splitting it across the agent's use cases, then write and + submit them one at a time. A large ask comes back a batch at a time rather than all at once. + writes the whole thing and saves it, and you report what came back. +4. For a single scenario: work out the solution, `try_calls` it with your `setup_code`, then `submit_scenario`. -4. Read what comes back. A refusal names which gate failed and why. -5. `save_scenarios` when you have the number that was asked for. +5. Read what comes back. A refusal names which gate failed and why. +6. `save_scenarios` when you have the number that was asked for. ## Finishing diff --git a/src/fi/simulate/simulation/engines/livekit.py b/src/fi/simulate/simulation/engines/livekit.py index ff10928c..5b3d46f3 100644 --- a/src/fi/simulate/simulation/engines/livekit.py +++ b/src/fi/simulate/simulation/engines/livekit.py @@ -16,7 +16,16 @@ try: from livekit import api, rtc - from livekit.agents import Agent, AgentSession, RunContext, function_tool, metrics + from livekit.agents import ( + Agent, + AgentSession, + AudioConfig, + BackgroundAudioPlayer, + RunContext, + function_tool, + metrics, + ) + from livekit.agents.voice.background_audio import BuiltinAudioClip from livekit.agents.types import ( ATTRIBUTE_TRANSCRIPTION_TRACK_ID, TOPIC_TRANSCRIPTION, @@ -320,8 +329,80 @@ async def start_session( room=room, room_options=RoomOptions(**room_kwargs), ) + await self._maybe_start_background_audio(room, session) return session + async def _maybe_start_background_audio( + self, room: "rtc.Room", session: "AgentSession" + ) -> None: + """Mix caller-side ambient noise under the simulated caller, if the run asked for it. + + Off unless HARNESS_BACKGROUND_NOISE names a source: a LiveKit builtin clip name, or an + http(s) URL to an ambient file. Any failure is swallowed, because a call without ambience is + preferable to a dropped one. + """ + source = os.environ.get("HARNESS_BACKGROUND_NOISE", "").strip() + if not source: + return + + def _download() -> str | None: + import tempfile + import urllib.request + + try: + suffix = ( + ".mp3" if ".mp3" in source else ".ogg" if ".ogg" in source else ".wav" + ) + with urllib.request.urlopen(source, timeout=15) as response: + data = response.read() + handle = tempfile.NamedTemporaryFile(delete=False, suffix=suffix) + handle.write(data) + handle.close() + return handle.name + except Exception: + return None + + try: + volume = float(os.environ.get("HARNESS_BACKGROUND_NOISE_VOLUME", "0.3")) + if source.startswith(("http://", "https://")): + clip_source: Any = await asyncio.to_thread(_download) + if not clip_source: + return + self._background_noise_file = clip_source + else: + clip_source = getattr(BuiltinAudioClip, source, None) + if clip_source is None: + logger.warning("background audio clip %r is not one LiveKit ships", source) + return + player = BackgroundAudioPlayer( + ambient_sound=AudioConfig(clip_source, volume=volume) + ) + await player.start(room=room, agent_session=session) + self._background_player = player + except Exception: + logger.warning("background audio not started", exc_info=True) + + async def _stop_background_audio(self) -> None: + """Close the ambience player and remove any clip downloaded for it. + + Without this the mixer task, its audio source and the published track outlive the call, + and a suite leaks one of each (plus a temp file) per scenario. + """ + player = getattr(self, "_background_player", None) + if player is not None: + self._background_player = None + try: + await player.aclose() + except Exception: + logger.warning("background audio not closed cleanly", exc_info=True) + downloaded = getattr(self, "_background_noise_file", None) + if downloaded: + self._background_noise_file = None + try: + Path(downloaded).unlink(missing_ok=True) + except OSError: + logger.warning("background audio clip not removed: %s", downloaded) + def open_conversation(self) -> None: if self._session is None: raise RuntimeError("simulator_session_not_started") @@ -1305,6 +1386,13 @@ def on_target_transcription( details={"exception_type": type(exc).__name__}, ) finally: + # The ambience belongs to the caller agent, not the engine. Guarded because teardown + # must never be the reason a case fails. + if customer_agent is not None: + try: + await customer_agent._stop_background_audio() + except Exception: + logger.warning("background audio not closed cleanly", exc_info=True) if target_transcription_handler_registered: room.unregister_text_stream_handler(TOPIC_TRANSCRIPTION) pending_target_transcriptions.clear() @@ -1561,6 +1649,12 @@ async def _create_customer_agent( default_language=( simulator.stt.language if simulator is not None else None ), + variables={"instruction": persona.situation or ""}, + # Delivery cues are Cartesia only. Passing the provider here rather than reading it + # inside the prompt keeps the decision where the provider is actually known. + tts_provider=( + simulator.tts.provider if simulator is not None else None + ), ) if simulator is None: voice_provider = os.environ.get( diff --git a/src/fi/simulate/simulation/voice_prompt.py b/src/fi/simulate/simulation/voice_prompt.py index 0432a2e5..320cbf98 100644 --- a/src/fi/simulate/simulation/voice_prompt.py +++ b/src/fi/simulate/simulation/voice_prompt.py @@ -1,11 +1,15 @@ from __future__ import annotations +import logging +import re from typing import Any, Literal, Mapping from fi.simulate.simulation.models import Persona CallType = Literal["inbound", "outbound"] +logger = logging.getLogger(__name__) + VOICE_PERSONALITY_GUIDES: dict[str, str] = { "friendly and cooperative": "Be warm, approachable, and willing to work together. Show genuine interest and maintain a positive, collaborative attitude.", "professional and formal": "Maintain a business-like demeanor. Use formal language, stay focused, and keep interactions professional.", @@ -35,6 +39,32 @@ } +# Delivery cues Cartesia renders and every other engine speaks aloud as words. Added only when +# Cartesia is definitely the voice, because Deepgram's Aura neither renders nor strips them, so +# a caller on Aura would literally say "left bracket laughter right bracket". +# +# Deliberately narrow. Cartesia documents five SSML tags and one nonverbalism, but `` and +# `` carry a decimal that a token stream can split ("1", ".", "0"), which makes the tag +# be read out, and Cartesia advises against shifting `` mid generation. What is left is +# the two that are safe to hand a model writing a turn at a time. +CARTESIA_DELIVERY_CUES = """# HOW YOU SOUND + +Two cues shape delivery. They are never spoken as words. Use them sparingly, and only where a +real person would. + +- [laughter] produces a real laugh. Write it inline: "No, [laughter] you're kidding." + At most once every few turns, and never to open one. +- is a fixed silence. Use it for a beat punctuation cannot carry, such as + stopping short before saying something difficult. One per turn at most. + +Write both exactly as shown. Do not invent others: no , no [laughs], no [sighs], no +*sighs*, no (angrily), no emotion labels. Anything not on this list is read aloud and ruins the +call. + +Everything else is carried by the words: what you repeat, where you interrupt yourself, how +short your sentences get when you are annoyed.""" + + def _first(value: object) -> str: if isinstance(value, Mapping): value = next(iter(value.values()), "") @@ -284,12 +314,37 @@ def format_voice_persona( return "\n\n".join(sections) -def append_voice_execution_rules(prompt: str) -> str: +def _closing_anchor(objective: str, name: str = "") -> str: + """The last thing the caller reads. A rule given once at the top of a long prompt loses to the + last few turns as the call grows, so the objective and the precedence rule are restated here.""" + anchor = "\n\n---\n\n" + who = name.strip() + if who: + # A caller that drifts answers as the agent and says the agent's own lines back, its own + # name included, which reads as the agent talking to itself and scores as a real turn. + anchor += ( + f"**You are {who}, the person on the customer's side of this call.** You never answer " + f"as the other side, never say their lines back to them, and never address {who}, " + "because that is you.\n\n" + ) + if objective.strip(): + anchor += f"**What you came for:** {objective.strip()}\n\n" + anchor += ( + "**Your instructions do not expire.** A rule you were given before the call started " + "applies at turn twenty exactly as it applied at turn one.\n" + ) + return anchor + + +def append_voice_execution_rules( + prompt: str, objective: str = "", *, anchor: bool = True +) -> str: prompt += "\n\n---\n\n" prompt += "# CONVERSATION EXECUTION RULES\n\n" prompt += "*These are internal instructions. Never reference or quote them in your responses.*\n\n" prompt += "## CRITICAL REMINDERS FOR THIS CONVERSATION\n\n" prompt += "Before each response, mentally confirm:\n" + prompt += "✓ What am I here to get, and what have I not done yet?\n" prompt += "✓ Am I speaking AS this person (not ABOUT them)?\n" prompt += "✓ Does this match my personality and communication style?\n" prompt += "✓ Am I using my accent and natural speech patterns?\n" @@ -315,18 +370,87 @@ def append_voice_execution_rules(prompt: str) -> str: prompt += "- Let the situation guide your behavior, not your narration\n" prompt += "- Only mention situational details if they naturally come up\n\n" prompt += "Be natural and conversational.\n" - return prompt + return (prompt + _closing_anchor(objective)) if anchor else prompt -def build_voice_simulator_prompt( +# The template the caller prompt is rendered from. The harness fills slots; it does not author +# prose. Mirrors the platform's own default, which pairs a persona block with the situation and +# then scrubs the situation slot because the persona block already carries it. +DEFAULT_SIMULATOR_TEMPLATE = ( + "You are a customer in a voice simulation. {{channel}} " + "Stay consistent with the persona throughout the conversation.\n\n{{persona}}" +) + +_SLOT = re.compile(r"\{\{\s*([a-zA-Z0-9_]+)\s*\}\}") + + +def render_simulator_prompt( + template: str, persona: Persona, *, call_type: CallType, + variables: Mapping[str, Any] | None = None, agent_name: str | None = None, additional_instructions: str | None = None, default_language: str | None = None, + tts_provider: str | None = None, ) -> str: - channel = ( + """Fill a caller-prompt template, the way the platform fills its own. + + ``{{persona}}`` becomes the formatted persona block, ``{{channel}}`` the call direction + sentence, and every other ``{{slot}}`` is taken from ``variables``. ``{{situation}}`` is + dropped rather than filled, because the persona block already states the situation and the + platform removes it for the same reason. + + A template that cannot be rendered is returned to the caller unfilled rather than raising, so + a bad template degrades the call instead of ending the run. + """ + values = dict(variables or {}) + try: + persona_text = format_voice_persona( + persona, call_type=call_type, default_language=default_language + ) + except Exception: + logger.exception("persona_format_failed") + persona_text = "" + values.setdefault("persona", persona_text) + values.setdefault("channel", _channel_sentence(call_type, agent_name)) + + def fill(match: "re.Match[str]") -> str: + name = match.group(1) + if name == "situation": + return "" + if name in values: + return str(values[name]) + logger.warning("simulator_prompt_slot_unfilled", extra={"slot": name}) + return "" + + try: + prompt = _SLOT.sub(fill, template) + except Exception: + logger.exception("simulator_prompt_render_failed") + return template + # Tidy the hole a dropped situation slot leaves behind, as the platform does. + prompt = re.sub(r"Currently,\s*[.]", "", prompt) + prompt = re.sub(r"[ \t]{2,}", " ", prompt).strip() + + if tts_provider and tts_provider.strip().lower() == "cartesia": + prompt += "\n\n" + CARTESIA_DELIVERY_CUES + # The generic style rules go first and the scenario's own instructions after them: whatever + # lands last survives a long call best, and the scenario's rules are the ones worth keeping. + prompt = append_voice_execution_rules(prompt, anchor=False) + if additional_instructions and additional_instructions.strip(): + prompt += ( + "\n\n# ADDITIONAL SIMULATOR INSTRUCTIONS\n\n" + + additional_instructions.strip() + ) + return prompt + _closing_anchor( + persona.outcome or "", str(_persona_data(persona).get("name") or "") + ) + + +def _channel_sentence(call_type: CallType, agent_name: str | None) -> str: + return ( f"You will make a call to an agent named {agent_name}." if call_type == "inbound" and agent_name else "You will make a call to an agent." @@ -335,21 +459,34 @@ def build_voice_simulator_prompt( if agent_name else "You will receive a call from an agent." ) - prompt = ( - "You are a customer in a voice simulation. " - f"{channel} Stay consistent with the persona throughout the conversation.\n\n" - + format_voice_persona( - persona, - call_type=call_type, - default_language=default_language, - ) + + +def build_voice_simulator_prompt( + persona: Persona, + *, + call_type: CallType, + agent_name: str | None = None, + additional_instructions: str | None = None, + default_language: str | None = None, + template: str | None = None, + variables: Mapping[str, Any] | None = None, + tts_provider: str | None = None, +) -> str: + """The caller prompt for one simulated customer. + + Renders ``template`` when one is supplied, and the shipped default otherwise, so a run with no + template configured still produces the prompt it always did. + """ + return render_simulator_prompt( + template or DEFAULT_SIMULATOR_TEMPLATE, + persona, + call_type=call_type, + variables=variables, + agent_name=agent_name, + additional_instructions=additional_instructions, + default_language=default_language, + tts_provider=tts_provider, ) - if additional_instructions and additional_instructions.strip(): - prompt += ( - "\n\n# ADDITIONAL SIMULATOR INSTRUCTIONS\n\n" - + additional_instructions.strip() - ) - return append_voice_execution_rules(prompt) __all__ = [ @@ -357,6 +494,9 @@ def build_voice_simulator_prompt( "VOICE_COMMUNICATION_STYLE_GUIDES", "VOICE_PERSONALITY_GUIDES", "append_voice_execution_rules", + "DEFAULT_SIMULATOR_TEMPLATE", + "CARTESIA_DELIVERY_CUES", "build_voice_simulator_prompt", + "render_simulator_prompt", "format_voice_persona", ] diff --git a/tests/test_harness.py b/tests/test_harness.py index a06b4a59..9c624cc4 100644 --- a/tests/test_harness.py +++ b/tests/test_harness.py @@ -6294,6 +6294,70 @@ def result(self, call_execution_id, payload): assert reported.calls == {"a": "ce-a", "b": "ce-b"} +def test_scenario_carries_the_identity_the_hosted_scheduler_reads(): + """The guest scheduler reads ``scenario_key`` and ``scenario_id`` off every scenario. + + Both are plain attributes rather than optional extras: a pydantic model raises AttributeError + for a field it never declared, so a missing one fails at the first read rather than degrading. + """ + one = Scenario(name="dana-books-uberx-saved-card") + assert one.scenario_key == "dana-books-uberx-saved-card" + # Assigned by the platform at pre-allocation, never written at generation. + assert one.scenario_id == "" + + +@pytest.mark.parametrize( + "name,expected", + [ + ("Dana Books - Café", "dana-books-caf"), + ("already-a-slug", "already-a-slug"), + (" Spaced Out ", "spaced-out"), + ], +) +def test_scenario_key_is_ascii_and_slugged(name, expected): + assert Scenario(name=name).scenario_key == expected + + +def test_scenario_key_never_empties_onto_a_shared_idempotency_key(): + """A name with nothing ASCII in it still gets its own key rather than an empty one.""" + keys = {Scenario(name=name).scenario_key for name in ("日本語", "中文", "한국어")} + assert all(key.startswith("scenario-") for key in keys) + assert len(keys) == 3 + + +def test_background_noise_is_decided_the_same_way_twice(): + """A coin flip here made a seeded run unreproducible; the name decides it instead.""" + assert Scenario(name="a-b-c").background_noise == Scenario(name="a-b-c").background_noise + assert Scenario(name="x", background_noise="street").background_noise == "street" + + +def test_background_noise_needs_opting_in(monkeypatch): + """Silence is what a run with no environment set falls into: noise shortens calls, so it is + asked for rather than escaped. Anything unrecognised stays silent instead of turning it on.""" + from fi.alk.harness.background_noise import enabled + + monkeypatch.delenv("ALK_BACKGROUND_NOISE", raising=False) + assert enabled() is False + for off in ("", "0", "off", "false", "no", "ture"): + monkeypatch.setenv("ALK_BACKGROUND_NOISE", off) + assert enabled() is False, off + for on in ("1", "on", "TRUE", "yes"): + monkeypatch.setenv("ALK_BACKGROUND_NOISE", on) + assert enabled() is True, on + + +def test_a_transcript_line_keeps_one_speaker_not_two(): + """``agent: assistant: ...`` reached the judges as two speakers deep for every turn.""" + from fi.alk.harness.run.simulation import _said + + def said(line): + turn = _said(line) + return turn.speaker, turn.text + + assert said("assistant: Hi Dana.") == ("agent", "Hi Dana.") + assert said("user: I need a ride.") == ("customer", "I need a ride.") + # A colon inside speech is not a speaker label. + assert said("assistant: Call at 3:30 PM.") == ("agent", "Call at 3:30 PM.") def test_platform_call_start_uses_existing_ongoing_status_flow(): from fi.alk.harness import platform diff --git a/tests/test_voice_prompt.py b/tests/test_voice_prompt.py index a06009a1..1a21f0b4 100644 --- a/tests/test_voice_prompt.py +++ b/tests/test_voice_prompt.py @@ -59,6 +59,20 @@ def test_simulator_instructions_supplement_scenario_prompt() -> None: assert "Ask for an escalation" in prompt assert "Your specialist appointment was cancelled without notice." in prompt assert "Get a new appointment time and confirm the clinic location." in prompt - assert prompt.index("# ADDITIONAL SIMULATOR INSTRUCTIONS") < prompt.index( + # The call's own instructions come after the general rules, and the objective closes the + # prompt: what lands last is what survives a long conversation. + assert prompt.index("# ADDITIONAL SIMULATOR INSTRUCTIONS") > prompt.index( "# CONVERSATION EXECUTION RULES" ) + assert prompt.rstrip().endswith("applies at turn twenty exactly as it applied at turn one.") + + +def test_prompt_closes_by_naming_who_the_caller_is() -> None: + """A drifting caller answered as the agent and addressed itself by its own name, which reads + as the agent talking to itself. The identity is restated last, where it survives a long call.""" + prompt = build_voice_simulator_prompt(_persona(), call_type="inbound") + + tail = prompt.rsplit("---", 1)[-1] + assert "You are Priya" in tail + assert "never address Priya" in tail + assert tail.index("You are Priya") < tail.index("What you came for:")