From ac3f9a2d6ff07e410c16139f8e663d78c3e8e62c Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 29 Aug 2026 18:09:51 +0530 Subject: [PATCH 01/45] feat(harness): give the build stage a shell, a file editor and the repository --- src/fi/alk/harness/backends/claude.py | 6 +++++- src/fi/alk/harness/build.py | 15 ++++++++++++--- src/fi/alk/harness/config.py | 7 +++---- src/fi/alk/harness/world/workspace.py | 13 ++++++------- 4 files changed, 26 insertions(+), 15 deletions(-) diff --git a/src/fi/alk/harness/backends/claude.py b/src/fi/alk/harness/backends/claude.py index a15e2dbc..609b6325 100644 --- a/src/fi/alk/harness/backends/claude.py +++ b/src/fi/alk/harness/backends/claude.py @@ -183,7 +183,11 @@ def create(self, spec: SessionSpec) -> ClaudeSession: # is consulted, so a stage could rewrite an artifact by hand and skip the tool whose # whole job is to validate that change. options.permission_mode = "default" - options.disallowed_tools = list(UNWANTED) + # Deny only what this stage was not granted. A stage that asks for Bash or Write + # means it, and a blanket denial here would silently outrank its own tool list. + options.disallowed_tools = [ + name for name in UNWANTED if name not in set(allowed) + ] options.hooks = gate_hooks(allowed) options.can_use_tool = spec.permission_override or permission_gate( spec.ask, allowed diff --git a/src/fi/alk/harness/build.py b/src/fi/alk/harness/build.py index c6503395..51e53f0c 100644 --- a/src/fi/alk/harness/build.py +++ b/src/fi/alk/harness/build.py @@ -237,10 +237,19 @@ def open_stage( f"{load_skill(SKILL)}\n\n## This agent\n\n{contract.brief(with_data=True)}" + environment_note ), - # No file tools and no shell. Everything this stage can do goes through a tool that - # executes it and reports back, which is what makes the guardrails meaningful. servers={WORLD_SERVER: server}, - builtins=("AskUserQuestion",), + # A shell, a file editor and the repository. This stage builds infrastructure for an + # agent it has never seen, so the work is engineering rather than form filling: read the + # code, write what it needs, run it, read the error, fix it. The sandbox is the boundary. + builtins=( + "AskUserQuestion", + "Read", + "Glob", + "Grep", + "Write", + "Edit", + "Bash", + ), cwd=str(destination.parent if destination.parent.exists() else Path.cwd()), max_turns=environment_turns_for( contract, diff --git a/src/fi/alk/harness/config.py b/src/fi/alk/harness/config.py index 8cf6d8ff..c0628752 100644 --- a/src/fi/alk/harness/config.py +++ b/src/fi/alk/harness/config.py @@ -156,10 +156,9 @@ def read_only_session( ) -# Tools the host offers every session that no stage of this harness has any use for. Denying -# them at the gate works and is the backstop, but a denial still costs the turn that discovered -# it — and these get reached for in almost every stage. Naming them as disallowed keeps them out -# of the tool list the model is shown, so the turn is never spent. +# Host tools no stage gets unless it asks. Off by default because a tool nobody wants still +# costs the turn that discovers it, not because any of them is forbidden: a stage that names one +# in its builtins gets it, and the backend removes it from this list for that session. UNWANTED = ( "ToolSearch", "Bash", diff --git a/src/fi/alk/harness/world/workspace.py b/src/fi/alk/harness/world/workspace.py index d790b434..bd0acfde 100644 --- a/src/fi/alk/harness/world/workspace.py +++ b/src/fi/alk/harness/world/workspace.py @@ -25,9 +25,10 @@ ENV = "env" -# Only these. Not a general shell: a tool that can run anything is a tool with no guardrail, and -# the whole point of routing through here is that what happens is inspectable and bounded. -ALLOWED = ("docker", "docker-compose") +# Everything runs. The sandbox is the boundary: a stage that can build an environment for an +# agent nobody anticipated needs to install, compile and test whatever that agent depends on, +# and an allowlist of two container commands can only ever describe the agents we already knew. +ALLOWED: tuple[str, ...] = () # Long enough for an image build that downloads a base layer, short enough that a hung build is # reported rather than waited on forever. @@ -104,11 +105,9 @@ def run( return 1, f"could not parse container command: {exc}" if not words: return 1, "no command given" - if words[0] not in ALLOWED: + if ALLOWED and words[0] not in ALLOWED: return 1, ( - f"{words[0]!r} is not something this can run. Only {' and '.join(ALLOWED)} commands, " - "because a general shell here would be a guardrail with nothing behind it. Everything " - "the environment needs should be in a file it builds from, not in a command." + f"{words[0]!r} is not something this can run. Only {' and '.join(ALLOWED)} commands." ) blocked = available() if blocked: From cc4ec0a2de390ce6f15398c0a7c9416042810726 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 29 Aug 2026 18:12:15 +0530 Subject: [PATCH 02/45] fix(harness): stop probe scoring unexecuted runtime tools as passing --- src/fi/alk/harness/world/probe.py | 67 +++++++++++++++++++++++++------ 1 file changed, 55 insertions(+), 12 deletions(-) diff --git a/src/fi/alk/harness/world/probe.py b/src/fi/alk/harness/world/probe.py index 8eb588a3..be94a5cc 100644 --- a/src/fi/alk/harness/world/probe.py +++ b/src/fi/alk/harness/world/probe.py @@ -50,6 +50,9 @@ class ProbeResult: @dataclass class ProbeReport: results: list[ProbeResult] = field(default_factory=list) + # Tools that could not be executed from here at all. Kept apart from results so they never + # count towards the score in either direction: they are not failures, and they are not proof. + unproven: list[str] = field(default_factory=list) @property def score(self) -> float: @@ -64,13 +67,22 @@ def failures(self) -> list[ProbeResult]: return [result for result in self.results if not result.passed] def summary(self) -> str: - if not self.results: + if not self.results and not self.unproven: return "no probes ran" - lines = [ - f"{len(self.results) - len(self.failures)}/{len(self.results)} probes passed" - ] + lines = ( + [ + f"{len(self.results) - len(self.failures)}/{len(self.results)} probes passed" + ] + if self.results + else [] + ) for failure in self.failures: lines.append(f" {failure.kind}:{failure.name}: {failure.detail}") + if self.unproven: + lines.append( + f" {len(self.unproven)} tools not executed from here, so still unproven: " + + ", ".join(sorted(self.unproven)) + ) return "\n".join(lines) @@ -248,14 +260,11 @@ def probe( for tool in contract.tools: if tool.name in runtime_tools: - report.results.append( - ProbeResult( - tool.name, - COVERAGE, - True, - "executes inside the submitted agent runtime", - ) - ) + # Not executed here, so not claimed as passing. The runtime this tool lives in is + # built after this stage in the hosted lane, so the only honest report is that it + # remains unproven; verify_runtime_tools settles it once the runtime is up. Recording + # it as a pass is how a world with nothing exercised used to score perfectly. + report.unproven.append(tool.name) continue if tool.name not in world.handlers: continue @@ -411,3 +420,37 @@ def _run_sequence( if failures: return ProbeResult(name, SEQUENCE, False, failures[0]) return ProbeResult(name, SEQUENCE, True) + + +def verify_runtime_tools(world: Any, contract: Any) -> list[str]: + """Execute the agent's own tools against the world that was just built. + + The build stage cannot reach these: in the hosted lane the runtime they live in is started + after authoring finishes, so `probe` can only record them as unproven. This is where that + debt is settled, once there is something to call. + + A tool that refuses is working, so only a crash or a server error counts against it. Returns + one line per broken tool, empty when every declared tool answered. Nothing is written here; + the caller decides whether a broken tool stops the run. + """ + forward = getattr(world, "forward", None) + endpoints = getattr(world, "endpoint_for", {}) or {} + runtime_tools = set(getattr(world, "runtime_tools", set())) + if not callable(forward) or not runtime_tools: + return [] + broken: list[str] = [] + for tool in getattr(contract, "tools", []): + if tool.name not in runtime_tools: + continue + endpoint = endpoints.get(tool.name) + if not endpoint: + broken.append(f"{tool.name}: nothing bound to call, so it cannot be proven") + continue + try: + call = forward(endpoint, _valid_arguments(tool), record=False) + except Exception as exc: # noqa: BLE001 - a raising tool is exactly what we are looking for + broken.append(f"{tool.name}: raised {type(exc).__name__}: {exc}") + continue + if not call.ok and not call.refused: + broken.append(f"{tool.name}: {call.error or 'failed with no reason given'}") + return broken From df484c3312eaf2b03f198483307ef1f4d5de200a Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 29 Aug 2026 18:13:55 +0530 Subject: [PATCH 03/45] feat(harness): let the build stage choose how to work from sub-skills it reads --- src/fi/alk/harness/config.py | 46 +++++++++++++++++-- .../browser-and-computer-use.md | 15 ++++++ .../retrieval-and-assistants.md | 12 +++++ .../voice-hosted-platform.md | 23 ++++++++++ .../skills/build-environment/voice-livekit.md | 24 ++++++++++ 5 files changed, 115 insertions(+), 5 deletions(-) create mode 100644 src/fi/alk/harness/skills/build-environment/browser-and-computer-use.md create mode 100644 src/fi/alk/harness/skills/build-environment/retrieval-and-assistants.md create mode 100644 src/fi/alk/harness/skills/build-environment/voice-hosted-platform.md create mode 100644 src/fi/alk/harness/skills/build-environment/voice-livekit.md diff --git a/src/fi/alk/harness/config.py b/src/fi/alk/harness/config.py index c0628752..4f78166a 100644 --- a/src/fi/alk/harness/config.py +++ b/src/fi/alk/harness/config.py @@ -54,11 +54,7 @@ def chosen_model(model: str | None = None) -> str: With nothing named anywhere, the selected backend's own default runs, so switching ``ALK_HARNESS`` never sends one vendor's model name to another vendor's loop. """ - return ( - model - or os.environ.get("ALK_HARNESS_MODEL") - or resolve().default_model - ) + return model or os.environ.get("ALK_HARNESS_MODEL") or resolve().default_model def thinking_config() -> dict[str, Any]: @@ -267,6 +263,9 @@ def load_skill(name: str) -> str: if not path.exists(): raise FileNotFoundError(f"no skill at {path}") stage = path.read_text(encoding="utf-8") + catalogue = sub_skills(name) + if catalogue: + stage = f"{stage}\n\n{catalogue}" if not HARNESS.exists(): return stage return ( @@ -275,3 +274,40 @@ def load_skill(name: str) -> str: "# The stage you are in now\n\n" f"{stage}" ) + + +def sub_skills(name: str) -> str: + """The per-kind guidance available inside a stage, for the model to choose between. + + A stage skill says how the stage works for any agent. What differs between a LiveKit voice + agent, a Vapi assistant reached only by webhook, and a browser agent is method, not stage, so + that belongs in a sub-skill the model selects after it has read the contract rather than in a + constant chosen before anyone has looked at the repository. + + Every ``*.md`` beside the stage's ``SKILL.md`` is offered. Adding support for a new kind is a + file, not a release. + """ + directory = SKILLS_ROOT / name + found = sorted(p for p in directory.glob("*.md") if p.name != "SKILL.md") + if not found: + return "" + lines = [ + "## Ways of working available to you", + "", + "Read the contract first, then read whichever of these fits the agent in front of you.", + "Nothing selects one for you, and using none of them is a legitimate answer for an agent", + "none of them describes: they are accumulated experience, not a menu you must pick from.", + "", + ] + for entry in found: + first = "" + for line in entry.read_text(encoding="utf-8").splitlines(): + if line.strip() and not line.startswith("#"): + first = line.strip() + break + lines.append(f"- `{entry.name}`: {first or 'no summary line'}") + lines.append("") + lines.append( + f"They are in `{directory}`. Read one with the Read tool before you rely on it." + ) + return "\n".join(lines) diff --git a/src/fi/alk/harness/skills/build-environment/browser-and-computer-use.md b/src/fi/alk/harness/skills/build-environment/browser-and-computer-use.md new file mode 100644 index 00000000..28722b97 --- /dev/null +++ b/src/fi/alk/harness/skills/build-environment/browser-and-computer-use.md @@ -0,0 +1,15 @@ +# Agents that drive a browser or a desktop + +The world is a site or an application, not a database the agent queries. + +`fi/alk/harness/world/kinds.py` already maps `browser`, `computer_use` and `cua` to a browser +world, so start there rather than inventing a kind. + +What makes these different: +- State lives in the page, so a check reads the DOM or the application, not a table. +- Reset means the site returns to a known state, which usually means seeding whatever backs it. +- A journey crosses screens. A check that only looks at the final screen will pass a run that took + a wrong route to the right place. + +Build the site or application the agent acts on, seed it, and make sure it can be returned to that +seed between scenarios. diff --git a/src/fi/alk/harness/skills/build-environment/retrieval-and-assistants.md b/src/fi/alk/harness/skills/build-environment/retrieval-and-assistants.md new file mode 100644 index 00000000..becbd245 --- /dev/null +++ b/src/fi/alk/harness/skills/build-environment/retrieval-and-assistants.md @@ -0,0 +1,12 @@ +# RAG systems and assistants reached over an API + +The world is a corpus and whatever serves it. + +What you build: the documents, the index, and the retrieval endpoint the agent calls. Seed the +corpus so that questions have knowably right and knowably absent answers, because an assistant +that answers confidently from an empty index is the failure worth catching. + +Checks read what was retrieved and what was answered, so record both. A check that only reads the +answer cannot tell a correct answer from a lucky one. + +If the repository ships the retrieval service, run it. Only build your own when there is none. diff --git a/src/fi/alk/harness/skills/build-environment/voice-hosted-platform.md b/src/fi/alk/harness/skills/build-environment/voice-hosted-platform.md new file mode 100644 index 00000000..f39ef2da --- /dev/null +++ b/src/fi/alk/harness/skills/build-environment/voice-hosted-platform.md @@ -0,0 +1,23 @@ +# Voice agents hosted by a platform (Vapi, Retell, and anything like them) + +The direction is inverted here. There is no worker to start: their platform runs the agent and +calls you. What you build is something reachable that answers the way their tools expect. + +You will usually be given credentials, an assistant or agent id, and a repository holding the tool +implementations their assistant already calls by webhook. + +What that means for you: + +- Build the tool service from the repository and put a real store under it, the same as any other + agent. That part does not change. +- The service has to be reachable from their platform, not just from inside this sandbox. Work out + what the ingress is before you build anything on top of it. +- Their assistant configuration names the webhook. Point it at what you built, or state plainly + that you cannot and why. +- Their platform holds the conversation, so you do not own turn taking, barge-in or audio. + +If you cannot reach their platform from here, say so and stop. A world that looks right but +receives no calls is worse than an honest failure, because the run will look like an agent defect. + +ALK's own voice stack is for agents we run ourselves. Most of it does not apply. `world/` and the +stores do. diff --git a/src/fi/alk/harness/skills/build-environment/voice-livekit.md b/src/fi/alk/harness/skills/build-environment/voice-livekit.md new file mode 100644 index 00000000..2d2544e4 --- /dev/null +++ b/src/fi/alk/harness/skills/build-environment/voice-livekit.md @@ -0,0 +1,24 @@ +# Voice agents that bring their own LiveKit worker + +The agent joins a room and talks. You are building what its tools sit on, not the call itself. + +ALK already runs this end to end. Reuse it rather than writing your own: + +- `fi/alk/harness/call_runner.py` places the call in the hosted lane. +- `fi/alk/harness/simulator_voice.py` builds the caller: persona, scenario, providers, phone + identity. Both lanes share it, so a change here reaches local and hosted together. +- `fi/simulate/simulation/engines/livekit.py` is the engine: room, turn taking, the silence + backstops, `endCall`, and the transcript that comes out. +- `fi/simulate/simulation/livekit_models.py` resolves STT, TTS and LLM per persona. + +What is yours to build: the data the agent's tools read and write, and nothing else. If the +repository ships its tool service, run that service unchanged and put a real store under it. Do +not reimplement a tool the agent already has, because what you write will not be what production +runs. + +Worth knowing before you start: +- The agent's own worker is started for you. You are not responsible for it. +- Tools the agent calls at runtime cannot be executed from this stage. Build them so they work, + and expect `probe` to list them as unproven rather than passing. +- Numbers spoken aloud reach the tools as digits. A phone column that only accepts one format + will fail on a caller who says it differently. From 34ecc82db815323ee50e682886b94b91c6f91d9c Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 29 Aug 2026 18:14:29 +0530 Subject: [PATCH 04/45] feat(harness): register postgres as a world kind and drop the unused provision skill --- .../simulator-prompt.txt | 0 .../simulator-setup.json | 30 ++++ .../simulator-prompt.txt | 0 .../simulator-setup.json | 30 ++++ .../simulator-prompt.txt | 0 .../simulator-setup.json | 30 ++++ .../simulator-prompt.txt | 0 .../simulator-setup.json | 30 ++++ .../simulator-prompt.txt | 0 .../simulator-setup.json | 30 ++++ .../simulator-prompt.txt | 0 .../simulator-setup.json | 30 ++++ .../simulator-prompt.txt | 0 .../simulator-setup.json | 30 ++++ .../simulator-prompt.txt | 0 .../simulator-setup.json | 30 ++++ .../skills/provision-environment/SKILL.md | 136 ------------------ src/fi/alk/harness/world/kinds.py | 10 +- 18 files changed, 249 insertions(+), 137 deletions(-) create mode 100644 recordings/run_dispatch_fail/case_8c41bcb6daf45863b22807f4b41d0418/simulator-prompt.txt create mode 100644 recordings/run_dispatch_fail/case_8c41bcb6daf45863b22807f4b41d0418/simulator-setup.json create mode 100644 recordings/run_dispatch_order/case_f7fb31ace7405a5b96cc5f5dad4e9d81/simulator-prompt.txt create mode 100644 recordings/run_dispatch_order/case_f7fb31ace7405a5b96cc5f5dad4e9d81/simulator-setup.json create mode 100644 recordings/run_managed/case_3428ab8c34dd5d6a986a95073b69efb3/simulator-prompt.txt create mode 100644 recordings/run_managed/case_3428ab8c34dd5d6a986a95073b69efb3/simulator-setup.json create mode 100644 recordings/run_sip/case_845b00e731b35c3c8a9a909f789ffd02/simulator-prompt.txt create mode 100644 recordings/run_sip/case_845b00e731b35c3c8a9a909f789ffd02/simulator-setup.json create mode 100644 recordings/run_sip/case_c2ffea9fcff857e99bee59d872a173d1/simulator-prompt.txt create mode 100644 recordings/run_sip/case_c2ffea9fcff857e99bee59d872a173d1/simulator-setup.json create mode 100644 recordings/run_sip_fail/case_51335392121b575aaff054a25d4c4ca4/simulator-prompt.txt create mode 100644 recordings/run_sip_fail/case_51335392121b575aaff054a25d4c4ca4/simulator-setup.json create mode 100644 recordings/run_sip_in/case_c62061d379ed5b2dad2a38ad6182b67e/simulator-prompt.txt create mode 100644 recordings/run_sip_in/case_c62061d379ed5b2dad2a38ad6182b67e/simulator-setup.json create mode 100644 recordings/run_web_bridge/case_147ff72e9dc45c8191be7976820befab/simulator-prompt.txt create mode 100644 recordings/run_web_bridge/case_147ff72e9dc45c8191be7976820befab/simulator-setup.json delete mode 100644 src/fi/alk/harness/skills/provision-environment/SKILL.md diff --git a/recordings/run_dispatch_fail/case_8c41bcb6daf45863b22807f4b41d0418/simulator-prompt.txt b/recordings/run_dispatch_fail/case_8c41bcb6daf45863b22807f4b41d0418/simulator-prompt.txt new file mode 100644 index 00000000..e69de29b diff --git a/recordings/run_dispatch_fail/case_8c41bcb6daf45863b22807f4b41d0418/simulator-setup.json b/recordings/run_dispatch_fail/case_8c41bcb6daf45863b22807f4b41d0418/simulator-setup.json new file mode 100644 index 00000000..efd969a4 --- /dev/null +++ b/recordings/run_dispatch_fail/case_8c41bcb6daf45863b22807f4b41d0418/simulator-setup.json @@ -0,0 +1,30 @@ +{ + "persona": { + "persona": { + "name": "Caller 0" + }, + "situation": "I need help.", + "outcome": "The issue is resolved.", + "identity": null, + "temperament": null, + "behavior_policy": null, + "knowledge": [], + "attack": null, + "provenance": null, + "version": null + }, + "simulator_system_prompt": "", + "llm": null, + "stt": null, + "tts": null, + "turn_handling": null, + "room_name": "support-room-9dc3b37e91db-07f4b41d0418", + "agent_name": "support-agent", + "test_case_id": "case_8c41bcb6daf45863b22807f4b41d0418", + "run_id": "run_dispatch_fail", + "conversation_direction": "simulator_first", + "allow_interruptions": null, + "min_endpointing_delay": null, + "max_endpointing_delay": null, + "use_tts_aligned_transcript": null +} \ No newline at end of file diff --git a/recordings/run_dispatch_order/case_f7fb31ace7405a5b96cc5f5dad4e9d81/simulator-prompt.txt b/recordings/run_dispatch_order/case_f7fb31ace7405a5b96cc5f5dad4e9d81/simulator-prompt.txt new file mode 100644 index 00000000..e69de29b diff --git a/recordings/run_dispatch_order/case_f7fb31ace7405a5b96cc5f5dad4e9d81/simulator-setup.json b/recordings/run_dispatch_order/case_f7fb31ace7405a5b96cc5f5dad4e9d81/simulator-setup.json new file mode 100644 index 00000000..a384eb23 --- /dev/null +++ b/recordings/run_dispatch_order/case_f7fb31ace7405a5b96cc5f5dad4e9d81/simulator-setup.json @@ -0,0 +1,30 @@ +{ + "persona": { + "persona": { + "name": "Caller 0" + }, + "situation": "I need help.", + "outcome": "The issue is resolved.", + "identity": null, + "temperament": null, + "behavior_policy": null, + "knowledge": [], + "attack": null, + "provenance": null, + "version": null + }, + "simulator_system_prompt": "", + "llm": null, + "stt": null, + "tts": null, + "turn_handling": null, + "room_name": "support-room-3987a2c311af-5f5dad4e9d81", + "agent_name": "support-agent", + "test_case_id": "case_f7fb31ace7405a5b96cc5f5dad4e9d81", + "run_id": "run_dispatch_order", + "conversation_direction": "simulator_first", + "allow_interruptions": null, + "min_endpointing_delay": null, + "max_endpointing_delay": null, + "use_tts_aligned_transcript": null +} \ No newline at end of file diff --git a/recordings/run_managed/case_3428ab8c34dd5d6a986a95073b69efb3/simulator-prompt.txt b/recordings/run_managed/case_3428ab8c34dd5d6a986a95073b69efb3/simulator-prompt.txt new file mode 100644 index 00000000..e69de29b diff --git a/recordings/run_managed/case_3428ab8c34dd5d6a986a95073b69efb3/simulator-setup.json b/recordings/run_managed/case_3428ab8c34dd5d6a986a95073b69efb3/simulator-setup.json new file mode 100644 index 00000000..06f0cbc6 --- /dev/null +++ b/recordings/run_managed/case_3428ab8c34dd5d6a986a95073b69efb3/simulator-setup.json @@ -0,0 +1,30 @@ +{ + "persona": { + "persona": { + "name": "Caller 0" + }, + "situation": "I need help.", + "outcome": "The issue is resolved.", + "identity": null, + "temperament": null, + "behavior_policy": null, + "knowledge": [], + "attack": null, + "provenance": null, + "version": null + }, + "simulator_system_prompt": "", + "llm": null, + "stt": null, + "tts": null, + "turn_handling": null, + "room_name": "support-room-0967a5278739-95073b69efb3", + "agent_name": "support-agent", + "test_case_id": "case_3428ab8c34dd5d6a986a95073b69efb3", + "run_id": "run_managed", + "conversation_direction": "simulator_first", + "allow_interruptions": null, + "min_endpointing_delay": null, + "max_endpointing_delay": null, + "use_tts_aligned_transcript": null +} \ No newline at end of file diff --git a/recordings/run_sip/case_845b00e731b35c3c8a9a909f789ffd02/simulator-prompt.txt b/recordings/run_sip/case_845b00e731b35c3c8a9a909f789ffd02/simulator-prompt.txt new file mode 100644 index 00000000..e69de29b diff --git a/recordings/run_sip/case_845b00e731b35c3c8a9a909f789ffd02/simulator-setup.json b/recordings/run_sip/case_845b00e731b35c3c8a9a909f789ffd02/simulator-setup.json new file mode 100644 index 00000000..571dfd0d --- /dev/null +++ b/recordings/run_sip/case_845b00e731b35c3c8a9a909f789ffd02/simulator-setup.json @@ -0,0 +1,30 @@ +{ + "persona": { + "persona": { + "name": "Caller 0" + }, + "situation": "I need help.", + "outcome": "The issue is resolved.", + "identity": null, + "temperament": null, + "behavior_policy": null, + "knowledge": [], + "attack": null, + "provenance": null, + "version": null + }, + "simulator_system_prompt": "", + "llm": null, + "stt": null, + "tts": null, + "turn_handling": null, + "room_name": "sdk-suite-case_845b00e731b35c3c8a9a909f789ffd02-32ca064c8f46", + "agent_name": "support-agent", + "test_case_id": "case_845b00e731b35c3c8a9a909f789ffd02", + "run_id": "run_sip", + "conversation_direction": "simulator_first", + "allow_interruptions": null, + "min_endpointing_delay": null, + "max_endpointing_delay": null, + "use_tts_aligned_transcript": null +} \ No newline at end of file diff --git a/recordings/run_sip/case_c2ffea9fcff857e99bee59d872a173d1/simulator-prompt.txt b/recordings/run_sip/case_c2ffea9fcff857e99bee59d872a173d1/simulator-prompt.txt new file mode 100644 index 00000000..e69de29b diff --git a/recordings/run_sip/case_c2ffea9fcff857e99bee59d872a173d1/simulator-setup.json b/recordings/run_sip/case_c2ffea9fcff857e99bee59d872a173d1/simulator-setup.json new file mode 100644 index 00000000..db6fe93b --- /dev/null +++ b/recordings/run_sip/case_c2ffea9fcff857e99bee59d872a173d1/simulator-setup.json @@ -0,0 +1,30 @@ +{ + "persona": { + "persona": { + "name": "Caller 1" + }, + "situation": "I need help.", + "outcome": "The issue is resolved.", + "identity": null, + "temperament": null, + "behavior_policy": null, + "knowledge": [], + "attack": null, + "provenance": null, + "version": null + }, + "simulator_system_prompt": "", + "llm": null, + "stt": null, + "tts": null, + "turn_handling": null, + "room_name": "sdk-suite-case_c2ffea9fcff857e99bee59d872a173d1-32ca064c8f46", + "agent_name": "support-agent", + "test_case_id": "case_c2ffea9fcff857e99bee59d872a173d1", + "run_id": "run_sip", + "conversation_direction": "simulator_first", + "allow_interruptions": null, + "min_endpointing_delay": null, + "max_endpointing_delay": null, + "use_tts_aligned_transcript": null +} \ No newline at end of file diff --git a/recordings/run_sip_fail/case_51335392121b575aaff054a25d4c4ca4/simulator-prompt.txt b/recordings/run_sip_fail/case_51335392121b575aaff054a25d4c4ca4/simulator-prompt.txt new file mode 100644 index 00000000..e69de29b diff --git a/recordings/run_sip_fail/case_51335392121b575aaff054a25d4c4ca4/simulator-setup.json b/recordings/run_sip_fail/case_51335392121b575aaff054a25d4c4ca4/simulator-setup.json new file mode 100644 index 00000000..d75009a5 --- /dev/null +++ b/recordings/run_sip_fail/case_51335392121b575aaff054a25d4c4ca4/simulator-setup.json @@ -0,0 +1,30 @@ +{ + "persona": { + "persona": { + "name": "Caller 0" + }, + "situation": "I need help.", + "outcome": "The issue is resolved.", + "identity": null, + "temperament": null, + "behavior_policy": null, + "knowledge": [], + "attack": null, + "provenance": null, + "version": null + }, + "simulator_system_prompt": "", + "llm": null, + "stt": null, + "tts": null, + "turn_handling": null, + "room_name": "support-room-cb1bac4e5bd6-54a25d4c4ca4", + "agent_name": "support-agent", + "test_case_id": "case_51335392121b575aaff054a25d4c4ca4", + "run_id": "run_sip_fail", + "conversation_direction": "simulator_first", + "allow_interruptions": null, + "min_endpointing_delay": null, + "max_endpointing_delay": null, + "use_tts_aligned_transcript": null +} \ No newline at end of file diff --git a/recordings/run_sip_in/case_c62061d379ed5b2dad2a38ad6182b67e/simulator-prompt.txt b/recordings/run_sip_in/case_c62061d379ed5b2dad2a38ad6182b67e/simulator-prompt.txt new file mode 100644 index 00000000..e69de29b diff --git a/recordings/run_sip_in/case_c62061d379ed5b2dad2a38ad6182b67e/simulator-setup.json b/recordings/run_sip_in/case_c62061d379ed5b2dad2a38ad6182b67e/simulator-setup.json new file mode 100644 index 00000000..cef6fbac --- /dev/null +++ b/recordings/run_sip_in/case_c62061d379ed5b2dad2a38ad6182b67e/simulator-setup.json @@ -0,0 +1,30 @@ +{ + "persona": { + "persona": { + "name": "Caller 0" + }, + "situation": "I need help.", + "outcome": "The issue is resolved.", + "identity": null, + "temperament": null, + "behavior_policy": null, + "knowledge": [], + "attack": null, + "provenance": null, + "version": null + }, + "simulator_system_prompt": "", + "llm": null, + "stt": null, + "tts": null, + "turn_handling": null, + "room_name": "support-room-02cb8240e009-38ad6182b67e", + "agent_name": "support-agent", + "test_case_id": "case_c62061d379ed5b2dad2a38ad6182b67e", + "run_id": "run_sip_in", + "conversation_direction": "simulator_first", + "allow_interruptions": null, + "min_endpointing_delay": null, + "max_endpointing_delay": null, + "use_tts_aligned_transcript": null +} \ No newline at end of file diff --git a/recordings/run_web_bridge/case_147ff72e9dc45c8191be7976820befab/simulator-prompt.txt b/recordings/run_web_bridge/case_147ff72e9dc45c8191be7976820befab/simulator-prompt.txt new file mode 100644 index 00000000..e69de29b diff --git a/recordings/run_web_bridge/case_147ff72e9dc45c8191be7976820befab/simulator-setup.json b/recordings/run_web_bridge/case_147ff72e9dc45c8191be7976820befab/simulator-setup.json new file mode 100644 index 00000000..ed741e05 --- /dev/null +++ b/recordings/run_web_bridge/case_147ff72e9dc45c8191be7976820befab/simulator-setup.json @@ -0,0 +1,30 @@ +{ + "persona": { + "persona": { + "name": "Caller 0" + }, + "situation": "I need help.", + "outcome": "The issue is resolved.", + "identity": null, + "temperament": null, + "behavior_policy": null, + "knowledge": [], + "attack": null, + "provenance": null, + "version": null + }, + "simulator_system_prompt": "", + "llm": null, + "stt": null, + "tts": null, + "turn_handling": null, + "room_name": "sdk-web-case_147ff72e9dc45c8191be7976820befab-148f5af142ba", + "agent_name": "support-agent", + "test_case_id": "case_147ff72e9dc45c8191be7976820befab", + "run_id": "run_web_bridge", + "conversation_direction": "simulator_first", + "allow_interruptions": null, + "min_endpointing_delay": null, + "max_endpointing_delay": null, + "use_tts_aligned_transcript": null +} \ No newline at end of file diff --git a/src/fi/alk/harness/skills/provision-environment/SKILL.md b/src/fi/alk/harness/skills/provision-environment/SKILL.md deleted file mode 100644 index 234cf062..00000000 --- a/src/fi/alk/harness/skills/provision-environment/SKILL.md +++ /dev/null @@ -1,136 +0,0 @@ ---- -name: provision-environment -description: Stand up the real thing an agent connects to, and prove it, without touching the agent. ---- - -# Provision the environment - -You are standing up the world an AI agent will be tested in. Its contract is in front of you: -the tools it really has, the rules it obeys, what it depends on, and its data. - -**You are not rebuilding this agent. You are building what it connects to.** Its code runs -unmodified, its own client issues its own queries, and the only thing that differs from -production is which host answers them. That is the whole method, and everything below follows -from it. - -## The one rule - -**Never change the agent.** Not its source, not its config file, not a copy of it. You have no -tool that can, and that is deliberate: when a check fails there are two ways to make it green — -fix the environment, or edit the agent until it stops failing — and the second produces a green -suite about code nobody ships. - -If the agent cannot be pointed at your store, that is a **finding to report**, not a thing to -work around. Say so plainly and stop. - -## Being where the agent already looks - -The agent expects a database at some host, on some port, with some name, reached by some -variable. You do not change any of that. You build your store **to match it**. - -- It reads `DATABASE_URL` — you set `DATABASE_URL` when it launches. -- It reads `database.url` from a config file — you mount that file. -- It hardcodes `db.internal:5432` — you make `db.internal` resolve to your container. A - hardcoded host is not an obstacle; it is just a name you have to answer to. -- It hardcodes a database name and user — you create your store with exactly those. - -The contract records what it expects. Match it. - -## Talking - -You are talking to a person. Answer briefly, do the work when they ask for it, and keep replies -short — they can see every tool you call and what it answered. - -Ask them when a decision is genuinely theirs: what data should be in the store where the -contract carries none, whether an engine you cannot identify is worth guessing at. - -## How to work - -1. **`declare_engine`** with the engine the contract names. `inspect_environment` first if you - want to see what the harness can already stand up. - - **Never substitute a different engine.** Not a similar one, not a "lightweight equivalent", - not a server standing in for something held in memory. An agent whose tools read a dict is - not tested by putting that dict in Redis — its queries never run, and every result is about - code it does not have. This is the single mistake this whole path exists to prevent, and it - does not stop being that mistake because the substitute is convenient. - - Some agents have **no server at all**: they load files into memory and their tools read that - structure directly. That is `engine: inprocess`, it is already supported, and the contract - names the loader to call. Nothing is stood up and nothing is connected to. - - If the harness genuinely has never seen the engine — it is not in `inspect_environment`'s - list — then `write_store_ops`. That is expected, not a failure; an engine nobody wrote down - in advance is the normal case. - -2. **`run_migrations` with the agent's own migrations.** Find them: an `alembic/` directory, a - `migrations/` folder, `schema.sql`, the models it defines. Run those. - - **Never write a schema yourself.** One you invented is a guess, and every check written - against it inherits the guess. If you genuinely cannot find migrations, say so and ask — - do not fill the gap with tables you made up. - -3. **`seed`** from the contract's real data, including anything that looks like a mistake: a - misspelled id, an item marked unavailable, an odd price. The store is a replica of what the - agent has, not a corrected version, and a test written against a corrected one will not - catch the real bug. - - Leave it in its natural starting state: empty carts, no in-flight work. Scenarios add what - they need. - -4. **`add_sub_goal`** for each thing worth checking, with its check as code. - - A check is given the store and the calls that were recorded, and returns a sentence when - something is wrong or `None` when it held. Write it against what the run leaves behind: - - ```python - def check(world, calls): - rows = world.state()["orders"] - if len(rows) != 1: - return f"{len(rows)} orders, expected 1" - return None - ``` - - Use `judged` **only** where nothing observable settles it — whether a refusal was explained, - whether tone was right. If most of your sub-goals are judged, you have not looked hard enough - at what the store records. - -5. **`write_simulator_prompt`**, if this agent is conversational. - -6. **`prove_environment`**, and fix what it names. Repeat until it holds. - -7. **`save_environment`.** - -## What proving actually does - -You hand it a `mutation` — any statement this engine accepts that changes something. One insert -is plenty. Then, without knowing your engine: - -- your mutation has to **move** something, or a broken reset would look perfect -- `restore` has to reproduce the rows **exactly** -- **ids must not drift**: the same change is run twice from the same starting point and the two - results compared, so a reset that puts rows back but leaves a counter where it was is caught - without anyone naming what a counter is called on this engine -- every check you wrote has to **fail against an emptied store**. One that still holds when - there is nothing there is not measuring the environment - -A failure here is **yours or ours, never the agent's**. Nothing in it involves the agent. - -## Reading a failure - -The report names what broke, in your terms. `ids do not drift: the same change from the same -starting point produced something different the second time` means your `restore` puts rows -back but not the counter behind them. Fix the reset and prove again. - -If the same failure survives three attempts, stop and read it literally. Whatever you are -changing is not what is failing. - -You do not get to declare the environment sound. `save_environment` runs the gate again itself. - -## Finishing - -Say what you stood up: the engine and version, where its schema came from, roughly what is in -it, how the agent will be pointed at it, and the sub-goals with how many are settled by code. - -Then say plainly anything you were unsure about — especially where you could not find the -agent's migrations, or where its configuration seam was not obvious. diff --git a/src/fi/alk/harness/world/kinds.py b/src/fi/alk/harness/world/kinds.py index 8d62a201..f584d3fa 100644 --- a/src/fi/alk/harness/world/kinds.py +++ b/src/fi/alk/harness/world/kinds.py @@ -79,7 +79,12 @@ def describe(self, world: GeneratedWorld) -> str: class SqliteWorld: - """A world whose state is rows in tables. Tool APIs, and anything with a data store.""" + """A world whose state is rows in tables. Tool APIs, and anything with a data store. + + The name says SQLite for history; what this actually describes is the shape of the state, not + the engine holding it. Everything here reads ``world.state()``, so Postgres, and any other + row store a world is built on, are the same kind of world to look at. + """ key = "sqlite" label = "a database behind the agent's tools" @@ -152,6 +157,9 @@ def describe(self, world: GeneratedWorld) -> str: _REGISTRY: dict[str, Callable[[], WorldKind]] = { SqliteWorld.key: SqliteWorld, + # A contract naming its store as postgres asked for a kind that did not exist and got the + # sqlite fallback silently. Rows are rows: the same inspection serves both. + "postgres": SqliteWorld, BrowserWorld.key: BrowserWorld, InProcessWorld.key: InProcessWorld, } From e1992ac9821b87945e441b7ffb111974d9fe684f Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Sat, 29 Aug 2026 18:26:15 +0530 Subject: [PATCH 05/45] refactor(harness): remove the deny-by-default gate and the tools the shell replaces --- src/fi/alk/harness/__init__.py | 4 +- src/fi/alk/harness/backends/claude.py | 23 +---- src/fi/alk/harness/build.py | 2 +- src/fi/alk/harness/cli.py | 12 +-- src/fi/alk/harness/config.py | 99 ++++-------------- src/fi/alk/harness/understand.py | 4 +- src/fi/alk/harness/world/tools.py | 47 --------- src/fi/alk/harness/world/workspace.py | 143 -------------------------- 8 files changed, 37 insertions(+), 297 deletions(-) delete mode 100644 src/fi/alk/harness/world/workspace.py diff --git a/src/fi/alk/harness/__init__.py b/src/fi/alk/harness/__init__.py index 84cee4db..a6b38a2a 100644 --- a/src/fi/alk/harness/__init__.py +++ b/src/fi/alk/harness/__init__.py @@ -21,7 +21,7 @@ artifact_dir, load_skill, provider_env, - read_only_session, + working_session, ) from .contract import AgentContract, Runtime, RuntimeInterface, ToolSpec, validate_contract from .job import ExecutionMode, HarnessJob, HarnessStage @@ -64,7 +64,7 @@ "open_conversation", "open_stage", "provider_env", - "read_only_session", + "working_session", "register_source", "resolve", "seal_bundle", diff --git a/src/fi/alk/harness/backends/claude.py b/src/fi/alk/harness/backends/claude.py index 609b6325..c146f346 100644 --- a/src/fi/alk/harness/backends/claude.py +++ b/src/fi/alk/harness/backends/claude.py @@ -148,13 +148,7 @@ def can_drive(self, model: str) -> bool: return "claude" in (model or "").lower() def create(self, spec: SessionSpec) -> ClaudeSession: - from ..config import ( - UNWANTED, - gate_hooks, - permission_gate, - provider_env, - thinking_config, - ) + from ..config import operator_ask, provider_env, thinking_config allowed = [ *spec.builtins, @@ -179,19 +173,10 @@ def create(self, spec: SessionSpec) -> ClaudeSession: if spec.cwd is not None: options.cwd = spec.cwd if spec.gated: - # Not acceptEdits: that auto-approves Edit and Write before the permission callback - # is consulted, so a stage could rewrite an artifact by hand and skip the tool whose - # whole job is to validate that change. + # Kept only to route the model's questions to a human when one is attached. There is + # no denial here any more: a stage runs with the tools it was given, in a sandbox. options.permission_mode = "default" - # Deny only what this stage was not granted. A stage that asks for Bash or Write - # means it, and a blanket denial here would silently outrank its own tool list. - options.disallowed_tools = [ - name for name in UNWANTED if name not in set(allowed) - ] - options.hooks = gate_hooks(allowed) - options.can_use_tool = spec.permission_override or permission_gate( - spec.ask, allowed - ) + options.can_use_tool = spec.permission_override or operator_ask(spec.ask) if spec.thinking: options.thinking = thinking_config() return ClaudeSession(options) diff --git a/src/fi/alk/harness/build.py b/src/fi/alk/harness/build.py index 51e53f0c..9430d379 100644 --- a/src/fi/alk/harness/build.py +++ b/src/fi/alk/harness/build.py @@ -195,7 +195,7 @@ def open_stage( "environment endpoints: " + (", ".join(runtime_tools) or "none") + "\nDo not adopt either group, inspect their source again, or recreate any service " - "or behavior. Do not use run_env_command for source discovery. The contract already " + "or behavior. Do not go source hunting with the shell. The contract already " "contains that evidence. Inspect the live data once. Preserve useful repository seed " "rows. If the submitted schema is empty or lacks the records needed to exercise the " "contract's branches, add a small varied realistic baseline through seed only; never " diff --git a/src/fi/alk/harness/cli.py b/src/fi/alk/harness/cli.py index 51d3c291..07b44589 100644 --- a/src/fi/alk/harness/cli.py +++ b/src/fi/alk/harness/cli.py @@ -25,7 +25,7 @@ artifact_dir, chosen_model, credentials_hint, - permission_gate, + operator_ask, ) from .run.targets import supported as target_kinds from .scenarios import load as load_written @@ -146,7 +146,7 @@ async def _understand(args: argparse.Namespace) -> int: out=Path(args.out) if args.out else None, # Unattended, there is nobody to answer, so the model records what it could not # resolve in open_questions rather than blocking on a prompt nobody will see. - ask=permission_gate(_ask_operator) if args.interactive else None, + ask=operator_ask(_ask_operator) if args.interactive else None, ) print(f"agent: {source.name} ({source.kind})") @@ -248,7 +248,7 @@ async def _build(args: argparse.Namespace) -> int: stage, _ = build_stage( contract, out=destination, - ask=permission_gate(_ask_operator) if args.interactive else None, + ask=operator_ask(_ask_operator) if args.interactive else None, source_root=source_root, deferred_runtime=bool(getattr(args, "skip_source_provision", False)), ) @@ -385,7 +385,7 @@ async def _scenarios(args: argparse.Namespace) -> int: contract, out=destination, wanted=wanted, - ask=permission_gate(_ask_operator) if args.interactive else None, + ask=operator_ask(_ask_operator) if args.interactive else None, ) await _converse( stage, @@ -434,7 +434,7 @@ async def _live(args: argparse.Namespace) -> int: stage, _ = run_stage( contract, out=destination, - ask=permission_gate(_ask_operator) if args.interactive else None, + ask=operator_ask(_ask_operator) if args.interactive else None, ) await _converse( stage, run_opening(contract, destination), interactive=args.interactive @@ -1087,7 +1087,7 @@ async def _chat(args: argparse.Namespace) -> int: path=args.path or "", kind=args.kind, out=Path(args.out) if args.out else None, - ask=permission_gate(_ask_operator), + ask=operator_ask(_ask_operator), ) print(f"model: {chosen_model()}") print(credentials_hint()) diff --git a/src/fi/alk/harness/config.py b/src/fi/alk/harness/config.py index 4f78166a..bb7f49e2 100644 --- a/src/fi/alk/harness/config.py +++ b/src/fi/alk/harness/config.py @@ -124,7 +124,7 @@ def provider_env(model: str | None = None) -> dict[str, str]: return env -def read_only_session( +def working_session( *, system_prompt: str, cwd: str | Path, @@ -133,17 +133,27 @@ def read_only_session( max_turns: int = 40, model: str | None = None, ) -> SessionSpec: - """A session that may read the agent under test but never write to it. + """A session that can read, write and run things. - The agent under test is somebody's real repository. The harness reads it and writes its own - artifacts elsewhere, so the built-in write tools are simply not granted; the only way this - session can produce anything is by calling one of ours. + The sandbox is the boundary. A stage that has to work out what an unfamiliar agent is, build + the infrastructure it talks to, and prove that infrastructure answers, is doing engineering, + and engineering needs a shell and an editor. Withholding them did not make the work safer, it + made the stage unable to finish it and left the finishing to a person reading logs. """ return SessionSpec( system_prompt=system_prompt, servers=dict(servers or {}), builtins=tuple( - dict.fromkeys([*_READ_ONLY_TOOLS, "AskUserQuestion", *extra_builtins]) + dict.fromkeys( + [ + *_READ_ONLY_TOOLS, + "Write", + "Edit", + "Bash", + "AskUserQuestion", + *extra_builtins, + ] + ) ), cwd=str(cwd), max_turns=max_turns, @@ -152,84 +162,19 @@ def read_only_session( ) -# Host tools no stage gets unless it asks. Off by default because a tool nobody wants still -# costs the turn that discovers it, not because any of them is forbidden: a stage that names one -# in its builtins gets it, and the backend removes it from this list for that session. -UNWANTED = ( - "ToolSearch", - "Bash", - "Write", - "Edit", - "NotebookEdit", - "WebFetch", - "WebSearch", -) - - -def gate_hooks(granted: Iterable[str]) -> dict[str, Any]: - """Deny anything a stage was not given, at the point the SDK actually asks. +def operator_ask(ask: Any | None = None) -> Any: + """Route the model's questions to whoever is running this, and allow everything else. - ``can_use_tool`` alone does not do this. An ``allowed_tools`` entry approves those tools - before the callback is consulted, and the SDK then warns that the callback is shadowed — so - the gate never runs for the tools we granted, and in practice does not stop the ones we did - not either. A host ``ToolSearch`` reached every stage, returned nothing, and cost a turn each - time. - - A PreToolUse hook is consulted for every call, which is what the deny-by-default rule needed - in order to be true rather than intended. - """ - from claude_agent_sdk.types import HookMatcher - - permitted = {*granted, "AskUserQuestion"} - - async def refuse( - payload: dict[str, Any], _tool_use_id: Any, _context: Any - ) -> dict[str, Any]: - name = str(payload.get("tool_name") or "") - if not name or name in permitted: - return {} - return { - "hookSpecificOutput": { - "hookEventName": "PreToolUse", - "permissionDecision": "deny", - "permissionDecisionReason": ( - f"{name} is not part of this stage. You have " - f"{', '.join(sorted(permitted)) or 'no other tools'}, and everything you " - "produce goes through those, because those are what check it." - ), - } - } - - return {"PreToolUse": [HookMatcher(hooks=[refuse])]} - - -def permission_gate(ask: Any | None = None, granted: Iterable[str] = ()) -> Any: - """Decide what a stage may do: nothing it was not given. - - Deny by default, not deny-a-list. A session is offered whatever tools its host happens to - expose, and anything not named here is by definition not part of how this stage works. An - allow-by-default gate let a host search tool through, which returned nothing useful and cost - a stage its entire turn budget looping on it; the same hole would let a file write through. - - Tools granted through ``allowed_tools`` are approved before this is consulted, so this only - ever sees the ones that were not. + What used to be here also denied by default. That gate is gone: a stage is now trusted with + the tools it was given, and the sandbox is what stands between a mistake and anything real. """ - permitted = set(granted) async def gate(tool_name: str, payload: dict[str, Any], context: Any) -> Any: - from claude_agent_sdk.types import PermissionResultAllow, PermissionResultDeny + from claude_agent_sdk.types import PermissionResultAllow if tool_name == "AskUserQuestion" and ask is not None: return await ask(tool_name, payload, context) - if tool_name in permitted: - return PermissionResultAllow(updated_input=payload) - return PermissionResultDeny( - message=( - f"{tool_name} is not part of this stage. You have " - f"{', '.join(sorted(permitted)) or 'no other tools'}, and everything you " - "produce goes through those, because those are what check it." - ) - ) + return PermissionResultAllow(updated_input=payload) return gate diff --git a/src/fi/alk/harness/understand.py b/src/fi/alk/harness/understand.py index 2b6c1b62..fb4fe62a 100644 --- a/src/fi/alk/harness/understand.py +++ b/src/fi/alk/harness/understand.py @@ -13,7 +13,7 @@ from pathlib import Path from typing import Any, Callable -from .config import artifact_dir, load_skill, read_only_session +from .config import artifact_dir, load_skill, working_session from .contract import AgentContract from .session import Stage from .sources import AgentSource @@ -31,7 +31,7 @@ def open_stage( ) -> tuple[Stage, Path]: """A live understand-the-agent stage, and where it will write.""" destination = out or artifact_dir(source.name) - spec = read_only_session( + spec = working_session( system_prompt=f"{load_skill(SKILL)}\n\n## This agent\n\n{source.briefing()}", cwd=source.workdir(), servers={**source.servers(), CONTRACT_SERVER: contract_tools(destination)}, diff --git a/src/fi/alk/harness/world/tools.py b/src/fi/alk/harness/world/tools.py index 2d5e76b0..e206d8ec 100644 --- a/src/fi/alk/harness/world/tools.py +++ b/src/fi/alk/harness/world/tools.py @@ -1068,49 +1068,6 @@ async def add_sub_goal(args: dict[str, Any]) -> dict[str, Any]: f"{settled} settled by code: " + ", ".join(sorted(catalogue.names())) ) - @tool( - "write_env_file", - "Write one file the environment is built from: a Dockerfile, a compose file, a schema, an " - "entrypoint, whatever this agent needs. Paths are relative and stay inside the " - "environment directory. Call it once per file, then build with run_env_command.", - schema({"path": str, "contents": str}, ["path", "contents"]), - ) - async def write_env_file(args: dict[str, Any]) -> dict[str, Any]: - from .workspace import listing, write - - try: - written = write(destination, str(args["path"]), str(args["contents"])) - except ValueError as refused: - return _err(str(refused)) - lines = len(str(args["contents"]).splitlines()) - return _ok( - f"wrote {written.name}, {lines} lines. The environment now has: " - + ", ".join(listing(destination)) - ) - - @tool( - "run_env_command", - "Run one docker or docker compose command from the environment directory: build an image, " - "bring a store up, run something inside a container. Only container commands run here, so " - "anything the environment needs belongs in a file it builds from rather than in a " - "command. Returns the exit code and the output.", - schema({"command": str}, ["command"]), - ) - async def run_env_command(args: dict[str, Any]) -> dict[str, Any]: - from .workspace import run - - # Off the event loop: a docker build takes minutes, and run synchronously - # it deafens every API endpoint this server has until it finishes. - code, output = await asyncio.to_thread(run, destination, str(args["command"])) - shown = ( - output - if len(output) <= 2500 - else output[:1200] + "\n...\n" + output[-1200:] - ) - if code != 0: - return _err(f"exit {code}\n{shown or '(no output)'}") - return _ok(f"ok\n{shown or '(no output)'}") - @tool( "write_store_ops", "Teach the harness an engine it has never stood up: the image, the port it listens on, " @@ -1376,8 +1333,6 @@ async def save_world(args: dict[str, Any]) -> dict[str, Any]: adopt_tool, write_store_ops, add_world_check, - write_env_file, - run_env_command, check_world, save_world, ], @@ -1405,8 +1360,6 @@ async def save_world(args: dict[str, Any]) -> dict[str, Any]: "add_sub_goal", "write_store_ops", "add_world_check", - "write_env_file", - "run_env_command", "check_world", "save_world", ) diff --git a/src/fi/alk/harness/world/workspace.py b/src/fi/alk/harness/world/workspace.py deleted file mode 100644 index bd0acfde..00000000 --- a/src/fi/alk/harness/world/workspace.py +++ /dev/null @@ -1,143 +0,0 @@ -"""Standing the environment up in containers, with the harness deciding what that means. - -The harness has read the agent's repository, so it knows what running that agent's code takes: -which base image, which install command, which store, which services. Encoding any of that here -would be guessing on behalf of an agent nobody has seen yet, and would be wrong for the next one. - -So this provides two things and no opinions: - -- a place to write files, under the session's own ``env`` directory -- a way to run container commands from there, and read back what happened - -Everything else, the Dockerfile, the compose file, the schema, the entrypoint, is written by -whoever read the repository. What is enforced is only what keeps this safe to run on somebody's -machine: files stay inside the environment directory, and the only commands that run are container -commands. -""" - -from __future__ import annotations - -import os -import shlex -import shutil -import subprocess -from pathlib import Path - -ENV = "env" - -# Everything runs. The sandbox is the boundary: a stage that can build an environment for an -# agent nobody anticipated needs to install, compile and test whatever that agent depends on, -# and an allowlist of two container commands can only ever describe the agents we already knew. -ALLOWED: tuple[str, ...] = () - -# Long enough for an image build that downloads a base layer, short enough that a hung build is -# reported rather than waited on forever. -PATIENCE = 900 - - -def env_root(destination: Path) -> Path: - """Where this agent's environment definition lives, beside its world.""" - root = Path(destination) / ENV - root.mkdir(parents=True, exist_ok=True) - return root - - -def inside(destination: Path, path: str) -> Path: - """The full path for a file the harness wants to write, refused if it escapes. - - A path arrives as text from a model, so it is resolved and then checked rather than trusted. - Writing outside the environment directory would mean the harness could touch anything on the - machine it happens to be running on, which is not a thing to leave to a prompt. - """ - root = env_root(destination).resolve() - asked = (root / str(path).lstrip("/")).resolve() - if not asked.is_relative_to(root): - raise ValueError( - f"{path!r} is outside the environment directory. Everything the environment needs " - "lives under env/, so that building it cannot reach the rest of the machine." - ) - return asked - - -def write(destination: Path, path: str, contents: str) -> Path: - """Put one file into the environment definition.""" - target = inside(destination, path) - target.parent.mkdir(parents=True, exist_ok=True) - target.write_text(contents, encoding="utf-8") - return target - - -def listing(destination: Path) -> list[str]: - root = env_root(destination) - return sorted( - str(found.relative_to(root)) for found in root.rglob("*") if found.is_file() - ) - - -def available() -> str: - """Why containers cannot be used here, or an empty string when they can.""" - if not shutil.which("docker"): - return "docker is not installed, or not on the path" - done = subprocess.run( - ["docker", "info", "--format", "{{.ServerVersion}}"], - capture_output=True, - text=True, - timeout=30, - ) - if done.returncode != 0: - return ( - f"docker is installed but not running: {(done.stderr or '').strip()[:200]}" - ) - return "" - - -def run( - destination: Path, command: str, *, patience: int = PATIENCE -) -> tuple[int, str]: - """Run one container command from the environment directory. - - Returns the exit code and the output, both streams together, because a build failure explains - itself across the two and reading only one is how the actual cause gets lost. - """ - try: - words = shlex.split(command) - except ValueError as exc: - return 1, f"could not parse container command: {exc}" - if not words: - return 1, "no command given" - if ALLOWED and words[0] not in ALLOWED: - return 1, ( - f"{words[0]!r} is not something this can run. Only {' and '.join(ALLOWED)} commands." - ) - blocked = available() - if blocked: - return 1, blocked - # When the daemon is remote (DOCKER_HOST at a socket proxy), a bind mount - # names a path on the daemon's host — this container's own filesystem is - # invisible to it. The mount comes up empty and the failure reads as a - # missing file three steps later, so it is refused here with the reason. - if os.environ.get("DOCKER_HOST") and ( - " -v " in f" {command} " or "--volume" in command - ): - return 1, ( - "bind mounts cannot work in this deployment: the docker daemon runs " - "outside this container and does not see these paths. Run the script " - "inline instead (sh -c '