Derive a coverage grid from the agent's contract and sample it for any suite size

harness-ui/server.py

@@ -36,7 +36,7 @@
 from fi.alk.harness.chat import Conversation  # noqa: E402
 from fi.alk.harness.config import chosen_model, credentials_hint  # noqa: E402
 from fi.alk.harness.run import run_suite  # noqa: E402
-from fi.alk.harness.scenarios import load as load_scenarios  # noqa: E402
+from fi.alk.harness.scenariogen.store.suite import load_scenarios  # noqa: E402
 from fi.alk.harness.understand import load as load_contract  # noqa: E402
 
 app = FastAPI(title="harness")
@@ -469,8 +469,8 @@ async def scenarios():
     shown as unknown.
     """
     from fi.alk.harness.environment import load_catalogue
-    from fi.alk.harness.folder import folder_for
-    from fi.alk.harness.prove import prove
+    from fi.alk.harness.scenariogen.store.folder import folder_for
+    from fi.alk.harness.scenariogen.write.prove import prove
 
     out = current.path if current else None
     if not out:
@@ -518,7 +518,7 @@ async def scenario_file(name: str, path: str):
     Resolved and then checked to be inside that scenario's own folder: the path comes from a
     query string, and a page is not a trustworthy source of one.
     """
-    from fi.alk.harness.folder import folder_for
+    from fi.alk.harness.scenariogen.store.folder import folder_for
 
     out = current.path if current else None
     if not out:

src/fi/alk/harness/HOW-IT-WORKS.md

@@ -155,7 +155,7 @@ Then `world/snapshot.py` writes `world.sqlite`, `handlers/*.py`, `world.py` and
 
 ## 4. Scenarios — the conversations worth having
 
-`scenarios.py`, `skills/write-scenarios/SKILL.md`, tools in `scenario_tools.py`
+`scenarios.py`, `skills/scenarios/write/SKILL.md`, tools in `scenario_tools.py`
 
 A scenario is a **change on the base environment**, and it owns a folder (`folder.py`):
 

src/fi/alk/harness/__init__.py

@@ -25,7 +25,8 @@
 )
 from .contract import AgentContract, Runtime, RuntimeInterface, ToolSpec, validate_contract
 from .job import ExecutionMode, HarnessJob, HarnessStage
-from .scenario import Scenario, validate_scenario
+from .scenariogen.model.scenario import Scenario
+from .scenariogen.quality.checks import validate_scenario
 from .session import Stage, Turn
 from .sources import (
     AgentSource,

src/fi/alk/harness/authoring_entrypoint.py

@@ -18,7 +18,7 @@
 from .cli import _auto
 from .job import HarnessJob, ProviderExecutionMode
 from .provider_import import inspect_provider_target
-from .scenarios import load as load_written
+from .scenariogen.store.suite import load_scenarios as load_written
 from .understand import PROVIDER_IMPORT_PROFILE_PATH_ENV
 
 

src/fi/alk/harness/backends/__init__.py

@@ -25,6 +25,7 @@
     Say,
     SessionOpened,
     SessionSpec,
+    WorkerSpec,
     StageDone,
     ToolReturned,
     ToolServer,
@@ -45,6 +46,7 @@
     "Say",
     "SessionOpened",
     "SessionSpec",
+    "WorkerSpec",
     "StageDone",
     "ToolReturned",
     "ToolServer",
@@ -94,11 +96,15 @@ def backend_names() -> list[str]:
     return sorted(_LOADERS)
 
 
-def resolve(name: str | None = None) -> HarnessBackend:
+def resolve(name: str | None = None, model: str | None = None) -> HarnessBackend:
     """The backend a run will use: the one named, or ALK_HARNESS, or the default.
 
     An unknown name is a loud error naming what exists. Falling back silently would run a whole
     suite on the wrong harness, which is only discovered from the bill.
+
+    ``model`` is the model this caller will actually use, which is not always the run's. A stage
+    that names its own backend names its own model with it, and checking the pair against the
+    run's global model instead rejected the very combination the setting exists to express.
     """
     asked = (name or os.environ.get("ALK_HARNESS") or DEFAULT_BACKEND).strip().lower()
     asked = _ALIASES.get(asked, asked)
@@ -113,7 +119,7 @@ def resolve(name: str | None = None) -> HarnessBackend:
     # A named model that this backend cannot reach is a configuration mistake, and it is only
     # visible here. Left to run, the provider rejects the model mid-stage and the failure reads
     # as the harness having nothing to say rather than as the wrong pairing.
-    wanted = os.environ.get("ALK_HARNESS_MODEL", "").strip()
+    wanted = (model or os.environ.get("ALK_HARNESS_MODEL", "")).strip()
     if wanted and not backend.can_drive(wanted):
         raise ValueError(
             f"harness backend {backend.name!r} cannot drive model {wanted!r}; "

src/fi/alk/harness/backends/base.py

@@ -23,11 +23,25 @@
 
 from __future__ import annotations
 
+import os
 from dataclasses import dataclass, field
 from typing import Any, AsyncIterator, Awaitable, Callable, Protocol, runtime_checkable
 
 ToolHandler = Callable[[dict[str, Any]], Awaitable[dict[str, Any]]]
 
+# How many workers a backend may run at once. Both SDKs cap this themselves (one at twenty by
+# default), so this is the harness's own ceiling, set below theirs and shared so that the turn
+# budget reserved for a fan-out matches what can actually be in flight.
+# The one ceiling on fan-out: how many workers may run at the same time. It protects the
+# machine, and it is a constant rather than an environment variable because a setting
+# nobody remembers makes every run behave differently for reasons nothing on screen
+# explains. The scenarios stage imports this as its own ceiling and tells the model.
+# The hard ceiling on writers running at once, enforced by claim_slice rather than asked
+# for: a prompt is guidance and this is a safety limit on the machine. Ten is the number
+# a stage is told to aim at; the extra two are headroom so a brief overshoot is not a
+# refusal in the middle of a suite.
+MOST_WORKERS_AT_ONCE = 12
+
 
 def qualified(server: str, tool_name: str) -> str:
     """The fully qualified name a session grants and a model calls.
@@ -91,8 +105,44 @@ def tool_server(
 # a host CLI implements them from files.py. Anything else asked for as a builtin is refused at
 # session build time rather than silently dropped.
 FILE_TOOLS = ("Read", "Glob", "Grep")
+
+# Everything a backend without a host CLI can offer under the names Claude Code uses, so a
+# stage's skill text means the same thing whichever loop is running it. Reading tells you what
+# an agent is meant to do and running tells you what it does; the scenarios worth writing come
+# from the gap, so a stage trusted with one is trusted with both. Writing is absent on purpose:
+# a stage able to edit the agent under test could make its scenarios pass by changing the agent.
+HOST_TOOLS = (*FILE_TOOLS, "Bash")
 ASK_TOOL = "AskUserQuestion"
-KNOWN_BUILTINS = (*FILE_TOOLS, ASK_TOOL)
+DELEGATE_TOOL = "Delegate"
+KNOWN_BUILTINS = (*FILE_TOOLS, ASK_TOOL, DELEGATE_TOOL)
+
+
+@dataclass
+class WorkerSpec:
+    """A worker the model may run to do part of its stage, in its own session.
+
+    Both backends we ship can start a second model session and hand its answer back as a tool
+    result, and both decide at call time how many to run; only the vocabulary differs. This is
+    that capability said once, so a stage declares a worker and every backend honours it.
+
+    ``instructions`` is the worker's own system prompt. ``servers`` and ``builtins`` are its
+    tools, named exactly as ``SessionSpec`` names them, and default to the parent's when left
+    empty. ``max_turns`` bounds one worker; ``model`` overrides the parent's for it.
+
+    A worker never inherits the parent's conversation. Everything it needs comes from
+    ``instructions`` plus the brief the model writes when it calls, which is what keeps a large
+    fan-out from copying the whole stage history N times.
+    """
+
+    description: str
+    instructions: str
+    servers: dict[str, ToolServer] = field(default_factory=dict)
+    builtins: tuple[str, ...] = ()
+    max_turns: int = 40
+    model: str = ""
+    # How hard this worker may think. Empty leaves the model's own default alone. Held here
+    # rather than taken from the stage because a worker's job is not the stage's job.
+    effort: str = ""
 
 
 @dataclass
@@ -122,6 +172,21 @@ class SessionSpec:
     # stage (understand, interactive) passes its gate in fully built; backends without a
     # permission callback concept ignore it, which is safe because their gating is structural.
     permission_override: Any = None
+    # Workers this session may run, by name. Declaring any of these is what lets the model
+    # divide its own work; a stage that declares none behaves exactly as before.
+    workers: dict[str, WorkerSpec] = field(default_factory=dict)
+
+    def worker_turns(self) -> int:
+        """Turns to reserve beyond the parent's own, so a fan-out cannot run the budget dry.
+
+        One backend bills every worker's turns to the session that started them, so a budget
+        sized for the parent alone stops a fan-out partway through and loses the work. Reserving
+        the worst case here keeps that a backend detail rather than a stage's problem.
+        """
+        if not self.workers:
+            return 0
+        widest = max(worker.max_turns for worker in self.workers.values())
+        return widest * MOST_WORKERS_AT_ONCE
 
     def granted(self) -> list[str]:
         """Every tool name this session may call, qualified the way the model calls it."""

src/fi/alk/harness/backends/claude.py

@@ -14,6 +14,7 @@
 from typing import Any, AsyncIterator
 
 from claude_agent_sdk import (
+    AgentDefinition,
     AssistantMessage,
     ClaudeAgentOptions,
     ClaudeSDKClient,
@@ -27,6 +28,7 @@
 )
 
 from .base import (
+    MOST_WORKERS_AT_ONCE,
     Call,
     ModelReply,
     Say,
@@ -40,6 +42,26 @@
 
 DEFAULT_MODEL = "claude-sonnet-4-6"
 
+# What this SDK calls the tool that runs a worker. The harness calls the capability
+# ``Delegate``; only this line knows the vendor's name for it.
+_DELEGATION_TOOL = "Agent"
+
+
+def _ask_only(ask: Any) -> Any:
+    """Allow everything, but route the one tool that has a person on the other end.
+
+    An ungated stage is trusted with its tools; it is not therefore talking to nobody.
+    """
+
+    async def gate(tool_name: str, payload: dict[str, Any], context: Any) -> Any:
+        from claude_agent_sdk.types import PermissionResultAllow
+
+        if tool_name == "AskUserQuestion":
+            return await ask(tool_name, payload, context)
+        return PermissionResultAllow(updated_input=payload)
+
+    return gate
+
 
 def _sdk_server(server: ToolServer) -> Any:
     """A ToolServer as the in-process MCP server the SDK routes calls to."""
@@ -164,20 +186,95 @@ def create(self, spec: SessionSpec) -> ClaudeSession:
                 for tool_spec in server.tools
             ),
         ]
+        servers = {
+            server_name: _sdk_server(server)
+            for server_name, server in spec.servers.items()
+        }
+        agents: dict[str, Any] = {}
+        for name, worker in spec.workers.items():
+            worker_servers = worker.servers or spec.servers
+            for server_name, server in worker_servers.items():
+                servers.setdefault(server_name, _sdk_server(server))
+            agents[name] = AgentDefinition(
+                description=worker.description,
+                prompt=worker.instructions,
+                tools=[
+                    *(worker.builtins or spec.builtins),
+                    *(
+                        qualified(server_name, tool_spec.name)
+                        for server_name, server in worker_servers.items()
+                        for tool_spec in server.tools
+                    ),
+                ],
+                mcpServers=list(worker_servers),
+                model=worker.model or "inherit",
+                maxTurns=worker.max_turns,
+                # Blocking, so the call returns the worker's result. Left unset these launch in
+                # the background and the caller is told it will be notified later: a stage that
+                # dealt fifty scenarios across five writers then ended at its next turn, killing
+                # all five, and reported success having saved one. The parent has nothing useful
+                # to do while a slice is written, and it must not finish before the work does.
+                background=False,
+                # Only when asked for: the field has its own default and passing a blank
+                # through would override it with nothing.
+                **({"effort": worker.effort} if worker.effort else {}),
+            )
+        if agents:
+            # The delegation tool is named for the model, and it has to be granted here or the
+            # gate below refuses the very call these workers exist to receive.
+            allowed = [*allowed, _DELEGATION_TOOL]
+        env = dict(provider_env(spec.model))
+        if agents:
+            env.setdefault(
+                "CLAUDE_CODE_MAX_CONCURRENT_SUBAGENTS", str(MOST_WORKERS_AT_ONCE)
+            )
+            # One level only. A worker that delegates again multiplies the fan-out by a factor
+            # nothing in the stage accounted for, and the depth is free to raise later.
+            env.setdefault("CLAUDE_CODE_MAX_SUBAGENT_SPAWN_DEPTH", "1")
+        # A declared worker is reached through the SDK's own sub-agent tool, which asks *which*
+        # kind to run. Nothing otherwise tells the model that ours exists, and left to guess it
+        # dispatches the generic kind: that one launches detached, answers "launched
+        # successfully, you will be notified", and returns nothing, so a stage dealt eight slices,
+        # dispatched eight agents holding none of its tools, declared success and exited having
+        # written nothing. On the other backend a worker is a plain tool in the list, which is why
+        # this only ever bit here. Naming them costs a line and removes the guess.
+        said = spec.system_prompt
+        if spec.workers:
+            said += (
+                "\n\n## Your workers\n\nYou have these sub-agents: "
+                + ", ".join(sorted(spec.workers))
+                + ". Dispatch one by naming it exactly as the kind of sub-agent to run. They "
+                "return what they produced when they finish. Do not dispatch a general-purpose "
+                "sub-agent instead: that kind runs detached, holds none of your tools, and its "
+                "work is lost when you finish your turn."
+            )
         options = ClaudeAgentOptions(
-            system_prompt=spec.system_prompt,
+            system_prompt=said,
             allowed_tools=allowed,
-            mcp_servers={
-                server_name: _sdk_server(server)
-                for server_name, server in spec.servers.items()
-            },
+            mcp_servers=servers,
             setting_sources=[],
             max_turns=spec.max_turns,
             model=spec.model,
-            env=provider_env(spec.model),
+            env=env,
         )
+        if agents:
+            options.agents = agents
         if spec.cwd is not None:
             options.cwd = spec.cwd
+        if not spec.gated:
+            # An ungated stage is given the host's whole tool list on purpose, and an unattended
+            # run cannot answer a prompt, so approval has to be settled here rather than left to
+            # the default. The boundary for such a stage is the sandbox it runs in, not the tool
+            # list: it is reading an agent's own repository to write tests against it, and every
+            # artifact it produces still goes through the three gates before it is kept.
+            options.permission_mode = "bypassPermissions"
+            if spec.permission_override is not None:
+                options.can_use_tool = spec.permission_override
+            elif spec.ask is not None:
+                # Ungated does not mean unattended. In a conversation there is a person on the
+                # other side, and the stage asking them something is the point of keeping it
+                # open. Without this the question is approved silently and never reaches them.
+                options.can_use_tool = _ask_only(spec.ask)
         if spec.gated:
             # Not acceptEdits: that auto-approves Edit and Write before the permission callback
             # is consulted, so a stage could rewrite an artifact by hand and skip the tool whose