From 98e3d88aefb762217255870ace36842dac241604 Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 13:02:26 +0530 Subject: [PATCH 1/2] fix(harness): mount credentials safely, pass provider keys through, and bound stage idle time --- src/fi/alk/harness/provision.py | 27 +++++++++++++++ src/fi/alk/harness/run/live.py | 31 +++++++++++++++++ src/fi/alk/harness/secrets.py | 37 +++++++++++++++++++++ src/fi/alk/harness/session.py | 2 +- src/fi/alk/harness/world/stores/postgres.py | 11 ++++-- 5 files changed, 104 insertions(+), 4 deletions(-) diff --git a/src/fi/alk/harness/provision.py b/src/fi/alk/harness/provision.py index 0836b5f2..aecce513 100644 --- a/src/fi/alk/harness/provision.py +++ b/src/fi/alk/harness/provision.py @@ -1804,6 +1804,17 @@ def provision( else: compose = None managed = False + # The harness's fidelity order is provisioned > adopted > generated: run the agent's real + # services whenever it ships them. _managed_compose only models "agent + datastore" and + # silently drops any other service the agent's tools are actually served by -- an HTTP + # tools-api, a queue, a mock upstream -- which then leaves the world with no endpoint to + # forward to, so every tool call comes back "no such tool". So prefer the agent's own shipped + # Compose whenever it ships one (its real tool services come up and the world forwards to + # them), and fall back to the generated adapter only for agents that ship no usable Compose. + if compose is None: + shipped = compose_file(source_root) + if shipped is not None: + compose = shipped if compose is None and contract is not None: if not packaging.candidates and not (source_root / "Dockerfile").is_file(): try: @@ -2227,6 +2238,18 @@ def _runtime_credential_mounts( # the worker to see the placeholder. Platform credentials always get an # ALK-owned destination that cannot collide with submitted mounts. target = f"/run/harness-secrets/{runtime.name}" + if source is None or not _valid_google_credentials(source): + # A local sandbox supplies the credential through its own environment rather than + # an uploaded runtime configuration. That credential belongs to whoever is running + # the harness, not to the submitted agent, so it is only ever handed over when the + # operator says so: without this opt-in a hosted runner would mount its own + # platform key into a container it does not trust. + if os.environ.get("ALK_ALLOW_HOST_GOOGLE_CREDENTIALS", "").strip() == "1": + env_value = os.environ.get(name, "").strip() + env_path = Path(env_value).expanduser() if env_value else None + if env_path is not None and _valid_google_credentials(env_path): + source = env_path + target = f"/run/harness-secrets/{env_path.name}" if source is None or not _valid_google_credentials(source): raise ProvisionError( "the submitted runtime needs GOOGLE_APPLICATION_CREDENTIALS, but neither its " @@ -2358,6 +2381,10 @@ def start_runtime( arguments.extend(("--volume", f"{source}:{target}:ro")) injected[name] = target mounted_credentials.add(name) + # A submitted service that declares its own credential volume has already been mounted (the + # host secret now lives at the container target). Re-validating that container path as a host + # file below would always fail, so only resolve GOOGLE_APPLICATION_CREDENTIALS here when it + # arrived as an injected host path (the generated-runtime path, which mounts no credentials). google_path = injected.get("GOOGLE_APPLICATION_CREDENTIALS", "").strip() if google_path and "GOOGLE_APPLICATION_CREDENTIALS" not in mounted_credentials: google_source = Path(google_path).expanduser() diff --git a/src/fi/alk/harness/run/live.py b/src/fi/alk/harness/run/live.py index 90b4978e..e6873e64 100644 --- a/src/fi/alk/harness/run/live.py +++ b/src/fi/alk/harness/run/live.py @@ -278,6 +278,37 @@ def wire( "TOOLS_API_URL": url, "LIVEKIT_AGENT_NAME": agent_name, } + # The harness-generated agent-runtime does not carry the agent's own .env.local, and + # runtime_configuration_names only covers datastore config -- not the provider + # credentials the worker needs to actually run: LiveKit to register and place the + # call, Deepgram for STT/TTS, Vertex for the LLM. Pass them through from this + # process's environment (the sandbox has placed them here). + for _cred in ( + "LIVEKIT_URL", + "LIVEKIT_API_KEY", + "LIVEKIT_API_SECRET", + "DEEPGRAM_API_KEY", + "CARTESIA_API_KEY", + "GOOGLE_APPLICATION_CREDENTIALS", + "GOOGLE_CLOUD_PROJECT", + "GOOGLE_CLOUD_LOCATION", + ): + _value = os.environ.get(_cred, "").strip() + if _value: + runtime_overrides.setdefault(_cred, _value) + # An agent's Compose commonly mounts its Vertex credential from an env-var source, e.g. + # ${VERTEX_CREDENTIALS:-/dev/null}:/etc/vertex/creds.json. With that variable unset the + # placeholder is mounted and the agent's own LLM cannot authenticate, so point it at the + # same resolved Google credential the harness already holds for the run. + _google_creds = runtime_overrides.get("GOOGLE_APPLICATION_CREDENTIALS", "").strip() + if _google_creds: + runtime_overrides.setdefault("VERTEX_CREDENTIALS", _google_creds) + # Voice agents often gate an audio-enhancement plugin on a license the harness cannot + # supply (e.g. ai_coustics noise cancellation). Unauthorized, it raises on the first + # inbound audio frame and the worker drops the call right after its greeting. Agents + # that read this flag skip that plugin; agents that do not simply ignore it. A run may + # override it to keep enhancement on when a license is present. + runtime_overrides.setdefault("DISABLE_AI_COUSTICS", "1") os.environ["LIVEKIT_TARGET_AGENT_NAME"] = agent_name caller_phone = fixture_phone(scenario) if caller_phone: diff --git a/src/fi/alk/harness/secrets.py b/src/fi/alk/harness/secrets.py index 018ecdbf..497fcf6b 100644 --- a/src/fi/alk/harness/secrets.py +++ b/src/fi/alk/harness/secrets.py @@ -95,6 +95,9 @@ def worker_environment( # Runner-owned model configuration. Uploaded agent values with these names remain in the # runtime namespace and cannot replace controller credentials. "ALK_HARNESS_MODEL", + # Lets a local sandbox hand its own Google credential to the runtime it builds. A hosted + # runner leaves this unset so its platform key is never mounted into a submitted agent. + "ALK_ALLOW_HOST_GOOGLE_CREDENTIALS", "ANTHROPIC_MODEL", "ANTHROPIC_VERTEX_PROJECT_ID", "CLAUDE_CODE_USE_VERTEX", @@ -108,10 +111,44 @@ def worker_environment( "FI_BASE_URL", "FI_API_KEY", "FI_SECRET_KEY", + # Runner-owned Docker runtime + voice configuration. The harness starts store and agent + # containers on the host daemon and must reach them: ALK_DOCKER_NETWORK lets the store be + # reached by container name on a shared network, ALK_DOCKER_PUBLISHED_HOST/BIND_HOST give + # the host published services are on. Without these the child defaults to 127.0.0.1 -- + # its own loopback inside the sandbox -- and every store probe is refused. + "ALK_DOCKER_NETWORK", + "ALK_DOCKER_PUBLISHED_HOST", + "ALK_DOCKER_BIND_HOST", + "ALK_RUNNER_CONTAINER", + "ALK_HARNESS_MODEL", + "ALK_AGENT_MODEL", + "ALK_JUDGE_MODEL", + "ALK_USER_MODEL", + "CLOUD_ML_REGION", + "HARNESS_WEBHOOK_HOST", + "HARNESS_WEBHOOK_PORT", + "HARNESS_WEBHOOK_URL", + "HARNESS_RUNTIME_WEBHOOK_URL", + "HARNESS_VOICE_CASE", + "HARNESS_VOICE_INFRA_RETRIES", + "LIVEKIT_TARGET_AGENT_NAME", + # Local-dev convenience: let the developer's provider creds from their local environment + # reach the worker directly. A hosted provider supplies these through secret_refs instead. + "LIVEKIT_URL", + "LIVEKIT_API_KEY", + "LIVEKIT_API_SECRET", + "ACCEPTANCE_LIVEKIT_URL", + "DEEPGRAM_API_KEY", + "CARTESIA_API_KEY", + "GOOGLE_APPLICATION_CREDENTIALS", + "GOOGLE_CLOUD_PROJECT", + "GOOGLE_CLOUD_LOCATION", } child = {name: value for name, value in host.items() if name in allowed} reserved = { "ALK_HARNESS_MODEL", + # A submitted job must not be able to turn on the host-credential fallback for itself. + "ALK_ALLOW_HOST_GOOGLE_CREDENTIALS", "ANTHROPIC_MODEL", "ANTHROPIC_VERTEX_PROJECT_ID", "CLAUDE_CODE_USE_VERTEX", diff --git a/src/fi/alk/harness/session.py b/src/fi/alk/harness/session.py index af652abd..4f2db62a 100644 --- a/src/fi/alk/harness/session.py +++ b/src/fi/alk/harness/session.py @@ -38,7 +38,7 @@ # remain alive forever after a dropped upstream stream, though, which previously left a hosted # job looking healthy while making no progress. Bound *inactivity*, not total stage duration: # long scenario suites remain valid as long as they keep producing observable work. -STAGE_IDLE_TIMEOUT_SECONDS = float(os.getenv("ALK_STAGE_IDLE_TIMEOUT_SECONDS", "180")) +STAGE_IDLE_TIMEOUT_SECONDS = float(os.getenv("ALK_STAGE_IDLE_TIMEOUT_SECONDS", "600")) STAGE_IDLE_RETRIES = int(os.getenv("ALK_STAGE_IDLE_RETRIES", "1")) diff --git a/src/fi/alk/harness/world/stores/postgres.py b/src/fi/alk/harness/world/stores/postgres.py index 0a89b444..8f9f83b6 100644 --- a/src/fi/alk/harness/world/stores/postgres.py +++ b/src/fi/alk/harness/world/stores/postgres.py @@ -237,9 +237,14 @@ def save_to(self, path: str | Path) -> None: def load_from(self, path: str | Path) -> None: root = Path(path) schema = root / SCHEMA - if not schema.exists(): - raise StoreError(f"no saved Postgres schema at {schema}") - self.apply(schema.read_text(encoding="utf-8")) + # A standalone scenario store starts empty and needs the DDL; a compose-provisioned + # (Attached) store saves no schema.sql because store.json already carries the applied + # CREATE scripts, which Held.load_from replays. Apply schema.sql only when it exists and + # the tables are not already there, so restore works either way and never double-applies. + with self._connect() as connection: + has_schema = bool(self._tables(connection)) + if not has_schema and schema.exists(): + self.apply(schema.read_text(encoding="utf-8")) Held.load_from(self, root) # -- what a scenario changes ----------------------------------------------------- From 14bdee155a49c8f39c9531c5e4e7814bff9815ce Mon Sep 17 00:00:00 2001 From: KarthikAvinashFI Date: Tue, 25 Aug 2026 13:09:49 +0530 Subject: [PATCH 2/2] chore(harness): tighten comments on the credential and runtime changes --- src/fi/alk/harness/provision.py | 22 ++++++---------------- src/fi/alk/harness/run/live.py | 20 ++++++-------------- 2 files changed, 12 insertions(+), 30 deletions(-) diff --git a/src/fi/alk/harness/provision.py b/src/fi/alk/harness/provision.py index aecce513..1afdfc9e 100644 --- a/src/fi/alk/harness/provision.py +++ b/src/fi/alk/harness/provision.py @@ -1804,13 +1804,8 @@ def provision( else: compose = None managed = False - # The harness's fidelity order is provisioned > adopted > generated: run the agent's real - # services whenever it ships them. _managed_compose only models "agent + datastore" and - # silently drops any other service the agent's tools are actually served by -- an HTTP - # tools-api, a queue, a mock upstream -- which then leaves the world with no endpoint to - # forward to, so every tool call comes back "no such tool". So prefer the agent's own shipped - # Compose whenever it ships one (its real tool services come up and the world forwards to - # them), and fall back to the generated adapter only for agents that ship no usable Compose. + # Prefer the agent's own Compose: the generated adapter models only agent plus datastore, so + # any other service its tools are served by is dropped and every tool call fails. if compose is None: shipped = compose_file(source_root) if shipped is not None: @@ -2239,11 +2234,8 @@ def _runtime_credential_mounts( # ALK-owned destination that cannot collide with submitted mounts. target = f"/run/harness-secrets/{runtime.name}" if source is None or not _valid_google_credentials(source): - # A local sandbox supplies the credential through its own environment rather than - # an uploaded runtime configuration. That credential belongs to whoever is running - # the harness, not to the submitted agent, so it is only ever handed over when the - # operator says so: without this opt-in a hosted runner would mount its own - # platform key into a container it does not trust. + # The runner's own credential, handed over only when the operator opts in, so a + # hosted runner never mounts its platform key into a submitted agent. if os.environ.get("ALK_ALLOW_HOST_GOOGLE_CREDENTIALS", "").strip() == "1": env_value = os.environ.get(name, "").strip() env_path = Path(env_value).expanduser() if env_value else None @@ -2381,10 +2373,8 @@ def start_runtime( arguments.extend(("--volume", f"{source}:{target}:ro")) injected[name] = target mounted_credentials.add(name) - # A submitted service that declares its own credential volume has already been mounted (the - # host secret now lives at the container target). Re-validating that container path as a host - # file below would always fail, so only resolve GOOGLE_APPLICATION_CREDENTIALS here when it - # arrived as an injected host path (the generated-runtime path, which mounts no credentials). + # Only resolve an injected host path here. A credential the submitted service mounted itself + # already points at a container path, which would never validate as a host file. google_path = injected.get("GOOGLE_APPLICATION_CREDENTIALS", "").strip() if google_path and "GOOGLE_APPLICATION_CREDENTIALS" not in mounted_credentials: google_source = Path(google_path).expanduser() diff --git a/src/fi/alk/harness/run/live.py b/src/fi/alk/harness/run/live.py index e6873e64..a92aa132 100644 --- a/src/fi/alk/harness/run/live.py +++ b/src/fi/alk/harness/run/live.py @@ -278,11 +278,8 @@ def wire( "TOOLS_API_URL": url, "LIVEKIT_AGENT_NAME": agent_name, } - # The harness-generated agent-runtime does not carry the agent's own .env.local, and - # runtime_configuration_names only covers datastore config -- not the provider - # credentials the worker needs to actually run: LiveKit to register and place the - # call, Deepgram for STT/TTS, Vertex for the LLM. Pass them through from this - # process's environment (the sandbox has placed them here). + # The generated runtime carries no .env.local, and the runtime configuration covers + # datastore settings only, so the provider credentials it needs come from here. for _cred in ( "LIVEKIT_URL", "LIVEKIT_API_KEY", @@ -296,18 +293,13 @@ def wire( _value = os.environ.get(_cred, "").strip() if _value: runtime_overrides.setdefault(_cred, _value) - # An agent's Compose commonly mounts its Vertex credential from an env-var source, e.g. - # ${VERTEX_CREDENTIALS:-/dev/null}:/etc/vertex/creds.json. With that variable unset the - # placeholder is mounted and the agent's own LLM cannot authenticate, so point it at the - # same resolved Google credential the harness already holds for the run. + # An agent's Compose often mounts its Vertex credential from this variable, and with it + # unset a placeholder is mounted and the agent's own LLM cannot authenticate. _google_creds = runtime_overrides.get("GOOGLE_APPLICATION_CREDENTIALS", "").strip() if _google_creds: runtime_overrides.setdefault("VERTEX_CREDENTIALS", _google_creds) - # Voice agents often gate an audio-enhancement plugin on a license the harness cannot - # supply (e.g. ai_coustics noise cancellation). Unauthorized, it raises on the first - # inbound audio frame and the worker drops the call right after its greeting. Agents - # that read this flag skip that plugin; agents that do not simply ignore it. A run may - # override it to keep enhancement on when a license is present. + # Audio enhancement is usually licensed, and unauthorized it raises on the first + # inbound frame and drops the call. Agents that do not read this flag ignore it. runtime_overrides.setdefault("DISABLE_AI_COUSTICS", "1") os.environ["LIVEKIT_TARGET_AGENT_NAME"] = agent_name caller_phone = fixture_phone(scenario)