From 82ced34956620c009012ab919911a67e04cf7e38 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B3n=20Levy?= Date: Mon, 12 Oct 2026 16:45:40 +0000 Subject: [PATCH 1/9] Sync upstream main into fix/nsjail-no-cgroup-clone (resolved rebase) --- .github/scripts/image-revision.sh | 35 + .github/workflows/ci.yml | 151 ++- .github/workflows/images.yml | 350 +++++++ README.md | 101 ++ api/Dockerfile | 43 +- api/README.md | 67 ++ api/package.json | 2 +- api/src/anonymous-pipes.c | 34 + api/src/api/v2-cleanup.test.ts | 189 ++++ api/src/api/v2-files.test.ts | 103 +- api/src/api/v2-session-binding.test.ts | 86 ++ api/src/api/v2.ts | 159 +-- api/src/cleanup-metrics.test.ts | 110 ++ api/src/cleanup-metrics.ts | 185 ++++ api/src/cleanup.integration.test.ts | 184 ++++ api/src/entrypoint.sh | 14 +- api/src/hosted-app.ts | 26 +- api/src/index.ts | 4 - api/src/job-cleanup-metrics.test.ts | 118 +++ api/src/job.ts | 89 +- api/src/nsjail.test.ts | 49 +- api/src/nsjail.ts | 73 +- api/src/pipe-process.test.ts | 89 ++ api/src/pipe-process.ts | 116 +++ api/src/session-checkpoint.ts | 2 +- api/src/session-inputs.test.ts | 1 + api/src/spec-guard.c | 28 +- api/src/supervisor-policy.c | 60 ++ api/src/tool-call-pipe-bridge.c | 131 +++ api/src/tool-call-pipe-broker.test.ts | 158 +++ api/src/tool-call-pipe-broker.ts | 172 ++++ apparmor/README.md | 6 +- build-packages.sh | 7 +- docker-compose.yaml | 19 +- docker/Dockerfile.package-init | 1 + docker/Dockerfile.worker-sandbox | 31 +- docker/nsjail-anonymous-control.patch | 82 ++ docker/package-init.sh | 4 +- .../_sandbox_multiprocessing.py | 47 + docker/python-runtime/_sandbox_pipe.py | 32 + docker/python-runtime/install.py | 41 + docs/SECURITY-MAINTENANCE.md | 194 ++++ docs/adr/001-stateful-code-environments.md | 7 +- docs/adr/0010-worktree-metadata-admission.md | 101 ++ docs/adr/0011-tool-call-pipes.md | 60 ++ docs/adr/0012-pipe-only-python-runtime.md | 58 ++ docs/adr/0013-supervisor-anonymous-pipes.md | 65 ++ docs/byom-worker-admission.md | 58 +- docs/lambda-microvm/README.md | 2 +- docs/openshell/README.md | 189 ++++ docs/remote-bridge/README.md | 144 ++- docs/remote-bridge/projects.md | 6 +- docs/remote-bridge/worker-runbook.md | 81 +- helm/codeapi/Chart.yaml | 2 +- helm/codeapi/README.md | 37 + helm/codeapi/templates/network-policy.yaml | 10 + helm/codeapi/templates/package-init-job.yaml | 3 +- helm/codeapi/templates/podmonitor.yaml | 22 + helm/codeapi/values.yaml | 10 +- launcher/Dockerfile | 28 +- launcher/entrypoint.sh | 167 ++- launcher/src/main.rs | 46 +- packages/code/README.md | 467 ++++++++- packages/code/WORKSPACE-READS.md | 32 + packages/code/package-lock.json | 2 +- packages/code/package.json | 6 +- packages/code/src/cli.test.ts | 29 + packages/code/src/cli.ts | 344 ++++++- .../code/src/dependency-snapshots.test.ts | 284 ++++++ packages/code/src/dependency-snapshots.ts | 538 ++++++++++ packages/code/src/edits.test.ts | 964 ++++++++++++++++++ packages/code/src/edits.ts | 914 +++++++++++++++++ .../code/src/environment-preparation.test.ts | 365 +++++++ packages/code/src/environment-preparation.ts | 242 +++++ .../code/src/environment-resources.test.ts | 210 ++++ packages/code/src/environment-resources.ts | 164 +++ packages/code/src/environment.ts | 51 +- packages/code/src/github.test.ts | 266 +++++ packages/code/src/github.ts | 159 ++- packages/code/src/identity.test.ts | 43 + packages/code/src/identity.ts | 94 ++ packages/code/src/lane-git.test.ts | 830 +++++++++++++++ packages/code/src/lane-git.ts | 297 ++++++ .../src/linked-worktree-git-guard.test.ts | 289 ++++++ .../code/src/linked-worktree-git-guard.ts | 129 +++ .../code/src/linked-worktrees-live.test.ts | 108 ++ packages/code/src/linked-worktrees.test.ts | 347 +++++++ packages/code/src/linked-worktrees.ts | 534 ++++++++++ packages/code/src/native-process.test.ts | 16 + packages/code/src/native-process.ts | 4 + packages/code/src/native-sandbox.test.ts | 417 +++++++- packages/code/src/native-sandbox.ts | 401 ++++++-- packages/code/src/output.test.ts | 353 +++++++ packages/code/src/output.ts | 182 ++++ packages/code/src/process-lock.test.ts | 51 +- packages/code/src/process-lock.ts | 11 +- packages/code/src/projects.ts | 14 +- packages/code/src/protocol.test.ts | 232 +++++ packages/code/src/protocol.ts | 298 +++++- packages/code/src/root-access.ts | 79 +- .../code/src/root-git-denies-live.test.ts | 140 +++ packages/code/src/snapshot-lifecycle.test.ts | 274 +++++ packages/code/src/snapshot-lifecycle.ts | 324 ++++++ packages/code/src/storage.ts | 46 +- packages/code/src/worker-slots.test.ts | 201 ++++ packages/code/src/worker.ts | 273 ++++- packages/code/src/workspace-cli.test.ts | 41 + packages/code/src/workspace-read-io.test.ts | 147 +++ packages/code/src/workspace-read.test.ts | 488 +++++++++ packages/code/src/workspace-worker.test.ts | 242 +++++ packages/code/src/workspace.test.ts | 621 ++++++++++- packages/code/src/workspace.ts | 551 ++++++++-- packages/code/src/worktree-retirement.test.ts | 592 +++++++++++ packages/code/src/worktree-retirement.ts | 714 +++++++++++++ scripts/openshell/probe-policy.yaml | 11 + scripts/openshell/probe.mjs | 217 ++++ scripts/openshell/probe.test.mjs | 273 +++++ service/bun.lock | 102 +- service/openapi.yml | 105 ++ service/package.json | 10 +- service/scripts/dump-pipe-preamble.ts | 6 + service/src/bridge/admission.ts | 55 + service/src/bridge/concurrent-store.test.ts | 45 + service/src/bridge/index.ts | 19 +- service/src/bridge/lane-git.test.ts | 251 +++++ service/src/bridge/lane-git.ts | 36 + service/src/bridge/linked-worktree.test.ts | 197 ++++ service/src/bridge/pairing.ts | 480 ++++++++- service/src/bridge/recovery-router.test.ts | 213 ++++ service/src/bridge/recovery.test.ts | 518 ++++++++++ service/src/bridge/router.test.ts | 56 +- service/src/bridge/router.ts | 117 ++- service/src/bridge/slots.test.ts | 62 +- service/src/bridge/slots.ts | 60 +- service/src/bridge/store.ts | 477 +++++++-- service/src/bridge/worker-admission.test.ts | 92 ++ service/src/bridge/workspace-store.test.ts | 128 ++- service/src/config.spec.ts | 6 +- service/src/config.ts | 25 +- service/src/egress-gateway-client.test.ts | 3 + service/src/egress-gateway-client.ts | 4 + service/src/egress-gateway.test.ts | 37 + service/src/egress-gateway.ts | 15 +- service/src/file-server.ts | 92 +- service/src/hosted-app/microvm-runtime.ts | 3 + service/src/internal-redirect-policy.test.ts | 56 + service/src/minio-client.ts | 24 + service/src/minio-compat.test.ts | 39 + service/src/preamble.test.ts | 79 +- service/src/preamble.ts | 188 ++-- service/src/queue.ts | 2 +- service/src/runtime-session/checkpoint.ts | 4 + service/src/runtime-session/files.ts | 1 + service/src/sandbox-backend/http.test.ts | 24 +- service/src/sandbox-backend/http.ts | 1 + service/src/sandbox-backend/lambda-microvm.ts | 2 + service/src/service/programmatic-router.ts | 26 +- service/src/service/programmatic-state.ts | 21 +- service/src/service/replay-state.ts | 4 + service/src/service/router.ts | 7 +- service/src/types/service.ts | 4 + service/src/workers.ts | 2 +- service/src/workspace-tools/coordination.ts | 21 + service/src/workspace-tools/index.ts | 49 +- service/src/workspace-tools/outcome.ts | 4 +- .../workspace-tools/requests-router.test.ts | 96 ++ service/src/workspace-tools/requests.test.ts | 671 ++++++++++++ service/src/workspace-tools/requests.ts | 251 +++++ service/src/workspace-tools/router.test.ts | 74 +- service/src/workspace-tools/router.ts | 319 +++--- test-sandbox.sh | 24 +- tests/compose-bridge-config.cjs | 27 +- tests/kvm_guest_dns.sh | 115 ++- tests/nsjail_unix_gc.py | 152 +++ tests/package_init_runtime_rollout.sh | 31 + tests/pipe_relay_backpressure.py | 55 + tests/pipe_runtime_image.cjs | 61 ++ tests/python_pipe_runtime.py | 47 + tests/sandbox_runner_metrics.sh | 147 +++ tests/specguard_tool_call_pipes.py | 45 + tests/supervisor_dns.py | 49 + tests/supervisor_hosted_network.ts | 42 + tests/supervisor_policy.py | 53 + tests/tool_call_pipes.py | 171 ++++ tests/tool_call_runner.cjs | 131 +++ tests/worktree-metadata-phases.mjs | 23 + 186 files changed, 24714 insertions(+), 1052 deletions(-) create mode 100644 .github/scripts/image-revision.sh create mode 100644 .github/workflows/images.yml create mode 100644 api/src/anonymous-pipes.c create mode 100644 api/src/api/v2-cleanup.test.ts create mode 100644 api/src/cleanup-metrics.test.ts create mode 100644 api/src/cleanup-metrics.ts create mode 100644 api/src/cleanup.integration.test.ts mode change 100755 => 100644 api/src/entrypoint.sh create mode 100644 api/src/job-cleanup-metrics.test.ts create mode 100644 api/src/pipe-process.test.ts create mode 100644 api/src/pipe-process.ts create mode 100644 api/src/supervisor-policy.c create mode 100644 api/src/tool-call-pipe-bridge.c create mode 100644 api/src/tool-call-pipe-broker.test.ts create mode 100644 api/src/tool-call-pipe-broker.ts mode change 100755 => 100644 build-packages.sh create mode 100644 docker/nsjail-anonymous-control.patch create mode 100644 docker/python-runtime/_sandbox_multiprocessing.py create mode 100644 docker/python-runtime/_sandbox_pipe.py create mode 100644 docker/python-runtime/install.py create mode 100644 docs/SECURITY-MAINTENANCE.md create mode 100644 docs/adr/0010-worktree-metadata-admission.md create mode 100644 docs/adr/0011-tool-call-pipes.md create mode 100644 docs/adr/0012-pipe-only-python-runtime.md create mode 100644 docs/adr/0013-supervisor-anonymous-pipes.md create mode 100644 docs/openshell/README.md create mode 100644 packages/code/WORKSPACE-READS.md create mode 100644 packages/code/src/dependency-snapshots.test.ts create mode 100644 packages/code/src/dependency-snapshots.ts create mode 100644 packages/code/src/edits.test.ts create mode 100644 packages/code/src/edits.ts create mode 100644 packages/code/src/environment-preparation.test.ts create mode 100644 packages/code/src/environment-preparation.ts create mode 100644 packages/code/src/environment-resources.test.ts create mode 100644 packages/code/src/environment-resources.ts create mode 100644 packages/code/src/lane-git.test.ts create mode 100644 packages/code/src/lane-git.ts create mode 100644 packages/code/src/linked-worktree-git-guard.test.ts create mode 100644 packages/code/src/linked-worktree-git-guard.ts create mode 100644 packages/code/src/linked-worktrees-live.test.ts create mode 100644 packages/code/src/linked-worktrees.test.ts create mode 100644 packages/code/src/linked-worktrees.ts create mode 100644 packages/code/src/output.test.ts create mode 100644 packages/code/src/output.ts create mode 100644 packages/code/src/root-git-denies-live.test.ts create mode 100644 packages/code/src/snapshot-lifecycle.test.ts create mode 100644 packages/code/src/snapshot-lifecycle.ts create mode 100644 packages/code/src/workspace-read-io.test.ts create mode 100644 packages/code/src/workspace-read.test.ts create mode 100644 packages/code/src/worktree-retirement.test.ts create mode 100644 packages/code/src/worktree-retirement.ts create mode 100644 scripts/openshell/probe-policy.yaml create mode 100644 scripts/openshell/probe.mjs create mode 100644 scripts/openshell/probe.test.mjs create mode 100644 service/scripts/dump-pipe-preamble.ts create mode 100644 service/src/bridge/lane-git.test.ts create mode 100644 service/src/bridge/lane-git.ts create mode 100644 service/src/bridge/linked-worktree.test.ts create mode 100644 service/src/bridge/recovery-router.test.ts create mode 100644 service/src/bridge/recovery.test.ts create mode 100644 service/src/internal-redirect-policy.test.ts create mode 100644 service/src/minio-compat.test.ts create mode 100644 service/src/workspace-tools/coordination.ts create mode 100644 service/src/workspace-tools/requests-router.test.ts create mode 100644 service/src/workspace-tools/requests.test.ts create mode 100644 service/src/workspace-tools/requests.ts mode change 100755 => 100644 test-sandbox.sh mode change 100755 => 100644 tests/kvm_guest_dns.sh create mode 100644 tests/nsjail_unix_gc.py create mode 100644 tests/package_init_runtime_rollout.sh create mode 100644 tests/pipe_relay_backpressure.py create mode 100644 tests/pipe_runtime_image.cjs create mode 100644 tests/python_pipe_runtime.py create mode 100644 tests/sandbox_runner_metrics.sh create mode 100644 tests/specguard_tool_call_pipes.py create mode 100644 tests/supervisor_dns.py create mode 100644 tests/supervisor_hosted_network.ts create mode 100644 tests/supervisor_policy.py create mode 100644 tests/tool_call_pipes.py create mode 100644 tests/tool_call_runner.cjs create mode 100644 tests/worktree-metadata-phases.mjs diff --git a/.github/scripts/image-revision.sh b/.github/scripts/image-revision.sh new file mode 100644 index 00000000..ae4c88b4 --- /dev/null +++ b/.github/scripts/image-revision.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Prints the org.opencontainers.image.revision label of an image reference. +# Used by .github/workflows/images.yml, which must tell a missing tag apart +# from a registry that cannot answer. +# +# Exit 0 the revision was printed. +# Exit 3 the tag does not exist, or this token may not read it (a token that +# may not read a repository may not write to it either). +# Exit 1 the registry did not answer after retries, or the image has no +# revision label. +set -uo pipefail + +ref=${1:?usage: image-revision.sh REF} +attempts=${IMAGE_REVISION_ATTEMPTS:-3} +err=$(mktemp) +trap 'rm -f "$err"' EXIT + +for attempt in $(seq 1 "$attempts"); do + if json=$(docker buildx imagetools inspect "$ref" --format '{{ json .Image }}' 2>"$err"); then + if jq -er 'if has("config") then . else .["linux/amd64"] end + | .config.Labels["org.opencontainers.image.revision"]' <<<"$json"; then + exit 0 + fi + echo "$ref has no revision label" >&2 + exit 1 + fi + if grep -qiE ': not found$|denied|\b40[134]\b' "$err"; then + exit 3 + fi + cat "$err" >&2 + if [ "$attempt" -lt "$attempts" ]; then + sleep $(( attempt * 5 )) + fi +done +exit 1 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index b645f4b8..92e96ddb 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -31,18 +31,27 @@ jobs: - name: Block-root package delivery run: tests/block_root_package_delivery.sh + - name: Python runtime package-init rollout + run: bash tests/package_init_runtime_rollout.sh + - name: KVM guest resolver handoff run: tests/kvm_guest_dns.sh - name: Sandbox-runner liveness checks run: tests/sandbox_runner_healthcheck.sh + - name: Sandbox-runner metrics discovery and network policy + run: bash tests/sandbox_runner_metrics.sh + - name: Bridge pairing rollout safety run: tests/bridge_pairing_rollout.sh - name: Compose bridge configuration run: node tests/compose-bridge-config.cjs + - name: OpenShell evaluation probe (no gateway) + run: node --test scripts/openshell/probe.test.mjs + - name: Release versioning run: tests/release-versioning.sh @@ -53,6 +62,7 @@ jobs: run: | docker buildx build --check -f api/Dockerfile . docker buildx build --check -f docker/Dockerfile.worker-sandbox . + docker buildx build --check -f launcher/Dockerfile . launcher-unit-tests: name: Launcher Unit Tests @@ -83,19 +93,29 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.3.14 + bun-version: 1.4.2 - name: Cache Bun packages uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ~/.bun/install/cache - key: ${{ runner.os }}-codeapi-api-bun-1.3.14-${{ hashFiles('api/bun.lock') }} + key: ${{ runner.os }}-codeapi-api-bun-1.4.2-${{ hashFiles('api/bun.lock') }} restore-keys: | - ${{ runner.os }}-codeapi-api-bun-1.3.14- + ${{ runner.os }}-codeapi-api-bun-1.4.2- - name: Install dependencies run: bun ci + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: 22 + + - name: Compile anonymous-pipe addon + run: | + mkdir -p .build + NODE_HEADERS="$(dirname "$(dirname "$(command -v node)")")/include/node" + gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I"$NODE_HEADERS" -o .build/anonymous-pipes.node src/anonymous-pipes.c + - name: Build API run: bun run build @@ -121,6 +141,90 @@ jobs: - name: Bun tests run: bun run test + - name: File-heavy workspace cleanup on tmpfs + run: | + docker run --rm --user 0 \ + --tmpfs /tmp:rw,size=1g \ + --mount "type=bind,source=$GITHUB_WORKSPACE,target=/work,readonly" \ + --workdir /work/api \ + --env SANDBOX_CLEANUP_TMPFS_TEST=1 \ + --env SANDBOX_LOG_LEVEL=error \ + oven/bun:1.4.2-debian \ + bun test src/cleanup.integration.test.ts + + nsjail-ipc-tests: + name: NsJail IPC Filter (${{ matrix.runner }}) + runs-on: ${{ matrix.runner }} + timeout-minutes: 15 + strategy: + fail-fast: false + matrix: + runner: [ubuntu-latest, ubuntu-24.04-arm] + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 + with: + bun-version: 1.4.2 + - name: Render native sandbox policy + working-directory: api + run: | + bun ci + bun scripts/dump-seccomp-policy.ts > "$RUNNER_TEMP/nsjail.policy" + - name: Build pipe broker and generate Python client + run: | + bun run --cwd api build + bun ci --cwd service + bun service/scripts/dump-pipe-preamble.ts > "$RUNNER_TEMP/pipe-client.py" + - name: Build pinned NsJail and Kafel + run: docker build --target nsjail-ipc-test -t nsjail-ipc-test -f api/Dockerfile . + - name: Verify standalone launcher guest pipe runtime + run: | + docker build --target sandbox-build -t launcher-guest-test -f launcher/Dockerfile . + docker run --rm --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + --entrypoint /usr/local/bin/node \ + launcher-guest-test /tests/pipe_runtime_image.cjs + - name: Load legacy netfilter modules for hosted-app isolation + run: sudo modprobe -a ip_tables ip6_tables iptable_filter ip6table_filter xt_conntrack xt_owner ipt_REJECT ip6t_REJECT + - name: Verify supervisor filter and anonymous stdio + run: | + docker run --rm --init --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + nsjail-ipc-test python3 /tests/supervisor_policy.py + docker run --rm --init --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + nsjail-ipc-test python3 /tests/pipe_relay_backpressure.py + docker run --rm --init --dns 127.0.0.1 --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + nsjail-ipc-test python3 /tests/supervisor_dns.py + docker run --rm --init --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE,target=/work,readonly" \ + --entrypoint /usr/local/bin/sandbox-supervisor-policy \ + nsjail-ipc-test bun test /work/api/src/pipe-process.test.ts + docker run --rm --init --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + --entrypoint /usr/local/bin/sandbox-supervisor-policy \ + launcher-guest-test node /tests/pipe_runtime_image.cjs + docker run --rm --init --cap-add NET_ADMIN --security-opt seccomp=unconfined \ + --mount "type=bind,source=$GITHUB_WORKSPACE,target=/work,readonly" \ + --entrypoint /usr/local/bin/sandbox-supervisor-policy \ + launcher-guest-test bun /work/tests/supervisor_hosted_network.ts + + - name: Verify descriptor denial and runtime IPC with the installed filter + run: | + docker run --rm --init --security-opt seccomp=unconfined \ + --mount "type=bind,source=$RUNNER_TEMP/nsjail.policy,target=/policy,readonly" \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + nsjail-ipc-test \ + bash -c 'python3 /tests/nsjail_unix_gc.py /policy /nsjail/kafel/libkafel.so && python3 /tests/specguard_tool_call_pipes.py && gcc -O2 -static -Wall -Wextra -Werror -DSPEC_GUARD_TEST_DISABLE_CLOSE_RANGE -o /tmp/guard-fallback /tmp/spec-guard.c && SPEC_GUARD_BINARY=/tmp/guard-fallback python3 /tests/specguard_tool_call_pipes.py' + docker run --rm --init --security-opt seccomp=unconfined \ + --mount "type=bind,source=$RUNNER_TEMP/nsjail.policy,target=/policy,readonly" \ + --mount "type=bind,source=$RUNNER_TEMP/pipe-client.py,target=/client.py,readonly" \ + --mount "type=bind,source=$GITHUB_WORKSPACE/api/.build/tool-call-pipe-broker.cjs,target=/broker.cjs,readonly" \ + --mount "type=bind,source=$GITHUB_WORKSPACE/tests,target=/tests,readonly" \ + nsjail-ipc-test \ + python3 /tests/tool_call_pipes.py /policy /broker.cjs /client.py + service-unit-tests: name: Service Unit Tests runs-on: ubuntu-latest @@ -132,15 +236,15 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.3.14 + bun-version: 1.4.2 - name: Cache Bun packages uses: actions/cache@27d5ce7f107fe9357f9df03efb73ab90386fccae # v5 with: path: ~/.bun/install/cache - key: ${{ runner.os }}-codeapi-service-bun-1.3.14-${{ hashFiles('service/bun.lock') }} + key: ${{ runner.os }}-codeapi-service-bun-1.4.2-${{ hashFiles('service/bun.lock') }} restore-keys: | - ${{ runner.os }}-codeapi-service-bun-1.3.14- + ${{ runner.os }}-codeapi-service-bun-1.4.2- - name: Install dependencies run: bun ci @@ -153,7 +257,7 @@ jobs: redis_socket="$RUNNER_TEMP/byom-admission.sock" redis-server --port 0 --unixsocket "$redis_socket" --save '' --appendonly no --daemonize yes trap 'redis-cli -s "$redis_socket" shutdown nosave' EXIT - BRIDGE_TEST_REDIS_URL="$redis_socket" bun test src/bridge/fleet.test.ts + BRIDGE_TEST_REDIS_URL="$redis_socket" bun test src/bridge/fleet.test.ts src/workspace-tools/requests.test.ts - name: Build service run: bun run build @@ -167,7 +271,7 @@ jobs: strategy: fail-fast: false matrix: - node-version: ['20.11.0', '22.21.0', '24.16.0'] + node-version: ['22.21.0', '24.16.0'] defaults: run: working-directory: packages/code @@ -221,6 +325,33 @@ jobs: LIBRECHAT_CODE_LIVE_SRT_TESTS: '1' run: node --test dist/environment-live.test.js + linux-native-sandbox-tests: + name: Linux Native Sandbox Tests + runs-on: ubuntu-24.04 + defaults: + run: + working-directory: packages/code + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + - uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6 + with: + node-version: 24.16.0 + - name: Install bubblewrap + run: | + sudo apt-get update -qq + sudo apt-get install -y -qq bubblewrap socat ripgrep + # Ubuntu 24.04 blocks unprivileged user namespaces through AppArmor; bwrap needs them. + sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + - run: npm ci + - run: npm run build + - name: Linked worktree lane and root Git metadata containment + env: + LIBRECHAT_CODE_LIVE_SRT_TESTS: '1' + run: >- + node --test + dist/linked-worktrees-live.test.js + dist/root-git-denies-live.test.js + lambda-microvm-provisioning: name: Lambda MicroVM Provisioning runs-on: ubuntu-latest @@ -229,7 +360,7 @@ jobs: - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2 with: - bun-version: 1.3.14 + bun-version: 1.4.2 - name: Install service dependencies working-directory: service @@ -301,7 +432,7 @@ jobs: docker buildx inspect --bootstrap - name: Build Lambda MicroVM runner image - # The target pins Bun 1.3.14 and installs api/bun.lock frozen. Building + # The target pins Bun 1.4.2 and installs api/bun.lock frozen. Building # natively catches architecture-specific package and Docker layer # failures without retaining a second multi-gigabyte image copy. run: | diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml new file mode 100644 index 00000000..dbc42fd8 --- /dev/null +++ b/.github/workflows/images.yml @@ -0,0 +1,350 @@ +# Builds the Code API container images and publishes them to GHCR, so a host +# can pull a tested image instead of compiling one in place. Like ci.yml this +# file is inert inside the monorepo — GitHub only runs workflows from the repo +# root — and becomes a root workflow in the published repo. +# +# Entry points: +# +# * push to main — build every image, push `sha-` tags, and refresh +# each image's registry build cache. The promote job then points the +# `main` tags at the newest main commit that has all images. +# * pull_request — build every image without pushing, reading the cache that +# main wrote, so Dockerfile breakage surfaces before merge. +# * workflow_dispatch — on main, the same as a push; on any other branch, a +# build-only run. +# +# The image list lives in the plan job and feeds both the build matrix and the +# promotion, so the two cannot drift apart. README.md ("Prebuilt images") +# documents the tags and how a Compose host switches to them. +name: Images + +on: + push: + branches: [main] + # Everything the Dockerfiles below COPY from the build context, plus this + # workflow. Every run builds every image, so a commit that has tags has + # them for all images; unchanged images are cache hits. + paths: &image-inputs + - .dockerignore + - .github/scripts/image-revision.sh + - .github/workflows/images.yml + - api/** + - docker/** + - javascript-packages.txt + - launcher/** + - packages/code/src/** + - service/** + - shared/** + pull_request: + paths: *image-inputs + workflow_dispatch: + +permissions: + contents: read + +concurrency: + # Each main commit gets its own group and is never cancelled, so no commit's + # build can be dropped by a later queued run (GitHub keeps only one pending + # run per group). Other refs share one group per ref and cancel superseded + # runs. + group: images-${{ github.ref == 'refs/heads/main' && github.sha || github.ref }} + cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + +env: + NAMESPACE: ghcr.io/librechat-ai + +jobs: + plan: + name: Plan + runs-on: ubuntu-24.04 + outputs: + images: ${{ steps.images.outputs.images }} + publish: ${{ github.repository == 'LibreChat-AI/code-interpreter' && github.ref == 'refs/heads/main' }} + steps: + - name: List images + id: images + # One image per distinct Compose build in docker-compose.yaml. The + # sandbox runner's two Compose targets (KVM_ENABLED=true/false) are + # published as two images. The Lambda MicroVM targets are not built + # here; they are not used by Compose hosts. + run: | + images=$(jq -c . <<'JSON' + [ + {"image": "code-interpreter-api", "dockerfile": "service/Dockerfile", "target": "api", + "description": "Code API HTTP server (Compose service api)", "timeout": 30}, + {"image": "code-interpreter-worker", "dockerfile": "service/Dockerfile", "target": "worker", + "description": "Code API job worker (Compose service service-worker)", "timeout": 30}, + {"image": "code-interpreter-file-server", "dockerfile": "service/Dockerfile", "target": "production", + "description": "Code API file server (Compose service file_server)", "timeout": 30}, + {"image": "code-interpreter-egress-gateway", "dockerfile": "service/Dockerfile.egress-gateway", "target": "production", + "description": "Sandbox egress gateway (Compose service egress_gateway)", "timeout": 30}, + {"image": "code-interpreter-tool-call-server", "dockerfile": "service/Dockerfile.tool-call-server", "target": "production", + "description": "Tool call server (Compose service tool_call_server)", "timeout": 30}, + {"image": "code-interpreter-sandbox-runner", "dockerfile": "api/Dockerfile", "target": "sandbox-runner-true", + "description": "libkrun microVM sandbox runner with the guest rootfs and runtime packages baked in (Compose service sandbox-runner, KVM_ENABLED=true)", + "timeout": 60, "free_disk": true}, + {"image": "code-interpreter-sandbox-runner-direct", "dockerfile": "api/Dockerfile", "target": "sandbox-runner-false", + "description": "Directory-root sandbox runner; runtime packages are mounted from the host (Compose service sandbox-runner, KVM_ENABLED=false)", + "timeout": 30} + ] + JSON + ) + echo "images=$images" >> "$GITHUB_OUTPUT" + + build: + name: ${{ matrix.image }} + needs: plan + runs-on: ubuntu-24.04 + timeout-minutes: ${{ matrix.timeout }} + permissions: + contents: read + # Pushes happen only when needs.plan.outputs.publish is true (main). + # Pull request runs read the build cache and never push; GitHub caps a + # fork's token at read access regardless. + packages: write + strategy: + fail-fast: false + matrix: + include: ${{ fromJSON(needs.plan.outputs.images) }} + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + + - name: Free runner disk + # A cold KVM runner build peaks at roughly 17 GB of builder state. The + # hosted runner had about 87 GB free when measured, but GitHub only + # guarantees 14 GB, so this leg removes unused preinstalled SDKs + # (about 28 GB, 1.5 minutes) rather than depend on the surplus. + if: matrix.free_disk + run: | + sudo rm -rf \ + /opt/ghc \ + /opt/hostedtoolcache/CodeQL \ + /usr/local/.ghcup \ + /usr/local/lib/android \ + /usr/local/share/powershell \ + /usr/share/dotnet \ + /usr/share/swift + docker image prune --all --force >/dev/null + df -h / + + - name: Sample free disk + run: | + nohup bash -c 'while :; do df -BM --output=avail / | tail -n1 | tr -dc 0-9; echo; sleep 10; done' \ + >"$RUNNER_TEMP/disk-avail-mib.log" 2>/dev/null & + + - uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + - name: Log in to GHCR + # Pull request runs log in only to read the cache, and only from this + # repository; fork runs read public packages anonymously. + if: github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository + uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Check for an existing tag + # `sha-` tags are written once. A re-run (or a dispatch on the same + # commit) builds only the images whose tag is missing, so a failed + # re-run cannot leave the commit's set mixed across two builds. + id: existing + if: needs.plan.outputs.publish == 'true' + env: + REF: ${{ env.NAMESPACE }}/${{ matrix.image }}:sha-${{ github.sha }} + # A registry that cannot answer fails the job rather than risk + # rebuilding over a tag that may exist. + run: | + status=0 + revision=$(bash .github/scripts/image-revision.sh "$REF") || status=$? + case $status in + 0) + if [ "$revision" != "$GITHUB_SHA" ]; then + echo "::error::$REF records revision $revision" + exit 1 + fi + echo "$REF already exists; not rebuilding it." + echo "exists=true" >> "$GITHUB_OUTPUT" + ;; + 3) + echo "exists=false" >> "$GITHUB_OUTPUT" + ;; + *) + echo "::error::Could not read $REF from the registry." + exit 1 + ;; + esac + + - name: Image metadata + id: meta + if: steps.existing.outputs.exists != 'true' + uses: docker/metadata-action@dc802804100637a589fabce1cb79ff13a1411302 # v6.2.0 + env: + DOCKER_METADATA_ANNOTATIONS_LEVELS: manifest,index + with: + images: ${{ env.NAMESPACE }}/${{ matrix.image }} + tags: type=sha,format=long + labels: | + org.opencontainers.image.title=${{ matrix.image }} + org.opencontainers.image.description=${{ matrix.description }} + annotations: | + org.opencontainers.image.title=${{ matrix.image }} + org.opencontainers.image.description=${{ matrix.description }} + + - name: Build${{ needs.plan.outputs.publish == 'true' && ' and push' || '' }} + id: build + if: steps.existing.outputs.exists != 'true' + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + file: ${{ matrix.dockerfile }} + target: ${{ matrix.target }} + platforms: linux/amd64 + push: ${{ needs.plan.outputs.publish == 'true' }} + # Without a push, keep the result in the builder only: nothing is + # loaded into the runner's Docker engine, which would need a second + # copy of the multi-gigabyte sandbox image on disk. + outputs: ${{ needs.plan.outputs.publish != 'true' && 'type=cacheonly' || '' }} + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + annotations: ${{ steps.meta.outputs.annotations }} + # A registry cache next to each image, written only from main. + # mode=max keeps the intermediate stages (the KVM runner's package + # stage above all), which the GitHub Actions cache could not hold: + # its 10 GB repository limit would evict them between pushes. + cache-from: type=registry,ref=${{ env.NAMESPACE }}/${{ matrix.image }}:buildcache + cache-to: ${{ needs.plan.outputs.publish == 'true' && format('type=registry,ref={0}/{1}:buildcache,mode=max,ignore-error=true', env.NAMESPACE, matrix.image) || '' }} + + - name: Summarize + if: always() + env: + IMAGE_NAME: ${{ matrix.image }} + PUSHED: ${{ needs.plan.outputs.publish == 'true' && steps.build.outcome == 'success' }} + EXISTED: ${{ steps.existing.outputs.exists == 'true' }} + DIGEST: ${{ steps.build.outputs.digest }} + run: | + lowest=$(sort -n "$RUNNER_TEMP/disk-avail-mib.log" 2>/dev/null | head -n1) + { + echo "### $IMAGE_NAME" + if [ "$EXISTED" = true ]; then + echo "\`$NAMESPACE/$IMAGE_NAME:sha-$GITHUB_SHA\` was already published; left unchanged." + echo + fi + if [ "$PUSHED" = true ]; then + echo "Pushed \`$NAMESPACE/$IMAGE_NAME:sha-$GITHUB_SHA\`, pinnable as \`$NAMESPACE/$IMAGE_NAME@$DIGEST\`." + echo + fi + echo "Lowest free space on / while building: ${lowest:-unknown} MiB" + echo '```' + df -h / + docker buildx du 2>/dev/null | tail -n1 || true + echo '```' + } | tee -a "$GITHUB_STEP_SUMMARY" + + promote: + name: Move main tags + needs: [plan, build] + if: needs.plan.outputs.publish == 'true' + runs-on: ubuntu-24.04 + timeout-minutes: 15 + # Promotions run one at a time. Each resolves its target when it runs + # instead of trusting its own commit, so a stale re-run, or a queued + # promotion that GitHub replaces, cannot leave `main` behind or move it + # backwards. + concurrency: + group: images-promote + cancel-in-progress: false + permissions: + contents: read + packages: write + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6 + with: + persist-credentials: false + sparse-checkout: .github/scripts + + - uses: docker/login-action@dbcb813823bdd20940b903addbd779551569679f # v4.6.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ github.token }} + + - name: Point main at the newest complete commit + # Target: the newest of the last 100 commits on main for which every + # image has a `sha-` tag. No image's `main` moves to an older commit + # than the one it names, and a registry that cannot answer fails the + # promotion instead of being read as a missing tag, so an error can + # delay a promotion but never reverse one. A registry cannot move + # several repositories' tags in one transaction, so a failure part-way + # leaves some `main` tags behind; the next promotion, or a re-run of + # this job, finishes the move. Deployments should pin `sha-` tags + # rather than `main`. + env: + GH_TOKEN: ${{ github.token }} + IMAGES: ${{ needs.plan.outputs.images }} + run: | + set -euo pipefail + + # Prints the revision a tag records, or nothing if the tag does not + # exist; exits the step if the registry cannot answer. + revision_of() { + local revision status=0 + revision=$(bash .github/scripts/image-revision.sh "$1") || status=$? + case $status in + 0) echo "$revision" ;; + 3) ;; + *) echo "::error::Could not read $1 from the registry." >&2; exit 1 ;; + esac + } + + complete() { + local image revision + for image in "${images[@]}"; do + revision=$(revision_of "$NAMESPACE/$image:sha-$1") || exit 1 + [ "$revision" = "$1" ] || return 1 + done + } + + retag() { + local attempt + for attempt in 1 2 3; do + docker buildx imagetools create --tag "$1:main" "$1:sha-$2" && return 0 + sleep $(( attempt * 10 )) + done + return 1 + } + + mapfile -t images < <(jq -r '.[].image' <<<"$IMAGES") + mapfile -t commits < <(gh api "repos/$GITHUB_REPOSITORY/commits?sha=main&per_page=100" --jq '.[].sha') + declare -A position + for i in "${!commits[@]}"; do position[${commits[$i]}]=$i; done + + target='' + for sha in "${commits[@]}"; do + if complete "$sha"; then + target=$sha + break + fi + done + if [ -z "$target" ]; then + echo "::error::None of the last ${#commits[@]} commits on main has all images." + exit 1 + fi + + for image in "${images[@]}"; do + ref="$NAMESPACE/$image" + current=$(revision_of "$ref:main") || exit 1 + if [ "$current" = "$target" ]; then + echo "$ref:main already at sha-$target" | tee -a "$GITHUB_STEP_SUMMARY" + continue + fi + if [ -n "$current" ] && [ -n "${position[$current]:-}" ] && + [ "${position[$current]}" -lt "${position[$target]}" ]; then + echo "::warning::$ref:main names $current, newer than $target; leaving it." + continue + fi + retag "$ref" "$target" + echo "$ref:main -> sha-$target" | tee -a "$GITHUB_STEP_SUMMARY" + done diff --git a/README.md b/README.md index 3856b254..94ad3b9c 100644 --- a/README.md +++ b/README.md @@ -62,6 +62,12 @@ Omitting the expected-profile header remains supported for older clients, but provides no wrong-endpoint protection. There is deliberately no silent fallback between profiles and no automatic workspace or file migration. +## OpenShell evaluation + +An opt-in OpenShell backend is under [evaluation](docs/openshell/README.md). +The initial track provides a pinned lifecycle probe and security/compatibility +gates. It does not add a backend or change execution routing. + ## Sandbox Isolation Two modes are supported: @@ -122,6 +128,90 @@ Versions are `vMAJOR.MINOR.PATCH`, with `-rcN` release candidates published as pre-releases. See [docs/RELEASING.md](docs/RELEASING.md) for how releases are cut. +## Prebuilt images + +The [Images workflow](.github/workflows/images.yml) builds one `linux/amd64` +image per Compose build and publishes it to GHCR whenever a push to `main` +changes an image input: + +| Image (`ghcr.io/librechat-ai/…`) | Dockerfile (target) | Compose service | +|---|---|---| +| `code-interpreter-api` | `service/Dockerfile` (`api`) | `api` | +| `code-interpreter-worker` | `service/Dockerfile` (`worker`) | `service-worker` | +| `code-interpreter-file-server` | `service/Dockerfile` (`production`) | `file_server` | +| `code-interpreter-egress-gateway` | `service/Dockerfile.egress-gateway` (`production`) | `egress_gateway` | +| `code-interpreter-tool-call-server` | `service/Dockerfile.tool-call-server` (`production`) | `tool_call_server` | +| `code-interpreter-sandbox-runner` | `api/Dockerfile` (`sandbox-runner-true`) | `sandbox-runner`, `KVM_ENABLED=true` (default) | +| `code-interpreter-sandbox-runner-direct` | `api/Dockerfile` (`sandbox-runner-false`) | `sandbox-runner`, `KVM_ENABLED=false` | + +Tags: + +- `sha-` for every commit on `main` that changes an image + input. Commits that change none (docs, Helm, tests) get no tags, so take the + SHA from a successful Images run or the package page. +- `main`, pointed at the newest commit on `main` that has all seven images. + The seven tags move one after another, so pin `sha-` tags for deployments + and treat `main` as a convenience. + +There is no `latest`. `sha-` tags are written once: re-running the workflow +for a commit builds only the images whose tag is missing. For exact bytes, +replace a service's +`image:` below with `ghcr.io/librechat-ai/@sha256:`; each image +has its own digest, listed in the summary of the run that pushed it. + +Pulling a public package needs no login. A private package needs +`docker login ghcr.io` with a token that has `read:packages`; GHCR creates new +packages as private until an organization owner changes their visibility. + +To switch a Compose host from building to pulling, save this override as +`docker-compose.images.yml` (`!reset` needs Docker Compose 2.24.4 or later): + +```yaml +services: + api: + build: !reset null + image: ghcr.io/librechat-ai/code-interpreter-api:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing + service-worker: + build: !reset null + image: ghcr.io/librechat-ai/code-interpreter-worker:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing + file_server: + build: !reset null + image: ghcr.io/librechat-ai/code-interpreter-file-server:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing + egress_gateway: + build: !reset null + image: ghcr.io/librechat-ai/code-interpreter-egress-gateway:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing + tool_call_server: + build: !reset null + image: ghcr.io/librechat-ai/code-interpreter-tool-call-server:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing + sandbox-runner: + build: !reset null + # With KVM_ENABLED=false use code-interpreter-sandbox-runner-direct, which + # still reads runtime packages from SANDBOX_PACKAGES_PATH. + image: ghcr.io/librechat-ai/code-interpreter-sandbox-runner:${CODEAPI_IMAGE_TAG:?set CODEAPI_IMAGE_TAG} + pull_policy: missing +``` + +Then pin a built commit and start the stack, listing any host override after +it. Keep the checkout at the same commit as the images, so the Compose +configuration matches what the images expect. `docker-compose.mac.yml` runs +the unpublished `sandbox-build` target with its own entrypoint, so on macOS +leave `sandbox-runner` out of this override and keep building it: + +```bash +export CODEAPI_IMAGE_TAG=sha- +docker compose -f docker-compose.yaml -f docker-compose.images.yml pull +docker compose -f docker-compose.yaml -f docker-compose.images.yml up -d +``` + +`pull_policy: missing` never re-pulls a tag the host already has, which suits +an immutable `sha-` tag. To track `main` instead, set `CODEAPI_IMAGE_TAG=main` +and run `pull` before every `up`. + ## Local Development Copy `.env.example` to `.env` and set `CODEAPI_BRIDGE_TOKEN` to a private value @@ -155,6 +245,17 @@ read-only root disk does not need modification at boot. Rebuild the runner image to pick up this layout change. A missing resolver handoff fails startup rather than leaving the guest with an unrelated public DNS server. +The guest kernel keeps loopback traffic on its own loopback device instead of +proxying it through TSI, so it cannot reach a loopback resolver such as Docker's +embedded `127.0.0.11`. The launcher entrypoint relays the first loopback +nameserver (`127.0.0.0/8`, `::1` or IPv4-mapped `127.0.0.0/8`, in any spelling +glibc accepts) over UDP and TCP port 53 from the runner's +own IPv4 address with `socat`, and forwards that address to the guest in its +place; further loopback nameservers are dropped, and startup fails if the relay +cannot listen. Relay children exit when idle, and the TCP relay caps its +concurrent children. Routable nameservers, including Kubernetes cluster DNS, are forwarded +unchanged. + libkrun delivers the guest environment on the kernel command line, which only carries single-line printable ASCII and is capped at 2048 bytes by the guest kernel. The launcher entrypoint therefore forwards only the `nameserver`, diff --git a/api/Dockerfile b/api/Dockerfile index a8f3682e..ddf09cf8 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -26,14 +26,32 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ && git checkout 66ad78dc361721512fbf9470488e584364c75489 \ && git submodule update --init --recursive -RUN make -j$(nproc) +COPY docker/nsjail-anonymous-control.patch /tmp/nsjail-anonymous-control.patch +# Network-isolated jobs need no libnl transport; removing it avoids sendmsg +# in the supervisor and fails closed for interface/traffic-rule configuration. +RUN git apply /tmp/nsjail-anonymous-control.patch && make -j$(nproc) NL3_EXISTS=no COPY api/src/spec-guard.c /tmp/spec-guard.c +COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c +COPY api/src/anonymous-pipes.c /tmp/anonymous-pipes.c +COPY api/src/supervisor-policy.c /tmp/supervisor-policy.c +COPY --from=node:22-slim /usr/local/include/node /usr/local/include/node COPY docker/rootfs-setup.c /tmp/rootfs-setup.c -RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ +RUN gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I/usr/local/include/node -o /usr/local/lib/anonymous-pipes.node /tmp/anonymous-pipes.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/sandbox-supervisor-policy /tmp/supervisor-policy.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ + && gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup +# Native policy/transport verification without building the full runner. +FROM nsjail-builder AS nsjail-ipc-test +ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node +COPY docker/python-runtime /python-runtime +RUN python3 /python-runtime/install.py +COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node +COPY --from=oven/bun:1.4.2-debian /usr/local/bin/bun /usr/local/bin/bun + # ============================================================================ # Stage 1a: Fetch the pandoc binary (apro-pdf skill needs it on PATH; apt's # pandoc is stuck on an ancient 2.x, and apt/dpkg are deleted from the final @@ -68,7 +86,7 @@ FROM buildpack-deps:bookworm AS package-builder ENV DEBIAN_FRONTEND=noninteractive ENV PYTHON_VERSION=3.14.4 -ENV BUN_VERSION=1.3.14 +ENV BUN_VERSION=1.4.2 COPY docker/apt-install.sh /usr/local/bin/apt-install RUN chmod +x /usr/local/bin/apt-install @@ -83,6 +101,7 @@ RUN apt-install \ RUN mkdir -p /pkgs +COPY docker/python-runtime /python-runtime COPY docker/package-init.sh /package-init.sh COPY javascript-packages.txt /javascript-packages.txt RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh @@ -90,7 +109,7 @@ RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh # ============================================================================ # Stage 2: Build sandbox rootfs # ============================================================================ -FROM oven/bun:1.3.14-debian AS sandbox-build +FROM oven/bun:1.4.2-debian AS sandbox-build ENV DEBIAN_FRONTEND=noninteractive @@ -154,18 +173,22 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard +COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/tool-call-pipe-bridge +COPY --from=nsjail-builder /usr/local/bin/sandbox-supervisor-policy /usr/local/bin/sandbox-supervisor-policy +COPY --from=nsjail-builder /usr/local/lib/anonymous-pipes.node /usr/local/lib/anonymous-pipes.node +ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node RUN chmod +x /usr/sbin/nsjail # pandoc for the apro-pdf skill (document conversion); fetched in the # pandoc-fetch stage above since apt/bookworm only ships an ancient 2.x. COPY --from=pandoc-fetch /pandoc /usr/local/bin/pandoc -# The tool-call socket proxy must run under Node, not Bun. Bun's node:http -# compat layer never fires `'connection'` events and Bun.serve's idleTimeout -# does not close silent unix-socket connections — so our DoS protections -# (maxConnections, idleSocketTimeoutMs, accept-time tracking) silently no-op -# under Bun and a `for i in range(N): socket.connect` storm pins FDs in the -# proxy until the OS limit. Pulling just the Node binary from a slim image +# The tool-call socket proxy runs under Node, not Bun: Bun's node:http +# compatibility layer does not fire connection events and its idle timeout +# does not close silent sockets, so the proxy's connection and idle limits +# would not apply. The per-invocation broker also starts after MicroVM restore, +# so Node's OpenSSL state is not captured in the snapshot. Pulling just the +# Node binary from node:22-slim is about 30 MB and avoids npm/node_modules. # is ~30 MB and avoids inheriting npm/node_modules from a full apt install. COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node diff --git a/api/README.md b/api/README.md index 38e4cd98..34b32145 100644 --- a/api/README.md +++ b/api/README.md @@ -139,3 +139,70 @@ validation behavior. Roll out this sandbox behavior before enabling timeout forwarding in the service's plain `/exec` handler. Older sandboxes reject caps above their local runtime limit; older services remain compatible with updated sandboxes. + +### Runner memory after cleanup + +`Job.cleanup()` emits `Post-cleanup resources` after workspace removal and UID +release (or quarantine). `/execute` now waits for cleanup before sending success +or execution-failure responses. Artifact upload still finishes before cleanup. +Persistent session workspaces and their pinned UIDs are intentionally preserved; +failed disposable cleanup still quarantines the directory and retains its UID +for the existing retry path. This change does not alter those policies. + +The following metrics use only fixed, low-cardinality labels: + +| Metric suffix (prefix `codeapi_sandbox_`) | Meaning | +| --- | --- | +| `post_cleanup_memory_bytes{kind}` | `current`, `anon`, `file`, `shmem` at the visible cgroup v2 mount root | +| `post_cleanup_tmp_used{resource}` | `/tmp` allocated `bytes`, allocated `inodes`, and `is_tmpfs` (0 or 1) | +| `post_cleanup_workspaces{kind}` | Remaining `disposable`, `session`, and `other` entries in `/tmp/sandbox` | +| `cleanup_total{mode,outcome}` | Disposable/session cleanup attempts: `removed`, `preserved`, `retained`, `error` | +| `post_cleanup_sample_success{source}` | Whether `memory`, `tmp`, or `workspaces` was readable on the last sample | +| `post_cleanup_timestamp_seconds` | When the last cleanup sample was taken | + +These are runner-wide **last-cleanup samples**, not live gauges or per-job +memory attribution. Other jobs can still be running. Reaper-only changes are +visible on the next job cleanup, not immediately. Interpret the workspace counts +alongside active executions and the sample timestamp. Unavailable sources are +reported as unavailable and their old gauge values are removed, never replaced +with a healthy-looking zero. The structured log also includes active UID slots +and the number of retained cleanup retries. Synthetic jobs update metrics while +suppressing successful per-job logs as before. + +The visible cgroup root covers the API and sibling NsJail cgroups. The old +`Post-execution memory` log instead samples `/proc/self/cgroup` **before** job +cleanup, so it can have a different scope. `file` includes `shmem`; do not add +them or assume all `file` memory is reclaimable page cache. + +`statfs` measures allocation, including deleted-but-open files on the sampled +mount, without walking user files. The runner's 1 GiB `/tmp` mount is distinct +from each jail's 20 MiB `/tmp` mount. Its ceiling does not cap all container +memory. If workspace counts fall but shmem does not, investigate descendant +processes and retained mounts as well as files. A stable API-process FD count +alone cannot exclude those cases. Memory requests and HPA settings are unchanged. + +#### Cleanup stress tests + +Ordinary unit tests cover metrics, unavailable sources, session/disposable +classification and response ordering. The opt-in stress tests must run in an +**isolated test container**, never on an active runner: + +```bash +# From api/, with /tmp mounted as tmpfs and per-job chown available: +SANDBOX_CLEANUP_TMPFS_TEST=1 bun test src/cleanup.integration.test.ts + +# Additionally requires the runner's NsJail binary, spec-guard, config, +# and normal namespace/cgroup permissions. Run this file alone. +NSJAIL_CONFIG=/sandbox_api/config/sandbox.cfg \ +SANDBOX_CLEANUP_NSJAIL_TEST=1 bun test src/cleanup.integration.test.ts +``` + +The first test repeats real Job priming and cleanup with large payloads and many +small files, checking tmpfs bytes/inodes and UID slots after every iteration. +CI runs it in a dedicated Bun container with a 1 GiB `/tmp` tmpfs. It does not +execute NsJail. The second test uses real NsJail jobs covering normal completion, +timeout and output overflow, including a detached child holding an unlinked file. +It asserts no processes remain under the job UID before removing its workspace, +then checks workspace removal and post-cleanup tmpfs allocation. This test is +skipped unless explicitly enabled; a unit-test pass is not proof of namespace +teardown on the production kernel. diff --git a/api/package.json b/api/package.json index de62fb55..a27fd780 100644 --- a/api/package.json +++ b/api/package.json @@ -5,7 +5,7 @@ "main": "src/index.ts", "scripts": { "dev": "bun run --watch src/index.ts", - "build": "bun build ./src/index.ts --minify --outdir .build --target bun --external '@opentelemetry/*' && bun build ./src/tool-call-socket-proxy.ts --target=node --format=cjs --outfile=.build/tool-call-socket-proxy.cjs", + "build": "bun build ./src/index.ts --minify --outdir .build --target bun --external '@opentelemetry/*' && bun build ./src/tool-call-pipe-broker.ts --target=node --format=cjs --outfile=.build/tool-call-pipe-broker.cjs", "start": "bun run .build/index.js", "test": "bun test" }, diff --git a/api/src/anonymous-pipes.c b/api/src/anonymous-pipes.c new file mode 100644 index 00000000..524c3444 --- /dev/null +++ b/api/src/anonymous-pipes.c @@ -0,0 +1,34 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include + +static napi_value create_pipe(napi_env env, napi_callback_info info) { + (void)info; + int fds[2]; + if (pipe2(fds, O_CLOEXEC) != 0) { + napi_throw_error(env, NULL, strerror(errno)); + return NULL; + } + napi_value array, reader, writer; + if (napi_create_array_with_length(env, 2, &array) != napi_ok + || napi_create_int32(env, fds[0], &reader) != napi_ok + || napi_create_int32(env, fds[1], &writer) != napi_ok + || napi_set_element(env, array, 0, reader) != napi_ok + || napi_set_element(env, array, 1, writer) != napi_ok) { + close(fds[0]); close(fds[1]); + napi_throw_error(env, NULL, "cannot return anonymous pipe"); + return NULL; + } + return array; +} + +static napi_value initialize(napi_env env, napi_value exports) { + napi_value fn; + if (napi_create_function(env, "createPipe", NAPI_AUTO_LENGTH, create_pipe, NULL, &fn) != napi_ok + || napi_set_named_property(env, exports, "createPipe", fn) != napi_ok) return NULL; + return exports; +} +NAPI_MODULE(NODE_GYP_MODULE_NAME, initialize) diff --git a/api/src/api/v2-cleanup.test.ts b/api/src/api/v2-cleanup.test.ts new file mode 100644 index 00000000..d1efee5d --- /dev/null +++ b/api/src/api/v2-cleanup.test.ts @@ -0,0 +1,189 @@ +import { afterAll, afterEach, beforeAll, expect, test } from 'bun:test'; +import express from 'express'; +import { mkdtemp, rm, writeFile } from 'fs/promises'; +import type { Server } from 'http'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { config } from '../config'; +import { Job } from '../job'; +import { loadPackage } from '../runtime'; +import { ValidationError } from '../validation'; +import router from './v2'; + +let server: Server; +let url: string; +let directory: string; +const language = 'cleanup-order-test'; +const originalPrime = Job.prototype.prime; +const originalExecute = Job.prototype.execute; +const originalUpload = Job.prototype.uploadGeneratedFiles; +const originalCleanup = Job.prototype.cleanup; +const requireManifest = config.require_execution_manifest; + +beforeAll(async () => { + directory = await mkdtemp(join(tmpdir(), 'cleanup-order-')); + await writeFile( + join(directory, 'pkg-info.json'), + JSON.stringify({ language, version: '1.0.0', aliases: [] }) + ); + loadPackage(directory); + const app = express(); + app.use(router); + await new Promise(resolve => { + server = app.listen(0, '127.0.0.1', () => resolve()); + }); + const address = server.address(); + url = `http://127.0.0.1:${ + typeof address === 'object' && address ? address.port : 0 + }/execute`; +}); +afterAll(async () => { + await new Promise(resolve => server.close(() => resolve())); + await rm(directory, { recursive: true, force: true }); +}); +afterEach(() => { + Job.prototype.prime = originalPrime; + Job.prototype.execute = originalExecute; + Job.prototype.uploadGeneratedFiles = originalUpload; + Job.prototype.cleanup = originalCleanup; + config.require_execution_manifest = requireManifest; +}); + +for (const outcome of [ + 'success', + 'prime_failure', + 'execution_failure', + 'validation_failure', +] as const) { + test(`${outcome} waits for cleanup before sending a response and cleans exactly once`, async () => { + config.require_execution_manifest = false; + const events: string[] = []; + let releaseCleanup!: () => void; + const cleanupGate = new Promise(resolve => { + releaseCleanup = resolve; + }); + let markCleanupStarted!: () => void; + const cleanupStarted = new Promise(resolve => { + markCleanupStarted = resolve; + }); + Job.prototype.prime = async function () { + events.push('prime'); + if (outcome === 'prime_failure') + throw new Error('scripted prime failure'); + }; + Job.prototype.execute = async function () { + events.push('execute'); + if (outcome === 'execution_failure') + throw new Error('scripted execution failure'); + if (outcome === 'validation_failure') + throw new ValidationError('scripted validation failure'); + return { files: [{ id: 'artifact', name: 'result.txt' }] } as Awaited< + ReturnType + >; + }; + Job.prototype.uploadGeneratedFiles = async function () { + events.push('upload'); + return new Set(['artifact']); + }; + Job.prototype.cleanup = async function () { + events.push('cleanup-start'); + markCleanupStarted(); + await cleanupGate; + events.push('cleanup-end'); + }; + let responseArrived = false; + const pending = fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + language, + version: '1.0.0', + files: [{ name: 'main.txt', content: 'test' }], + }), + }).then(response => { + responseArrived = true; + return response; + }); + try { + await Promise.race([ + cleanupStarted, + pending.then(response => { + throw new Error(`Response preceded cleanup: ${response.status}`); + }), + ]); + // Give an incorrectly early res.json() time to reach the client. + await new Promise(resolve => setTimeout(resolve, 30)); + expect(responseArrived).toBe(false); + } finally { + releaseCleanup(); + const response = await pending; + expect(response.status).toBe( + outcome === 'success' + ? 200 + : outcome === 'validation_failure' + ? 400 + : 500 + ); + await response.text(); + } + expect(events.filter(event => event === 'cleanup-start')).toHaveLength(1); + expect(events[events.length - 1]).toBe('cleanup-end'); + if (outcome === 'success') + expect(events).toEqual([ + 'prime', + 'execute', + 'upload', + 'cleanup-start', + 'cleanup-end', + ]); + }); +} + +test('client disconnect does not clean a workspace while execution is still running', async () => { + config.require_execution_manifest = false; + let releaseExecution!: () => void; + const executionGate = new Promise(resolve => { + releaseExecution = resolve; + }); + let markExecuting!: () => void; + const executing = new Promise(resolve => { + markExecuting = resolve; + }); + let markCleaned!: () => void; + const cleaned = new Promise(resolve => { + markCleaned = resolve; + }); + let cleanupCount = 0; + Job.prototype.prime = async function () {}; + Job.prototype.execute = async function () { + markExecuting(); + await executionGate; + return {} as Awaited>; + }; + Job.prototype.cleanup = async function () { + cleanupCount++; + markCleaned(); + }; + const controller = new AbortController(); + const pending = fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + signal: controller.signal, + body: JSON.stringify({ + language, + version: '1.0.0', + files: [{ name: 'main.txt', content: 'test' }], + }), + }).catch(() => undefined); + try { + await executing; + controller.abort(); + await pending; + await new Promise(resolve => setTimeout(resolve, 30)); + expect(cleanupCount).toBe(0); + } finally { + releaseExecution(); + await cleaned; + } + expect(cleanupCount).toBe(1); +}); diff --git a/api/src/api/v2-files.test.ts b/api/src/api/v2-files.test.ts index 540b529b..83918884 100644 --- a/api/src/api/v2-files.test.ts +++ b/api/src/api/v2-files.test.ts @@ -1,7 +1,8 @@ import { describe, expect, test } from 'bun:test'; import { config } from '../config'; +import { collectExecuteRequestInputFiles } from '../execution-manifest-request'; import type { TFile } from '../job'; -import { validateExecuteArguments, validateExecuteFiles } from './v2'; +import { validateExecuteArguments, validateExecuteFiles, deduplicateFilesByDestination } from './v2'; function messageOf(fn: () => void): string { try { @@ -85,4 +86,104 @@ describe('execute file validation', () => { ]; expect(() => validateExecuteFiles(files)).not.toThrow(); }); + + test('deduplicateFilesByDestination keeps the latest upload in the original destination order', () => { + const files: TFile[] = [ + { name: 'data.csv', id: 'first', storage_session_id: 'uploads' }, + { name: 'data.csv', id: 'second', storage_session_id: 'uploads' }, + { name: 'other.csv', id: 'unique', storage_session_id: 'uploads' }, + { name: 'data.csv', id: 'third', storage_session_id: 'latest-uploads' }, + ]; + const result = deduplicateFilesByDestination(files); + expect(result.map(file => file.name)).toEqual(['data.csv', 'other.csv']); + expect(result[0].id).toBe('third'); + expect(result[0].storage_session_id).toBe('latest-uploads'); + }); + + test('keeps a by-reference program in the entrypoint position when its upload is refreshed', () => { + const files: TFile[] = [ + { name: 'run.py', id: 'old-script', storage_session_id: 'uploads' }, + { name: 'data.csv', id: 'data', storage_session_id: 'uploads' }, + { name: 'run.py', id: 'new-script', storage_session_id: 'uploads' }, + ]; + const result = deduplicateFilesByDestination(files); + expect(result.map(file => file.name)).toEqual(['run.py', 'data.csv']); + expect(result[0].id).toBe('new-script'); + expect(() => validateExecuteFiles(result)).not.toThrow(); + }); + + test('rejects an upload that targets inline source, regardless of file order', () => { + const source: TFile = { name: 'main.py', content: 'print("submitted program")' }; + const upload: TFile = { name: 'main.py', id: 'uploaded-source', storage_session_id: 'uploads' }; + for (const files of [[source, upload], [upload, source]]) { + const message = messageOf(() => deduplicateFilesByDestination(files)); + expect(message).toContain('duplicate destination "main.py"'); + expect(message).toContain('inline content'); + } + expect(messageOf(() => deduplicateFilesByDestination([ + { content: 'unnamed source' } as TFile, + { name: 'file0.code', id: 'uploaded-source', storage_session_id: 'uploads' }, + ]))).toContain('duplicate destination "file0.code"'); + expect(messageOf(() => deduplicateFilesByDestination([ + source, { name: 'main.py', content: 'different source' }, + ]))).toContain('inline content'); + }); + + test('deduplicateFilesByDestination returns the same array when there are no duplicates', () => { + const files: TFile[] = [ + { name: 'a.csv', content: 'a' }, + { name: 'b.csv', content: 'b' }, + ]; + const result = deduplicateFilesByDestination(files); + expect(result).toEqual(files); + }); + + test('preserves the original destination of an unnamed file reference after deduplication', () => { + const files: TFile[] = [ + { name: 'main.py', content: 'print(1)' }, + { name: 'data.csv', id: 'old', storage_session_id: 'storage-session' }, + { name: 'data.csv', id: 'new', storage_session_id: 'storage-session' }, + { id: 'file-ref', storage_session_id: 'storage-session' } as TFile, + ]; + const deduped = deduplicateFilesByDestination(files); + expect(deduped.map(file => file.name)).toEqual(['main.py', 'data.csv', 'file3.code']); + expect(deduped[1].id).toBe('new'); + const signedDestination = collectExecuteRequestInputFiles({ files }) + .find(file => file.id === 'file-ref'); + expect(collectExecuteRequestInputFiles({ files: deduped }) + .find(file => file.id === 'file-ref')).toEqual(signedDestination); + expect(() => validateExecuteFiles(deduped)).not.toThrow(); + }); + + test('rejects malformed files even if another entry owns their destination', () => { + expect(messageOf(() => deduplicateFilesByDestination([ + { name: 'file1.code', content: 'source' }, + null as unknown as TFile, + ]))).toContain('files[1] must be an object'); + expect(messageOf(() => deduplicateFilesByDestination([ + { name: 'data.csv', content: 'old', encoding: 'invalid' as TFile['encoding'] }, + { name: 'data.csv', content: 'new' }, + ]))).toContain('files[0].encoding'); + expect(messageOf(() => deduplicateFilesByDestination([ + { name: 'data.csv', content: 'old' }, + { name: 'data.csv', id: 'file-ref' }, + ]))).toContain('files[1].storage_session_id'); + }); + + test('caps raw input count before dropping duplicates', () => { + const files = Array.from({ length: config.max_input_files + 1 }, () => ({ + name: 'data.csv', content: 'duplicate', + })); + expect(messageOf(() => deduplicateFilesByDestination(files))).toContain('cannot contain more than'); + }); + + test('deduplicateFilesByDestination allows validateExecuteFiles to accept duplicate uploads', () => { + const files: TFile[] = [ + { name: 'data.csv', id: 'first', storage_session_id: 'uploads' }, + { name: 'data.csv', id: 'second', storage_session_id: 'uploads' }, + ]; + const deduped = deduplicateFilesByDestination(files); + expect(deduped[0].id).toBe('second'); + expect(() => validateExecuteFiles(deduped)).not.toThrow(); + }); }); diff --git a/api/src/api/v2-session-binding.test.ts b/api/src/api/v2-session-binding.test.ts index 32a245f6..0987a68a 100644 --- a/api/src/api/v2-session-binding.test.ts +++ b/api/src/api/v2-session-binding.test.ts @@ -281,6 +281,92 @@ describe('per-request session binding', () => { } }); + test('uses the latest upload without renumbering later unnamed files', async () => { + config.session_workspace_enabled = false; + config.require_execution_manifest = false; + + const originalPrime = Job.prototype.prime; + const originalExecute = Job.prototype.execute; + const originalCleanup = Job.prototype.cleanup; + + let primedFiles: Array<{ name: string; id?: string; content?: string }> = []; + Job.prototype.prime = async function captureFiles(): Promise { + primedFiles = this.files; + }; + Job.prototype.execute = async function executeWithoutSandbox() { + return {} as Awaited>; + }; + Job.prototype.cleanup = async function cleanupWithoutFilesystem(): Promise {}; + + try { + const response = await fetch(`${baseUrl}/api/v2/execute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + language: testLanguage, + version: testVersion, + files: [ + { name: 'main.txt', content: 'source' }, + { name: 'data.csv', id: 'old-upload', storage_session_id: 'uploads' }, + { name: 'data.csv', id: 'new-upload', storage_session_id: 'uploads' }, + { content: 'unnamed source' }, + ], + }), + }); + + expect(response.status).toBe(200); + expect(primedFiles.map(file => file.name)).toEqual(['main.txt', 'data.csv', 'file3.code']); + expect(primedFiles[0].content).toBe('source'); + expect(primedFiles[1].id).toBe('new-upload'); + expect(primedFiles[2].content).toBe('unnamed source'); + } finally { + Job.prototype.prime = originalPrime; + Job.prototype.execute = originalExecute; + Job.prototype.cleanup = originalCleanup; + } + }); + + test('rejects an upload targeting the submitted source before binding or priming', async () => { + config.session_workspace_enabled = true; + config.require_execution_manifest = false; + + const originalPrime = Job.prototype.prime; + const originalExecute = Job.prototype.execute; + const originalCleanup = Job.prototype.cleanup; + let primed = false; + Job.prototype.prime = async function trackPrime(): Promise { primed = true; }; + Job.prototype.execute = async function executeWithoutSandbox() { + return {} as Awaited>; + }; + Job.prototype.cleanup = async function cleanupWithoutFilesystem(): Promise {}; + + try { + const response = await fetch(`${baseUrl}/api/v2/execute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-Runtime-Session-Id': 'rt_source_collision' }, + body: JSON.stringify({ + language: testLanguage, + version: testVersion, + files: [ + { name: 'main.txt', content: 'submitted program' }, + { name: 'data.csv', id: 'data', storage_session_id: 'uploads' }, + { name: 'main.txt', id: 'uploaded-program', storage_session_id: 'uploads' }, + ], + }), + }); + + expect(response.status).toBe(400); + expect((await response.json() as { message: string }).message) + .toContain('duplicate destination "main.txt"'); + expect(primed).toBe(false); + expect(getBoundSessionWorkspace()).toBeUndefined(); + } finally { + Job.prototype.prime = originalPrime; + Job.prototype.execute = originalExecute; + Job.prototype.cleanup = originalCleanup; + } + }); + test('a post-prime failure still reports the workspace as dirty', async () => { config.session_workspace_enabled = true; config.require_execution_manifest = false; diff --git a/api/src/api/v2.ts b/api/src/api/v2.ts index e2f252ce..cba8e80a 100644 --- a/api/src/api/v2.ts +++ b/api/src/api/v2.ts @@ -26,7 +26,6 @@ import { parseSessionBindingFromHeader, } from '../session-workspace'; import { streamSessionCheckpoint, restoreSessionCheckpoint } from '../session-checkpoint'; -import { ensureToolCallSocketProxyReady } from '../tool-call-socket-process'; import { SESSION_INPUT_CACHE_MAX_OBJECTS, hasCachedInput, @@ -46,72 +45,111 @@ import { const router = express.Router(); const SYNTHETIC_PRINCIPAL_SOURCE = 'synthetic_test'; +/** + * Deduplicate files by destination name. Some callers (e.g. LibreChat) + * send the same file more than once per request when a user re-uploads a + * file in the same conversation. The sandbox rejects duplicate destinations, + * which surfaces to the end user as a confusing "sandbox is down" error. + * Keeps the latest uploaded file at each destination without moving its + * original position (the first runnable file is the program entrypoint). + * Never replace inline source code with an uploaded file. + */ +export function deduplicateFilesByDestination(files: TFile[]): TFile[] { + if (files.length > config.max_input_files) { + throw { message: `files cannot contain more than ${config.max_input_files} destinations` }; + } + const byDestination = new Map(); + for (const [i, file] of files.entries()) { + // Validate even entries that would otherwise be replaced by a later upload. + const destination = validateExecuteFile(file, i); + const previous = byDestination.get(destination); + if (previous && (typeof previous.content === 'string' || typeof file.content === 'string')) { + throw { message: `files contains duplicate destination "${destination}" involving inline content` }; + } + // Updating a Map value preserves its first position for Job.execute's entrypoint. + // Manifest claims use original indices; Job would otherwise renumber unnamed files. + byDestination.set(destination, file.name ? file : { ...file, name: destination }); + } + if (byDestination.size < files.length) { + logger.warn( + { original: files.length, deduped: byDestination.size }, + 'Deduplicated file list before validation', + ); + } + return [...byDestination.values()]; +} + function existingDestinationConflictMessage(existing: string, destination: string): string { return existing === destination ? `files contains duplicate destination "${destination}"` : `files contains conflicting destinations "${existing}" and "${destination}"`; } +function validateExecuteFile(value: TFile, i: number): string { + if (value == null || typeof value !== 'object' || Array.isArray(value)) { + throw { message: `files[${i}] must be an object` }; + } + const file = value as TFile; + const inline = typeof file.content === 'string'; + const byRef = typeof file.id === 'string' && file.id.length > 0; + if (inline === byRef) { + throw { + message: `files[${i}] must contain exactly one of non-empty id or string content`, + }; + } + if (file.id !== undefined && !byRef) { + throw { message: `files[${i}].id must be a non-empty string if provided` }; + } + if (byRef) { + if (typeof file.storage_session_id !== 'string' || file.storage_session_id.length === 0) { + throw { message: `files[${i}].storage_session_id is required as a non-empty string for file refs` }; + } + } else if (file.storage_session_id !== undefined || file.input_cache_key !== undefined) { + throw { + message: `files[${i}] inline content cannot include storage_session_id or input_cache_key`, + }; + } + if (file.name !== undefined && typeof file.name !== 'string') { + throw { message: `files[${i}].name must be a string if provided` }; + } + if ( + file.encoding !== undefined + && !(['base64', 'hex', 'utf8'] as const).includes(file.encoding) + ) { + throw { message: `files[${i}].encoding must be base64, hex, or utf8 if provided` }; + } + if (file.entity_id !== undefined && typeof file.entity_id !== 'string') { + throw { message: `files[${i}].entity_id must be a string if provided` }; + } + if ( + file.input_cache_key !== undefined && + ( + typeof file.input_cache_key !== 'string' || + !/^[0-9a-f]{64}$/.test(file.input_cache_key) + ) + ) { + throw { message: `files[${i}].input_cache_key must be a 64-character lowercase hex digest` }; + } + const destination = file.name || `file${i}.code`; + try { + validateFilePath(destination, '/tmp/codeapi-request-validation'); + } catch (error) { + throw { + message: error instanceof Error + ? `files[${i}].name is invalid: ${error.message}` + : `files[${i}].name is invalid`, + }; + } + return destination; +} + export function validateExecuteFiles(files: TFile[]): void { if (files.length > config.max_input_files) { throw { message: `files cannot contain more than ${config.max_input_files} destinations` }; } const destinations = new Set(); for (const [i, value] of files.entries()) { - if (value == null || typeof value !== 'object' || Array.isArray(value)) { - throw { message: `files[${i}] must be an object` }; - } - const file = value as TFile; - const inline = typeof file.content === 'string'; - const byRef = typeof file.id === 'string' && file.id.length > 0; - if (inline === byRef) { - throw { - message: `files[${i}] must contain exactly one of non-empty id or string content`, - }; - } - if (file.id !== undefined && !byRef) { - throw { message: `files[${i}].id must be a non-empty string if provided` }; - } - if (byRef) { - if (typeof file.storage_session_id !== 'string' || file.storage_session_id.length === 0) { - throw { message: `files[${i}].storage_session_id is required as a non-empty string for file refs` }; - } - } else if (file.storage_session_id !== undefined || file.input_cache_key !== undefined) { - throw { - message: `files[${i}] inline content cannot include storage_session_id or input_cache_key`, - }; - } - if (file.name !== undefined && typeof file.name !== 'string') { - throw { message: `files[${i}].name must be a string if provided` }; - } - if ( - file.encoding !== undefined - && !(['base64', 'hex', 'utf8'] as const).includes(file.encoding) - ) { - throw { message: `files[${i}].encoding must be base64, hex, or utf8 if provided` }; - } - if (file.entity_id !== undefined && typeof file.entity_id !== 'string') { - throw { message: `files[${i}].entity_id must be a string if provided` }; - } - if ( - file.input_cache_key !== undefined && - ( - typeof file.input_cache_key !== 'string' || - !/^[0-9a-f]{64}$/.test(file.input_cache_key) - ) - ) { - throw { message: `files[${i}].input_cache_key must be a 64-character lowercase hex digest` }; - } - const destination = file.name || `file${i}.code`; - try { - validateFilePath(destination, '/tmp/codeapi-request-validation'); - } catch (error) { - throw { - message: error instanceof Error - ? `files[${i}].name is invalid: ${error.message}` - : `files[${i}].name is invalid`, - }; - } + const destination = validateExecuteFile(value, i); const conflict = [...destinations].find( existing => existing === destination || @@ -251,12 +289,13 @@ function getJob( runtimeSessionHeader?: string | string[], ): Job { const { - session_id, language, version, args, stdin, files, + session_id, language, version, args, stdin, files: rawFiles, compile_memory_limit, run_memory_limit, compile_timeout, run_cpu_time, compile_cpu_time, env_vars, } = body; + let files = rawFiles; if (!language || typeof language !== 'string') { throw { message: 'language is required as a string' }; @@ -271,6 +310,7 @@ function getJob( throw { message: 'tool_call_socket must be a boolean if specified' }; } validateExecuteArguments(args, stdin); + files = deduplicateFilesByDestination(files); validateExecuteFiles(files); const rt = getLatestRuntimeMatchingLanguageVersion(language, version); @@ -538,9 +578,6 @@ router.post('/execute', express.json({ limit: config.execute_body_limit }), asyn } try { - if (toolCallSocketEnabled) { - await ensureToolCallSocketProxyReady(); - } await withSpan('codeapi.sandbox.prime', { 'codeapi.language': job.runtime.language, }, () => job!.prime()); @@ -588,9 +625,13 @@ router.post('/execute', express.json({ limit: config.execute_body_limit }), asyn } } + /* Upload must finish before cleanup, and cleanup must settle before + * acknowledging completion. Failed removals retain a quarantined UID. */ + await cleanupHandler(); metricsOutcome = 'success'; return res.status(200).json(result); } catch (error) { + await cleanupHandler(); /* Deliberately BEFORE the ValidationError branch below: once priming has * completed, the workspace has been written to, so any later failure — * including a validation one — leaves state the next execute must not diff --git a/api/src/cleanup-metrics.test.ts b/api/src/cleanup-metrics.test.ts new file mode 100644 index 00000000..e1f61b22 --- /dev/null +++ b/api/src/cleanup-metrics.test.ts @@ -0,0 +1,110 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { + postCleanupMemory, + postCleanupTmp, + postCleanupWorkspaces, + readCleanupResources, + recordCleanupResources, +} from './cleanup-metrics'; + +const roots: string[] = []; +afterEach(() => { + for (const root of roots.splice(0)) + fs.rmSync(root, { recursive: true, force: true }); +}); + +function fixture() { + const root = fs.mkdtempSync(path.join(os.tmpdir(), 'cleanup-metrics-')); + roots.push(root); + const cgroupRoot = path.join(root, 'cgroup'); + const workspaceRoot = path.join(root, 'workspaces'); + fs.mkdirSync(cgroupRoot); + fs.mkdirSync(workspaceRoot); + fs.writeFileSync(path.join(cgroupRoot, 'memory.current'), '1048576\n'); + fs.writeFileSync( + path.join(cgroupRoot, 'memory.stat'), + 'anon 262144\nfile 786432\nshmem 524288\n' + ); + return { cgroupRoot, workspaceRoot, tmpRoot: root }; +} + +describe('post-cleanup resources', () => { + test('reads byte-accurate cgroup counters without adding shmem to file', () => { + const paths = fixture(); + // The API child cgroup excludes sibling jails. Sample the visible root. + fs.mkdirSync(path.join(paths.cgroupRoot, 'sandbox_api')); + fs.writeFileSync( + path.join(paths.cgroupRoot, 'sandbox_api/memory.current'), + '1' + ); + const snapshot = readCleanupResources(paths); + expect(snapshot.memory).toEqual({ + current: 1048576, + anon: 262144, + file: 786432, + shmem: 524288, + }); + expect(snapshot.tmp?.bytes).toBeGreaterThanOrEqual(0); + expect(snapshot.tmp?.inodes).toBeGreaterThanOrEqual(0); + expect(snapshot.errors).toEqual({}); + }); + + test('separates disposable, persistent and unexpected entries without following symlinks', () => { + const paths = fixture(); + fs.mkdirSync(path.join(paths.workspaceRoot, 'ws_active')); + fs.mkdirSync(path.join(paths.workspaceRoot, 'session')); + fs.mkdirSync(path.join(paths.workspaceRoot, 'unknown')); + fs.symlinkSync(paths.tmpRoot, path.join(paths.workspaceRoot, 'ws_symlink')); + expect(readCleanupResources(paths).workspaces).toEqual({ + disposable: 1, + session: 1, + other: 2, + }); + }); + + test('does not turn missing or malformed statistics into a healthy zero', () => { + const paths = fixture(); + fs.writeFileSync( + path.join(paths.cgroupRoot, 'memory.stat'), + 'anon 1\nfile 2\n' + ); + expect(readCleanupResources(paths).memory).toBeNull(); + fs.writeFileSync( + path.join(paths.cgroupRoot, 'memory.current'), + 'not-a-number' + ); + const snapshot = readCleanupResources(paths); + expect(snapshot.memory).toBeNull(); + expect(snapshot.tmp).not.toBeNull(); + expect(snapshot.workspaces).not.toBeNull(); + expect(snapshot.errors.memory).toBe('unavailable'); + }); + + test('unreadable sources remove stale gauges without failing cleanup', async () => { + const paths = fixture(); + const context = { + job: 'test', + mode: 'disposable', + outcome: 'removed', + suppressSuccessLogs: true, + } as const; + recordCleanupResources(context, readCleanupResources(paths)); + expect((await postCleanupMemory.get()).values).toHaveLength(4); + expect((await postCleanupTmp.get()).values).toHaveLength(3); + expect((await postCleanupWorkspaces.get()).values).toHaveLength(3); + fs.rmSync(paths.tmpRoot, { recursive: true }); + const snapshot = readCleanupResources(paths); + expect(snapshot.errors).toEqual({ + memory: 'ENOENT', + tmp: 'ENOENT', + workspaces: 'ENOENT', + }); + expect(() => recordCleanupResources(context, snapshot)).not.toThrow(); + expect((await postCleanupMemory.get()).values).toHaveLength(0); + expect((await postCleanupTmp.get()).values).toHaveLength(0); + expect((await postCleanupWorkspaces.get()).values).toHaveLength(0); + }); +}); diff --git a/api/src/cleanup-metrics.ts b/api/src/cleanup-metrics.ts new file mode 100644 index 00000000..7444960d --- /dev/null +++ b/api/src/cleanup-metrics.ts @@ -0,0 +1,185 @@ +import * as fs from 'fs'; +import * as path from 'path'; +import { Counter, Gauge } from 'prom-client'; +import { logger } from './logger'; +import { + SANDBOX_WORKSPACE_ROOT, + SESSION_WORKSPACE_ID, + WORKSPACE_ID_PREFIX, + retainedWorkspaceCleanupCount, + sandboxJobUidPool, +} from './workspace-isolation'; + +const memoryKinds = ['current', 'anon', 'file', 'shmem'] as const; +const workspaceKinds = ['disposable', 'session', 'other'] as const; +type MemoryKind = typeof memoryKinds[number]; +type WorkspaceKind = typeof workspaceKinds[number]; + +export const postCleanupMemory = new Gauge({ + name: 'codeapi_sandbox_post_cleanup_memory_bytes', + help: 'Last post-cleanup cgroup-root memory sample; file includes shmem, not additive', + labelNames: ['kind'] as const, +}); +export const postCleanupTmp = new Gauge({ + name: 'codeapi_sandbox_post_cleanup_tmp_used', + help: 'Last post-cleanup /tmp filesystem allocation, including unlinked open files', + labelNames: ['resource'] as const, +}); +export const postCleanupWorkspaces = new Gauge({ + name: 'codeapi_sandbox_post_cleanup_workspaces', + help: 'Remaining workspace-root entries after cleanup; includes other active jobs', + labelNames: ['kind'] as const, +}); +const cleanupAttempts = new Counter({ + name: 'codeapi_sandbox_cleanup_total', + help: 'Job cleanup attempts; session workspaces are intentionally preserved', + labelNames: ['mode', 'outcome'] as const, +}); +const sampleSuccess = new Gauge({ + name: 'codeapi_sandbox_post_cleanup_sample_success', + help: 'Whether each source was readable on the last post-cleanup sample', + labelNames: ['source'] as const, +}); +const sampleTimestamp = new Gauge({ + name: 'codeapi_sandbox_post_cleanup_timestamp_seconds', + help: 'Timestamp of the last job cleanup resource sample, not a live usage gauge', +}); + +export interface CleanupResourceSnapshot { + memory: Record | null; + tmp: { bytes: number; inodes: number; isTmpfs: boolean } | null; + workspaces: Record | null; + errors: Partial>; +} + +/** Read only fixed kernel files and the workspace root, never recurse into + * untrusted workspace trees or follow their symlinks. Synchronous sampling + * avoids older, slower samples overwriting newer ones during parallel cleanup. + * The visible cgroup mount root includes the API AND sibling NsJail cgroups; + * /proc/self/cgroup may point only at the delegated sandbox_api child instead. + * These are runner-wide observations, not memory attributable to one job. */ +export function readCleanupResources( + paths: { + cgroupRoot?: string; + tmpRoot?: string; + workspaceRoot?: string; + } = {} +): CleanupResourceSnapshot { + const snapshot: CleanupResourceSnapshot = { + memory: null, + tmp: null, + workspaces: null, + errors: {}, + }; + function sample( + source: keyof CleanupResourceSnapshot['errors'], + read: () => T + ): T | null { + try { + return read(); + } catch (error) { + snapshot.errors[source] = + (error as NodeJS.ErrnoException).code ?? 'unavailable'; + return null; + } + } + snapshot.memory = sample('memory', () => { + const root = paths.cgroupRoot ?? '/sys/fs/cgroup'; + const current = fs + .readFileSync(path.join(root, 'memory.current'), 'utf8') + .trim(); + const stat = fs.readFileSync(path.join(root, 'memory.stat'), 'utf8'); + const values = Object.fromEntries( + stat + .trim() + .split('\n') + .map(line => line.trim().split(/\s+/)) + ); + const result = {} as Record; + for (const kind of memoryKinds) { + const raw = kind === 'current' ? current : values[kind]; + if (!raw || !/^\d+$/.test(raw) || !Number.isSafeInteger(Number(raw))) { + throw new Error('Invalid memory statistic'); + } + result[kind] = Number(raw); + } + return result; + }); + snapshot.tmp = sample('tmp', () => { + const stat = fs.statfsSync(paths.tmpRoot ?? '/tmp'); + return { + bytes: (stat.blocks - stat.bfree) * stat.bsize, + inodes: stat.files - stat.ffree, + isTmpfs: stat.type === 0x01021994, + }; + }); + snapshot.workspaces = sample('workspaces', () => { + const counts = { disposable: 0, session: 0, other: 0 }; + for (const entry of fs.readdirSync( + paths.workspaceRoot ?? SANDBOX_WORKSPACE_ROOT, + { withFileTypes: true } + )) { + const kind = + entry.isDirectory() && entry.name === SESSION_WORKSPACE_ID + ? 'session' + : entry.isDirectory() && entry.name.startsWith(WORKSPACE_ID_PREFIX) + ? 'disposable' + : 'other'; + counts[kind]++; + } + return counts; + }); + return snapshot; +} + +export function recordCleanupResources( + context: { + job: string; + mode: 'disposable' | 'session'; + outcome: 'removed' | 'preserved' | 'retained' | 'error'; + suppressSuccessLogs?: boolean; + }, + snapshot = readCleanupResources() +): void { + cleanupAttempts.inc({ mode: context.mode, outcome: context.outcome }); + sampleTimestamp.set(Date.now() / 1000); + for (const source of ['memory', 'tmp', 'workspaces'] as const) { + sampleSuccess.set({ source }, snapshot[source] === null ? 0 : 1); + } + for (const kind of memoryKinds) { + if (snapshot.memory) postCleanupMemory.set({ kind }, snapshot.memory[kind]); + else postCleanupMemory.remove({ kind }); + } + for (const resource of ['bytes', 'inodes', 'is_tmpfs'] as const) { + if (snapshot.tmp) { + postCleanupTmp.set( + { resource }, + resource === 'is_tmpfs' + ? Number(snapshot.tmp.isTmpfs) + : snapshot.tmp[resource] + ); + } else postCleanupTmp.remove({ resource }); + } + for (const kind of workspaceKinds) { + if (snapshot.workspaces) + postCleanupWorkspaces.set({ kind }, snapshot.workspaces[kind]); + else postCleanupWorkspaces.remove({ kind }); + } + if ( + !context.suppressSuccessLogs || + context.outcome === 'retained' || + context.outcome === 'error' + ) { + logger.info( + { + job: context.job, + mode: context.mode, + outcome: context.outcome, + ...snapshot, + activeUidSlots: sandboxJobUidPool.activeCount(), + retainedCleanups: retainedWorkspaceCleanupCount(), + }, + 'Post-cleanup resources' + ); + } +} diff --git a/api/src/cleanup.integration.test.ts b/api/src/cleanup.integration.test.ts new file mode 100644 index 00000000..0d08e804 --- /dev/null +++ b/api/src/cleanup.integration.test.ts @@ -0,0 +1,184 @@ +import { expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as fsp from 'fs/promises'; +import * as os from 'os'; +import * as path from 'path'; +import * as semver from 'semver'; +import { config } from './config'; +import { Job } from './job'; +import type { Runtime } from './runtime'; +import { readCleanupResources } from './cleanup-metrics'; +import { prepareWorkspaceRoot, sandboxJobUidPool } from './workspace-isolation'; + +// Run only in a dedicated Linux test container with /tmp mounted as tmpfs. +// The real-jail variant additionally needs the runner image's namespace/cgroup +// permissions, nsjail, spec-guard and NSJAIL_CONFIG. Never target a live runner. +const tmpfsEnabled = process.env.SANDBOX_CLEANUP_TMPFS_TEST === '1'; +const nsjailEnabled = process.env.SANDBOX_CLEANUP_NSJAIL_TEST === '1'; + +function runtime(pkgdir: string): Runtime { + return { + language: 'bash', + version: new semver.SemVer('5.2.0'), + aliases: [], + pkgdir, + compiled: false, + env_vars: {}, + timeouts: { compile: 1000, run: 1000 }, + cpu_times: { compile: 1000, run: 1000 }, + memory_limits: { compile: 128 * 1048576, run: 128 * 1048576 }, + max_process_count: 64, + max_open_files: 64, + max_file_size: 16 * 1048576, + output_max_size: 4096, + }; +} + +function jobFor(rt: Runtime, content: string): Job { + return new Job({ + session_id: 'cleanup-stress', + runtime: rt, + files: [{ name: 'main.sh', content }], + args: [], + stdin: '', + timeouts: rt.timeouts, + cpu_times: rt.cpu_times, + memory_limits: rt.memory_limits, + is_synthetic: true, + }); +} + +function workspace(job: Job): string { + return (job as unknown as { submissionDir: string }).submissionDir; +} + +function assertTmpfsBaseline( + baseline: ReturnType +): void { + const after = readCleanupResources(); + expect(after.tmp?.isTmpfs).toBe(true); + // Allow a few metadata pages, not one retained 8 MiB payload per execution. + expect(after.tmp!.bytes).toBeLessThanOrEqual(baseline.tmp!.bytes + 64 * 1024); + expect(after.tmp!.inodes).toBeLessThanOrEqual(baseline.tmp!.inodes + 4); + expect(after.workspaces).toEqual(baseline.workspaces); +} + +test.skipIf(!tmpfsEnabled)( + 'repeated file-heavy Job cleanup returns tmpfs bytes and inodes near baseline', + async () => { + expect(process.platform).toBe('linux'); + await prepareWorkspaceRoot(); + const baseline = readCleanupResources(); + expect(baseline.tmp?.isTmpfs).toBe(true); + const activeBefore = sandboxJobUidPool.activeCount(); + const payload = Buffer.alloc(8 * 1048576, 0x61); + for (let iteration = 0; iteration < 24; iteration++) { + const job = jobFor(runtime('/tmp'), 'echo test'); + let dir = ''; + try { + await job.prime(); + dir = workspace(job); + await fsp.writeFile(path.join(dir, 'payload'), payload); + for (let i = 0; i < 64; i++) + await fsp.writeFile(path.join(dir, `small-${i}`), 'x'); + expect(readCleanupResources().tmp!.bytes).toBeGreaterThan( + baseline.tmp!.bytes + payload.length + ); + } finally { + await job.cleanup(); + } + expect(fs.existsSync(dir)).toBe(false); + expect(sandboxJobUidPool.activeCount()).toBe(activeBefore); + assertTmpfsBaseline(baseline); + } + }, + 30_000 +); + +function processesForUid(uid: number): string[] { + return fs.readdirSync('/proc').filter(pid => { + if (!/^\d+$/.test(pid)) return false; + try { + return ( + fs + .readFileSync(`/proc/${pid}/status`, 'utf8') + .match(/^Uid:\s+(\d+)/m)?.[1] === String(uid) + ); + } catch (error) { + if ( + (error as NodeJS.ErrnoException).code === 'ENOENT' || + (error as NodeJS.ErrnoException).code === 'ESRCH' + ) + return false; + throw error; + } + }); +} + +test.skipIf(!nsjailEnabled)( + 'real NsJail releases detached descendants and file-heavy workspaces on success, timeout and overflow', + async () => { + expect(process.platform).toBe('linux'); + expect(config.per_job_uids).toBe(true); + expect(process.getuid?.()).toBe(0); + await prepareWorkspaceRoot(); + const pkgdir = await fsp.mkdtemp( + path.join(os.tmpdir(), 'cleanup-runtime-') + ); + await fsp.chmod(pkgdir, 0o755); + await fsp.writeFile(path.join(pkgdir, 'run'), 'exec /bin/bash "$@"\n', { + mode: 0o644, + }); + const baseline = readCleanupResources(); + expect(baseline.tmp?.isTmpfs).toBe(true); + const activeBefore = sandboxJobUidPool.activeCount(); + try { + for (let iteration = 0; iteration < 12; iteration++) { + const ending = iteration % 3; + const job = jobFor( + runtime(pkgdir), + `set -eu +# Both linked workspace data and an unlinked open file held by a detached child. +dd if=/dev/zero of=payload bs=1048576 count=8 status=none +# Per-jail /tmp is a different mount from the runner's /tmp. +dd if=/dev/zero of=/tmp/held bs=1048576 count=8 status=none +exec 3 /mnt/data/child-ready; exec sleep 30' >/dev/null 2>&1 & +while [ ! -f child-ready ]; do sleep 0.01; done +${ + ending === 0 + ? 'echo complete' + : ending === 1 + ? 'sleep 30' + : "while :; do printf '%1024s' x; done" +} +` + ); + let dir = ''; + let uid = -1; + try { + await job.prime(); + dir = workspace(job); + uid = (job as unknown as { jobIdentity: { uid: number } }).jobIdentity + .uid; + const result = await job.execute(); + expect(result.run?.status).toBe( + ending === 0 ? null : ending === 1 ? 'TO' : 'OL' + ); + if (ending === 0) expect(result.run?.stdout).toContain('complete'); + // Check before workspace removal can conceal a surviving writer. + expect(processesForUid(uid)).toEqual([]); + } finally { + await job.cleanup(); + } + expect(fs.existsSync(dir)).toBe(false); + expect(sandboxJobUidPool.activeCount()).toBe(activeBefore); + assertTmpfsBaseline(baseline); + } + } finally { + await fsp.rm(pkgdir, { recursive: true, force: true }); + } + }, + 60_000 +); diff --git a/api/src/entrypoint.sh b/api/src/entrypoint.sh old mode 100755 new mode 100644 index 5f3fa141..06fa0b6b --- a/api/src/entrypoint.sh +++ b/api/src/entrypoint.sh @@ -124,8 +124,8 @@ fi ALLOWED_PORT="${SANDBOX_ALLOWED_LOCAL_NETWORK_PORT:-0}" # NsJail runs sandboxed processes as inside UID 65534 (nobody), mapped to a -# per-job outside UID by the sandbox API. The proxy socket remains mounted -# into each jail, so it does not need per-UID ownership. +# per-job outside UID by the sandbox API. Authorized tool-call invocations +# receive two anonymous pipe ends; no socket path is mounted into a jail. SANDBOX_UID=65534 # Extract API port from bind address (default 2000) @@ -137,11 +137,11 @@ if [ "$ALLOWED_PORT" -gt 0 ] 2>/dev/null; then exit 1 fi - echo "Tool-call socket forwarding enabled for UID $SANDBOX_UID (port $ALLOWED_PORT)" + echo "Tool-call pipe forwarding enabled for UID $SANDBOX_UID (port $ALLOWED_PORT)" # Do not start Node here. Lambda MicroVM image creation snapshots this # already-running container, and the official Node binary embeds OpenSSL. - # The API starts and awaits the narrow Unix-socket proxy only after a - # post-restore /execute is authorized for tool calls. + # The API starts a per-invocation pipe broker only after a post-restore + # /execute is authorized for tool calls. fi # Package permissions are finalized by package-init when the PVC is populated. @@ -179,7 +179,7 @@ if [ "$SANDBOX_USE_CGROUPV2" = "true" ]; then NSJAIL_CGROUP_ARGS=(--use_cgroupv2) fi -if timeout 10 "${NSJAIL_PATH:-/usr/sbin/nsjail}" --config "${NSJAIL_CONFIG:-/sandbox_api/config/sandbox.cfg}" \ +if timeout 10 /usr/local/bin/sandbox-supervisor-policy "${NSJAIL_PATH:-/usr/sbin/nsjail}" --config "${NSJAIL_CONFIG:-/sandbox_api/config/sandbox.cfg}" \ "${NSJAIL_CGROUP_ARGS[@]}" --log "$SMOKE_LOG" \ --user "65534:${SMOKE_OUTSIDE_UID}:1" --group "65534:${SMOKE_OUTSIDE_GID}:1" \ -s /usr/bin:/bin -s /usr/lib:/lib -s /usr/lib64:/lib64 \ @@ -203,4 +203,4 @@ rm -rf "$SMOKE_DIR" echo "Starting sandbox API server..." # Absolute path: the Lambda MicroVM guest does not carry the image PATH. -exec /usr/local/bin/bun run /sandbox_api/.build/index.js +exec /usr/local/bin/sandbox-supervisor-policy /usr/local/bin/bun run /sandbox_api/.build/index.js diff --git a/api/src/hosted-app.ts b/api/src/hosted-app.ts index 64a5aab8..0ce57370 100644 --- a/api/src/hosted-app.ts +++ b/api/src/hosted-app.ts @@ -1,10 +1,10 @@ -import { execFile, spawn, type ChildProcess, type SpawnOptions } from 'node:child_process'; +import { spawnPipeProcess as spawn } from './pipe-process'; +import type { ChildProcess, SpawnOptions } from 'node:child_process'; import { createHash } from 'node:crypto'; import * as fsp from 'node:fs/promises'; import * as net from 'node:net'; import * as path from 'node:path'; import type { Readable } from 'node:stream'; -import { promisify } from 'node:util'; import { config } from './config'; import { aggregateBashExtras, filterExtraEnvVars } from './job'; import { logger } from './logger'; @@ -12,7 +12,22 @@ import { getLatestRuntimeMatchingLanguageVersion, getRuntimes, type Runtime } fr import { getBoundSessionWorkspace, type SessionWorkspace } from './session-workspace'; import { ValidationError, validateFilePath } from './validation'; -const execFileAsync = promisify(execFile); +// Capture bounded diagnostics through an anonymous pipe; execFile's runtime +// capture would create socketpairs prohibited by the supervisor policy. +function execFileAsync(binary: string, args: string[]): Promise { + return new Promise((resolve, reject) => { + const child = spawn(binary, args, { stdio: ['ignore', 'ignore', 'pipe'] }); + let diagnostics = Buffer.alloc(0); + child.stderr.on('data', (chunk: Buffer) => { + diagnostics = Buffer.concat([diagnostics, chunk]).subarray(-8192); + }); + child.once('error', reject); + child.once('close', (code, signal) => { + if (code === 0) resolve(); + else reject(new Error(`${binary} failed (${signal ?? code}): ${diagnostics.toString().trim()}`)); + }); + }); +} const APP_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,63}$/; const REVISION_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/; const ENV_NAME_PATTERN = /^[A-Za-z_][A-Za-z0-9_]*$/; @@ -342,7 +357,10 @@ async function commandSucceeds(binary: string, args: string[]): Promise * packets for accepted inbound connections remain allowed by conntrack. */ export async function installHostedAppNetworkGuard(uid: number): Promise { - for (const binary of ['/usr/sbin/iptables', '/usr/sbin/ip6tables']) { + // nft's netlink transport needs sendmsg, which the supervisor denies. The + // packaged legacy backend uses getsockopt/setsockopt instead. Failure to + // install either family remains fatal before admitting any hosted app. + for (const binary of ['/usr/sbin/iptables-legacy', '/usr/sbin/ip6tables-legacy']) { if (!(await commandSucceeds(binary, ['-w', '5', '-L', HOSTED_APP_EGRESS_CHAIN]))) { await execFileAsync(binary, ['-w', '5', '-N', HOSTED_APP_EGRESS_CHAIN]); } diff --git a/api/src/index.ts b/api/src/index.ts index cfbc936c..00d10e9d 100644 --- a/api/src/index.ts +++ b/api/src/index.ts @@ -7,7 +7,6 @@ import { initializeSandboxWorkspaceIsolation, startWorkspaceReaper } from './wor import { httpMetricsMiddleware, metricsHandler } from './metrics'; import { positiveInt, shutdownTelemetry, traceHttpRequest } from './telemetry'; import { startWarmupCommand } from './warmup'; -import { stopToolCallSocketProxy } from './tool-call-socket-process'; import { hostedAppSupervisor, validateHostedAppStartup } from './hosted-app'; import v2Router from './api/v2'; import lifecycleRouter, { LIFECYCLE_HOOK_BASE_PATH } from './api/lifecycle'; @@ -116,9 +115,6 @@ async function main(): Promise { await hostedAppSupervisor.shutdown().catch((err) => { logger.warn({ err }, 'Hosted-app process shutdown failed'); }); - await stopToolCallSocketProxy().catch((err) => { - logger.warn({ err }, 'Tool-call socket proxy shutdown failed'); - }); try { await shutdownTelemetry(); } catch (err) { diff --git a/api/src/job-cleanup-metrics.test.ts b/api/src/job-cleanup-metrics.test.ts new file mode 100644 index 00000000..6806f637 --- /dev/null +++ b/api/src/job-cleanup-metrics.test.ts @@ -0,0 +1,118 @@ +import { afterEach, expect, spyOn, test } from 'bun:test'; +import * as fsp from 'fs/promises'; +import * as os from 'os'; +import * as path from 'path'; +import * as semver from 'semver'; +import { register } from 'prom-client'; +import { Job } from './job'; +import type { Runtime } from './runtime'; +import type { SessionWorkspace } from './session-workspace'; +import { + clearRetainedWorkspaceCleanupsForTest, + retainedWorkspaceCleanupCount, + retryRetainedWorkspaceCleanups, + sandboxJobUidPool, +} from './workspace-isolation'; + +import type { + SandboxJobIdentity, + SandboxWorkspaceLease, +} from './workspace-isolation'; + +interface Internals { + submissionDir: string; + workspaceLease?: SandboxWorkspaceLease; + jobIdentity?: SandboxJobIdentity; +} +const roots: string[] = []; +afterEach(async () => { + await retryRetainedWorkspaceCleanups(); + clearRetainedWorkspaceCleanupsForTest(); + for (const root of roots.splice(0)) + await fsp.rm(root, { recursive: true, force: true }); +}); + +async function fixture(persistent = false) { + const dir = await fsp.mkdtemp(path.join(os.tmpdir(), 'job-cleanup-metric-')); + roots.push(dir); + const identity = sandboxJobUidPool.acquire()!; + expect(identity).not.toBeNull(); + const lease = { dir, workspaceId: path.basename(dir), identity }; + const rt: Runtime = { + language: 'bash', + version: new semver.SemVer('5.2.0'), + aliases: [], + pkgdir: '/tmp', + compiled: false, + env_vars: {}, + timeouts: { compile: 1000, run: 1000 }, + cpu_times: { compile: 1000, run: 1000 }, + memory_limits: { compile: -1, run: -1 }, + max_process_count: 64, + max_open_files: 64, + max_file_size: 1000, + output_max_size: 1000, + }; + const job = new Job({ + runtime: rt, + files: [], + args: [], + stdin: '', + timeouts: rt.timeouts, + cpu_times: rt.cpu_times, + memory_limits: rt.memory_limits, + is_synthetic: true, + session: persistent ? ({} as SessionWorkspace) : undefined, + }); + Object.assign(job as unknown as Internals, { + submissionDir: dir, + workspaceLease: lease, + jobIdentity: identity, + }); + return { job, dir, identity }; +} + +async function counter(mode: string, outcome: string): Promise { + const metric = register.getSingleMetric('codeapi_sandbox_cleanup_total')!; + return ( + (await metric.get()).values.find( + value => value.labels.mode === mode && value.labels.outcome === outcome + )?.value ?? 0 + ); +} + +test('failed removal is measured as retained and does not release its UID before retry', async () => { + const activeBefore = sandboxJobUidPool.activeCount(); + const retainedBefore = await counter('disposable', 'retained'); + const { job, dir } = await fixture(); + const rm = spyOn(fsp, 'rm').mockRejectedValueOnce( + Object.assign(new Error('test busy workspace'), { code: 'EBUSY' }) + ); + try { + await job.cleanup(); + expect(await counter('disposable', 'retained')).toBe(retainedBefore + 1); + expect(sandboxJobUidPool.activeCount()).toBe(activeBefore + 1); + expect(retainedWorkspaceCleanupCount()).toBe(1); + expect((await fsp.lstat(dir)).isDirectory()).toBe(true); + } finally { + rm.mockRestore(); + await retryRetainedWorkspaceCleanups(); + } + expect(sandboxJobUidPool.activeCount()).toBe(activeBefore); + expect(await fsp.lstat(dir).catch(() => null)).toBeNull(); +}); + +test('session cleanup is measured as preserved and retains both files and pinned UID', async () => { + const activeBefore = sandboxJobUidPool.activeCount(); + const preservedBefore = await counter('session', 'preserved'); + const { job, dir, identity } = await fixture(true); + try { + await fsp.writeFile(path.join(dir, 'state'), 'keep'); + await job.cleanup(); + expect(await counter('session', 'preserved')).toBe(preservedBefore + 1); + expect(await fsp.readFile(path.join(dir, 'state'), 'utf8')).toBe('keep'); + expect(sandboxJobUidPool.activeCount()).toBe(activeBefore + 1); + } finally { + sandboxJobUidPool.release(identity); + } +}); diff --git a/api/src/job.ts b/api/src/job.ts index 0939de08..bfa99b63 100644 --- a/api/src/job.ts +++ b/api/src/job.ts @@ -14,6 +14,7 @@ import { logger as rootLogger } from './logger'; import { getRuntimes } from './runtime'; import { execute } from './nsjail'; import { config } from './config'; +import { recordCleanupResources } from './cleanup-metrics'; import { internalServiceHeaders } from './internal-service-auth'; import { EGRESS_GRANT_HEADER, EGRESS_ERROR_CODE_HEADER } from './egress'; import { injectTraceHeaders } from './telemetry'; @@ -700,7 +701,7 @@ export class Job { memory_limits: { run: number; compile: number }; extra_env_vars?: Record; egressGrantToken?: string; - toolCallSocketEnabled: boolean; + toolCallPipesEnabled: boolean; isSynthetic: boolean; outputSessionId: string; @@ -785,7 +786,7 @@ export class Job { this.memory_limits = opts.memory_limits; this.extra_env_vars = opts.extra_env_vars; this.egressGrantToken = opts.egress_grant; - this.toolCallSocketEnabled = opts.tool_call_socket_enabled === true; + this.toolCallPipesEnabled = opts.tool_call_socket_enabled === true; this.isSynthetic = opts.is_synthetic === true; } @@ -1643,7 +1644,7 @@ export class Job { stdin, extraPkgdirs, identity: this.sandboxIdentity(), - enableToolCallSocket: this.toolCallSocketEnabled && script === 'run', + enableToolCallPipes: this.toolCallPipesEnabled && script === 'run', suppressSuccessLogs: this.isSynthetic, }); } @@ -2716,49 +2717,61 @@ export class Job { this.log.info('Cleaning up'); } - /* Session mode: the workspace and pinned UID belong to the long-lived - * session, not this job. Keep both so the next call sees prior files; - * teardown happens on the /terminate hook (or explicit session reset). */ - if (this.session) { - this.workspaceLease = undefined; - this.submissionDir = ''; - this.jobIdentity = undefined; - return; - } - - let workspaceRemoved = true; - const workspaceLease = this.workspaceLease; - const jobIdentity = this.jobIdentity; - - if (workspaceLease) { - try { - workspaceRemoved = await cleanupSandboxWorkspace(workspaceLease); - } catch (error) { - workspaceRemoved = false; - this.log.error({ submissionDir: this.submissionDir, err: error }, 'Failed to clean up'); - } finally { + let outcome: 'removed' | 'preserved' | 'retained' | 'error' = 'error'; + try { + /* Session mode: the workspace and pinned UID belong to the long-lived + * session, not this job. Keep both so the next call sees prior files; + * teardown happens on the /terminate hook (or explicit session reset). */ + if (this.session) { this.workspaceLease = undefined; this.submissionDir = ''; + this.jobIdentity = undefined; + outcome = 'preserved'; + return; } - } - if (jobIdentity) { - if (!workspaceLease || workspaceRemoved) { - releaseJobIdentity(jobIdentity); - } else { - retainWorkspaceCleanupUntilRemoved(workspaceLease, () => { + let workspaceRemoved = true; + const workspaceLease = this.workspaceLease; + const jobIdentity = this.jobIdentity; + + if (workspaceLease) { + try { + workspaceRemoved = await cleanupSandboxWorkspace(workspaceLease); + } catch (error) { + workspaceRemoved = false; + this.log.error({ submissionDir: this.submissionDir, err: error }, 'Failed to clean up'); + } finally { + this.workspaceLease = undefined; + this.submissionDir = ''; + } + } + + if (jobIdentity) { + if (!workspaceLease || workspaceRemoved) { releaseJobIdentity(jobIdentity); - this.log.info( + } else { + retainWorkspaceCleanupUntilRemoved(workspaceLease, () => { + releaseJobIdentity(jobIdentity); + this.log.info( + { uid: jobIdentity.uid, gid: jobIdentity.gid, slot: jobIdentity.slot }, + 'Released retained sandbox job UID slot after workspace cleanup', + ); + }); + this.log.error( { uid: jobIdentity.uid, gid: jobIdentity.gid, slot: jobIdentity.slot }, - 'Released retained sandbox job UID slot after workspace cleanup', + 'Retaining sandbox job UID slot after failed workspace cleanup', ); - }); - this.log.error( - { uid: jobIdentity.uid, gid: jobIdentity.gid, slot: jobIdentity.slot }, - 'Retaining sandbox job UID slot after failed workspace cleanup', - ); + } + this.jobIdentity = undefined; } - this.jobIdentity = undefined; + outcome = workspaceRemoved ? 'removed' : 'retained'; + } finally { + recordCleanupResources({ + job: this.uuid, + mode: this.session ? 'session' : 'disposable', + outcome, + suppressSuccessLogs: this.isSynthetic, + }); } } } diff --git a/api/src/nsjail.test.ts b/api/src/nsjail.test.ts index ee6fbbc8..ab046b7f 100644 --- a/api/src/nsjail.test.ts +++ b/api/src/nsjail.test.ts @@ -91,7 +91,7 @@ describe('NsJail args', () => { expect(valueAfter(args, '--config')).toBe('/tmp/nsjail-job-xyz.cfg'); }); - test('does not export TOOL_CALL_SOCKET into the jail (preamble references the literal path)', () => { + test('does not export a legacy tool-call socket path into the jail', () => { const args = buildArgs({ logPath: '/tmp/nsjail-test.log', pkgdir: '/pkgs/python/3.14.4', @@ -109,7 +109,7 @@ describe('NsJail args', () => { } }); - test('binds the tool-call socket only for jobs that explicitly request it', () => { + test('passes tool-call pipes only for jobs that explicitly request the capability', () => { const originalAllowedPort = config.allowed_local_network_port; config.allowed_local_network_port = 3190; try { @@ -126,8 +126,12 @@ describe('NsJail args', () => { const withoutSocket = buildArgs(baseOpts); expect(hasArgPair(withoutSocket, '-B', '/tmp/tcs.sock:/tmp/tcs.sock')).toBe(false); - const withSocket = buildArgs({ ...baseOpts, enableToolCallSocket: true }); - expect(hasArgPair(withSocket, '-B', '/tmp/tcs.sock:/tmp/tcs.sock')).toBe(true); + const withSocket = buildArgs({ ...baseOpts, enableToolCallPipes: true }); + expect(hasArgPair(withSocket, '-B', '/tmp/tcs.sock:/tmp/tcs.sock')).toBe(false); + expect(hasArgPair(withSocket, '--pass_fd', '3')).toBe(true); + expect(hasArgPair(withSocket, '--pass_fd', '4')).toBe(true); + expect(withSocket).toContain('--tool-call-pipes'); + expect(withoutSocket).not.toContain('--pass_fd'); } finally { config.allowed_local_network_port = originalAllowedPort; } @@ -371,7 +375,37 @@ describe('NsJail seccomp policy', () => { const errnoSocketRule = errnoBlock.split('\n').find(line => line.includes('socket(domain)')); expect(errnoSocketRule).toBeDefined(); - expect(errnoSocketRule).not.toContain('AF_VSOCK'); + expect(errnoSocketRule).toContain('domain != AF_VSOCK'); + }); + + test('blocks priority-inheritance futex operations without denying ordinary wait/wake', () => { + const policy = seccompPolicy(); + expect(policy).toContain('#define FUTEX_CMD_MASK 0x7f'); + const rule = policy.split('ERRNO(1)')[1]?.split('\n') + .find(line => line.includes('futex(uaddr, op)')); + expect(rule).toBeDefined(); + for (const [operation, command] of [ + ['LOCK_PI', 6], ['UNLOCK_PI', 7], ['TRYLOCK_PI', 8], + ['WAIT_REQUEUE_PI', 11], ['CMP_REQUEUE_PI', 12], ['LOCK_PI2', 13], + ] as const) { + expect(policy).toContain(`#define FUTEX_${operation} ${command}`); + expect(rule).toContain(`(op & FUTEX_CMD_MASK) == FUTEX_${operation}`); + } + expect(rule).not.toMatch(/== FUTEX_(WAIT|WAKE)(?:\s|$)/); + }); + + test('blocks descriptor-passing sends and all socketpair creation', () => { + const policy = seccompPolicy(); + const errnoBlock = policy.split('ERRNO(1) {')[1]?.split(' }')[0] ?? ''; + for (const name of ['sendmsg', 'sendmmsg', 'io_uring_setup', 'io_uring_enter', 'io_uring_register']) { + expect(errnoBlock).toMatch(new RegExp(`\\b${name}\\b[,\\s]`)); + } + // The patched runtime and tool calls use anonymous pipes. + expect(policy).toContain(' socketpair'); + for (const name of ['read', 'write', 'sendto', 'recvfrom']) { + expect(policy).not.toMatch(new RegExp(`\\b${name}\\b`)); + } + expect(policy).toContain('USE sandbox DEFAULT ALLOW'); }); test('rejects Copy Fail and Dirty Frag socket entry points in the sandbox', () => { @@ -379,10 +413,7 @@ describe('NsJail seccomp policy', () => { const errnoBlock = policy.split('ERRNO(1)')[1] ?? ''; const socketRule = errnoBlock.split('\n').find(line => line.includes('socket(domain)')); expect(socketRule).toBeDefined(); - for (const family of ['AF_ALG', 'AF_RXRPC', 'AF_INET', 'AF_INET6']) { - expect(socketRule).toContain(family); - } - expect(socketRule).not.toContain('AF_VSOCK'); + expect(socketRule).toContain('domain != AF_VSOCK'); }); test('KILLs the defense-in-depth batch from the audit', () => { diff --git a/api/src/nsjail.ts b/api/src/nsjail.ts index 87a2b791..23c048a6 100644 --- a/api/src/nsjail.ts +++ b/api/src/nsjail.ts @@ -1,6 +1,7 @@ +import { spawnPipeProcess as spawn } from './pipe-process'; import * as fs from 'fs'; import * as path from 'path'; -import { spawn, type ChildProcessWithoutNullStreams } from 'node:child_process'; +import type { ChildProcessWithoutNullStreams } from 'node:child_process'; import { nanoid } from 'nanoid'; import { config } from './config'; import { logger } from './logger'; @@ -104,6 +105,13 @@ const SECCOMP_POLICY = [ '#define AF_VSOCK 40', '#define CLONE_NAMESPACE_FLAGS 0x7e020000', '#define KVM_IOCTL_MAGIC 0xAE00', + '#define FUTEX_CMD_MASK 0x7f', + '#define FUTEX_LOCK_PI 6', + '#define FUTEX_UNLOCK_PI 7', + '#define FUTEX_TRYLOCK_PI 8', + '#define FUTEX_WAIT_REQUEUE_PI 11', + '#define FUTEX_CMP_REQUEUE_PI 12', + '#define FUTEX_LOCK_PI2 13', 'POLICY sandbox {', ' KILL {', ' ptrace, memfd_create, personality, userfaultfd,', @@ -141,6 +149,17 @@ const SECCOMP_POLICY = [ ' },', ' ERRNO(1) {', ' io_uring_setup, io_uring_enter, io_uring_register, sched_setaffinity, vmsplice,', + /* CVE-2026-80521: AF_UNIX GC is reachable through SCM_RIGHTS descriptor + * passing. Seccomp cannot inspect the msghdr/mmsghdr control data, so + * refuse both send APIs outright, including on inherited sockets. + * The job receives only anonymous tool-call pipes. Anonymous + * pipes replace socketpairs in the packaged Python runtime. + * io_uring above must remain blocked: + * its sendmsg operations bypass syscall-level filtering. */ + ' sendmsg, sendmmsg,', + /* GhostLock (CVE-2026-43499): deny PI operations, including private and + * realtime variants, without blocking ordinary runtime futex wait/wake. */ + ' futex(uaddr, op) { (op & FUTEX_CMD_MASK) == FUTEX_LOCK_PI || (op & FUTEX_CMD_MASK) == FUTEX_UNLOCK_PI || (op & FUTEX_CMD_MASK) == FUTEX_TRYLOCK_PI || (op & FUTEX_CMD_MASK) == FUTEX_WAIT_REQUEUE_PI || (op & FUTEX_CMD_MASK) == FUTEX_CMP_REQUEUE_PI || (op & FUTEX_CMD_MASK) == FUTEX_LOCK_PI2 },', ' clone(flags) { (flags & CLONE_NAMESPACE_FLAGS) != 0 },', /* Block signals to PID 1 of the sandbox PID namespace (the NsJail * monitor). With clone_newpid the user can't reach other tenants — but @@ -164,10 +183,11 @@ const SECCOMP_POLICY = [ * runtimes the sandbox supports. */ ' pidfd_open(pid) { pid == 1 },', ' pidfd_send_signal,', - /* Block direct network and kernel-control socket domains from sandboxed - * code. AF_ALG covers the Linux kernel crypto API used by Copy Fail, and - * AF_RXRPC covers the RxRPC family used by Dirty Frag. */ - ' socket(domain) { domain == AF_INET || domain == AF_INET6 || domain == AF_NETLINK || domain == AF_KEY || domain == AF_RXRPC || domain == AF_ALG }', + /* Deny every socket() family. VSOCK keeps its stronger KILL action above. + * Deny socketpairs as well; packaged runtime IPC uses anonymous pipes. + * Keep sendmsg/sendmmsg denied for inherited descriptors too. */ + ' socket(domain) { domain != AF_VSOCK },', + ' socketpair', ' }', '}', 'USE sandbox DEFAULT ALLOW', @@ -225,7 +245,7 @@ interface ExecuteOptions { stdin?: string; extraPkgdirs?: string[]; identity: SandboxJobIdentity; - enableToolCallSocket?: boolean; + enableToolCallPipes?: boolean; suppressSuccessLogs?: boolean; } @@ -241,7 +261,7 @@ export async function execute(opts: ExecuteOptions, setupGate: NsJailSetupGate = stdin, extraPkgdirs, identity, - enableToolCallSocket, + enableToolCallPipes, suppressSuccessLogs, } = opts; const logId = nanoid(); @@ -260,7 +280,7 @@ export async function execute(opts: ExecuteOptions, setupGate: NsJailSetupGate = command, extraPkgdirs, identity, - enableToolCallSocket, + enableToolCallPipes, }); const startTime = Date.now(); @@ -382,11 +402,20 @@ export async function execute(opts: ExecuteOptions, setupGate: NsJailSetupGate = const childDiedSignal = new AbortController(); try { ({ value: proc, markerSeen, pollError } = await setupGate.runSetup(logPath, () => { - const child = spawn(config.nsjail_path, nsjailArgs, { + const pipeMode = enableToolCallPipes === true; + if (pipeMode && (config.allowed_local_network_port <= 0 || !process.env.SANDBOX_FORWARD_TARGET?.trim())) { + throw new Error('tool-call pipe broker is not configured'); + } + const child = spawn(pipeMode ? (process.env.TCS_PIPE_BRIDGE || '/usr/local/bin/tool-call-pipe-bridge') : config.nsjail_path, pipeMode ? [ + '--broker', process.env.TCS_NODE_BINARY || 'node', + process.env.TCS_PIPE_BROKER_BUNDLE ?? path.resolve(__dirname, '..', '.build', 'tool-call-pipe-broker.cjs'), + config.nsjail_path, ...nsjailArgs, + ] : nsjailArgs, { /* stdin always 'pipe' so we have a writable handle to .end() even * when no input is supplied. The child sees EOF on stdin * immediately, equivalent to the previous Bun.spawn 'ignore'. */ stdio: ['pipe', 'pipe', 'pipe'], + env: { ...process.env, TCS_REQUEST_TIMEOUT_MS: String(timeout) }, }) as ChildProcessWithoutNullStreams; /* Attach the 'error' listener SYNCHRONOUSLY before any await. The * setup gate then polls the log file for ~tens of ms; during that @@ -523,10 +552,10 @@ export async function execute(opts: ExecuteOptions, setupGate: NsJailSetupGate = }); const wallTime = Date.now() - startTime; - // Log memory metrics after each execution to track potential leaks. + // Pre-cleanup diagnostic only: Job.cleanup records post-cleanup resources. // Reads from /proc/self/cgroup to find the actual cgroup path, then reads // memory.current and memory.stat from that cgroup. - // Distinguishes real usage (anon) from reclaimable kernel page cache (file). + // `file` includes shmem/tmpfs and is not all reclaimable page cache. try { const cgroupLine = fs.readFileSync('/proc/self/cgroup', 'utf8').trim(); // cgroup v2 format: "0::" @@ -637,7 +666,7 @@ interface BuildArgsOptions { command: string[]; extraPkgdirs?: string[]; identity: SandboxJobIdentity; - enableToolCallSocket?: boolean; + enableToolCallPipes?: boolean; } export function buildArgs(opts: BuildArgsOptions): string[] { @@ -651,7 +680,7 @@ export function buildArgs(opts: BuildArgsOptions): string[] { command, extraPkgdirs, identity, - enableToolCallSocket, + enableToolCallPipes, } = opts; const timeoutSecs = Math.max(1, Math.ceil(timeout / 1000)); @@ -704,25 +733,17 @@ export function buildArgs(opts: BuildArgsOptions): string[] { args.push('--cgroup_mem_max', String(memoryLimit)); } - if (config.allowed_local_network_port > 0 && enableToolCallSocket === true) { - const socketPath = '/tmp/tcs.sock'; - args.push('-B', `${socketPath}:${socketPath}`); + if (enableToolCallPipes === true) { + args.push('--pass_fd', '3', '--pass_fd', '4'); } for (const [key, value] of Object.entries(envVars)) { args.push('-E', `${key}=${value}`); } - /* TOOL_CALL_SOCKET is intentionally NOT exported: the path is fixed at - * /tmp/tcs.sock and the blocking-mode runtime preamble references it - * as a literal. - * Skipping the env entry shrinks the surface user code can introspect - * (`os.environ`). The per-job /tmp/tcs.sock bind-mount above is what - * actually makes the path connectable, and ordinary execute/replay jobs - * do not receive that mount. */ - - args.push('--'); - args.push('/usr/local/bin/spec-guard', ...command); + args.push('--', '/usr/local/bin/spec-guard'); + if (enableToolCallPipes === true) args.push('--tool-call-pipes'); + args.push(...command); return args; } diff --git a/api/src/pipe-process.test.ts b/api/src/pipe-process.test.ts new file mode 100644 index 00000000..608002c8 --- /dev/null +++ b/api/src/pipe-process.test.ts @@ -0,0 +1,89 @@ +import { describe, test, expect } from 'bun:test'; +import { once } from 'node:events'; +import { pipeline } from 'node:stream/promises'; +import { Readable } from 'node:stream'; +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import { spawnPipeProcess } from './pipe-process'; + +async function collect(stream: Readable): Promise { + const chunks: Buffer[] = []; + for await (const chunk of stream) chunks.push(Buffer.from(chunk)); + return Buffer.concat(chunks); +} + +describe('anonymous process pipes', () => { + test('large concurrent streams preserve backpressure and output before close', async () => { + const bytes = Buffer.alloc(2 * 1024 * 1024, 'p'); + await Promise.all(Array.from({ length: 8 }, async () => { + const child = spawnPipeProcess('/bin/sh', ['-c', 'cat; printf stderr >&2']); + const closed = once(child, 'close'); + const output = collect(child.stdout); + const errors = collect(child.stderr); + await pipeline(Readable.from([bytes]), child.stdin); + const [code] = await closed; + expect(code).toBe(0); + expect(await output).toEqual(bytes); + expect((await errors).toString()).toBe('stderr'); + })); + }, 15000); + + test.skipIf(process.platform !== 'linux')('Linux children receive FIFO stdin, stdout and stderr', async () => { + const child = spawnPipeProcess('/bin/sh', ['-c', 'test -p /proc/self/fd/0 && test -p /proc/self/fd/1 && test -p /proc/self/fd/2']); + child.stdin.end(); + const [code] = await once(child, 'close'); + if (process.platform === 'linux') expect(code).toBe(0); + }); + + test('cancellation releases a blocked stdin writer', async () => { + const child = spawnPipeProcess('/bin/sleep', ['30']); + const closed = once(child, 'close'); + const writing = pipeline(Readable.from([Buffer.alloc(2 * 1024 * 1024)]), child.stdin); + // Observe immediately: cancellation may reject before the exit listener. + const rejected = writing.then(() => false, () => true); + await new Promise(resolve => setTimeout(resolve, 20)); + child.kill('SIGKILL'); + await closed; + expect(await rejected).toBe(true); + expect(child.stdin.destroyed).toBe(true); + }, 5000); + + test('spawn failures and killed children release every pipe', async () => { + const ownedPipes = new Set(); + const addon = process.platform === 'linux' + ? require(process.env.SANDBOX_PIPE_ADDON || path.resolve(__dirname, '..', '.build', 'anonymous-pipes.node')) : undefined; + const create = addon?.createPipe; + if (addon) addon.createPipe = () => { + const fds = create(); + for (const fd of fds) ownedPipes.add(fs.readlinkSync(`/proc/self/fd/${fd}`)); + return fds; + }; + try { + for (let n = 0; n < 12; n++) { + const missing = spawnPipeProcess('/missing-codeapi-executable'); + const missingClosed = new Promise(resolve => missing.once('close', () => resolve())); + const error = await new Promise(resolve => missing.once('error', resolve)); + expect(error.code).toBe('ENOENT'); + await missingClosed; + const child = spawnPipeProcess('/bin/sleep', ['30']); + const closed = once(child, 'close'); + child.kill('SIGKILL'); + await closed; + expect(child.stdin.destroyed).toBe(true); + expect(child.stdout.destroyed).toBe(true); + expect(child.stderr.destroyed).toBe(true); + } + } finally { + if (addon) addon.createPipe = create; + } + if (process.platform === 'linux') { + // Runtime watcher/pid descriptors can appear asynchronously. Assert the + // exact pipe inodes we allocated, including duplicates held by FileSink. + const survivors = fs.readdirSync('/proc/self/fd').flatMap(fd => { + try { const inode = fs.readlinkSync(`/proc/self/fd/${fd}`); return ownedPipes.has(inode) ? [inode] : []; } + catch { return []; } + }); + expect(survivors).toEqual([]); + } + }, 10000); +}); diff --git a/api/src/pipe-process.ts b/api/src/pipe-process.ts new file mode 100644 index 00000000..a4bc6b84 --- /dev/null +++ b/api/src/pipe-process.ts @@ -0,0 +1,116 @@ +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import * as net from 'node:net'; +import { spawn, type SpawnOptions, type ChildProcessWithoutNullStreams } from 'node:child_process'; +import { Readable, Writable } from 'node:stream'; + +let native: { createPipe(): [number, number] } | undefined; +function endpoints(): [number, number] { + native ??= require(process.env.SANDBOX_PIPE_ADDON || path.resolve(__dirname, '..', '.build', 'anonymous-pipes.node')); + return native!.createPipe(); +} +function close(fd: number): void { try { fs.closeSync(fd); } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'EBADF') throw error; } } +function readable(fd: number): Readable { + if (typeof Bun === 'undefined') return new net.Socket({ fd, readable: true, writable: false }); + const stream = Readable.fromWeb(Bun.file(fd).stream() as never); + stream.once('close', () => close(fd)); + return stream; +} +function writable(fd: number): Writable { + if (typeof Bun === 'undefined') return new net.Socket({ fd, readable: false, writable: true }); + const sink = Bun.file(fd).writer(); + return new Writable({ + write(chunk, _encoding, callback) { + try { sink.write(chunk); Promise.resolve(sink.flush()).then(() => callback(), callback); } + catch (error) { callback(error as Error); } + }, + final(callback) { + try { Promise.resolve(sink.end()).then(() => callback(), callback); } + catch (error) { callback(error as Error); } + }, + destroy(error, callback) { + // FileSink duplicates the supplied FD; end also releases that duplicate + // on spawn failure/cancellation, when _final was never reached. + try { void Promise.resolve(sink.end()).catch(() => {}); } catch {} + close(fd); callback(error); + }, + }); +} +/** Linux child stdio uses pipe2 endpoints, never the runtime's socketpairs. + * Non-Linux development keeps its native spawn implementation. */ +export function spawnPipeProcess(command: string, args: readonly string[] = [], options: SpawnOptions = {}): ChildProcessWithoutNullStreams { + if (process.platform !== 'linux') return spawn(command, args, options) as ChildProcessWithoutNullStreams; + const requested = options.stdio ?? ['pipe', 'pipe', 'pipe']; + const modes = typeof requested === 'string' ? [requested, requested, requested] : requested; + if (!Array.isArray(modes) || modes.slice(3).some(mode => typeof mode !== 'number')) throw new Error('unsupported anonymous-pipe process stdio'); + const childFds: number[] = []; + const parentFds: Array<[number, number]> = []; + const streams: Array = [null, null, null]; + let child: ReturnType | undefined; + try { + const stdio = Array.from({ length: Math.max(3, modes.length) }, (_, index) => index).map(index => { + const mode = modes[index] ?? 'pipe'; + if (mode !== 'pipe') { + if (mode !== 'ignore' && mode !== 'inherit' && typeof mode !== 'number') throw new Error('unsupported anonymous-pipe process stdio'); + return mode; + } + const [reader, writer] = endpoints(); + const parentFd = index === 0 ? writer : reader; + const childFd = index === 0 ? reader : writer; + childFds.push(childFd); + parentFds.push([index, parentFd]); + return childFd; + }); + child = spawn(command, args, { ...options, stdio }); + for (const [index, fd] of parentFds) streams[index] = index === 0 ? writable(fd) : readable(fd); + Object.defineProperties(child, { + stdin: { value: streams[0] }, stdout: { value: streams[1] }, stderr: { value: streams[2] }, + }); + // Integer stdio descriptors are not owned by ChildProcess. Reproduce its + // close-after-output semantics and release stdin when the child exits. + const outputClosed = Promise.all(streams.slice(1).map(stream => stream + ? new Promise(resolve => stream.once('close', resolve)) : undefined)); + const emit = child.emit; + child.emit = function (event: string | symbol, ...values: unknown[]): boolean { + if (event === 'close') { + void outputClosed.then(() => emit.call(this, event, ...values)); + return true; + } + return emit.call(this, event, ...values); + }; + child.once('exit', () => { + streams[0]?.destroy(); + // Like native spawn, drain output that has no consumer after exit. + for (const stream of streams.slice(1)) { + if (stream && stream.listenerCount('data') === 0 && stream.listenerCount('readable') === 0) (stream as Readable).resume(); + } + }); + child.once('error', () => { for (const stream of streams) stream?.destroy(); }); + return child as ChildProcessWithoutNullStreams; + } catch (error) { + child?.kill('SIGKILL'); + for (const [index, fd] of parentFds) { if (streams[index]) streams[index]!.destroy(); else close(fd); } + throw error; + } finally { for (const fd of childFds) close(fd); } +} + +export function toolCallPipes(): { child: [number, number]; reader: Readable; writer: Writable; closeChild(): void } { + const [requestReader, requestWriter] = endpoints(); + let responseReader: number, responseWriter: number; + try { [responseReader, responseWriter] = endpoints(); } + catch (error) { close(requestReader); close(requestWriter); throw error; } + let reader: Readable | undefined; + let writer: Writable | undefined; + try { + reader = readable(requestReader); + writer = writable(responseWriter); + let childClosed = false; + return { child: [requestWriter, responseReader], reader, writer, + closeChild() { if (!childClosed) { childClosed = true; close(requestWriter); close(responseReader); } } }; + } catch (error) { + if (reader) reader.destroy(); else close(requestReader); + if (writer) writer.destroy(); else close(responseWriter); + close(requestWriter); close(responseReader); + throw error; + } +} diff --git a/api/src/session-checkpoint.ts b/api/src/session-checkpoint.ts index 8ddd7390..dc642619 100644 --- a/api/src/session-checkpoint.ts +++ b/api/src/session-checkpoint.ts @@ -1,4 +1,4 @@ -import { spawn } from 'child_process'; +import { spawnPipeProcess as spawn } from './pipe-process'; import * as fsp from 'fs/promises'; import * as os from 'os'; import * as path from 'path'; diff --git a/api/src/session-inputs.test.ts b/api/src/session-inputs.test.ts index 639ccadd..6efdb610 100644 --- a/api/src/session-inputs.test.ts +++ b/api/src/session-inputs.test.ts @@ -151,6 +151,7 @@ describe('pushed input cache', () => { const readOnly = cachedInputResponse((await openCachedInput('s1', 'ro'))!); expect(readOnly.headers.get('x-read-only')).toBe('true'); + await readOnly.text(); // Consume the response so its FileHandle is closed. }); test('an opened cache hit remains readable when concurrent pruning unlinks its path', async () => { diff --git a/api/src/spec-guard.c b/api/src/spec-guard.c index 31f153f9..4ce56de4 100644 --- a/api/src/spec-guard.c +++ b/api/src/spec-guard.c @@ -34,8 +34,10 @@ # endif #endif +static int first_untrusted_fd = 3; + static void close_fd_loop(long start, long end) { - if (start < 3) start = 3; + if (start < first_untrusted_fd) start = first_untrusted_fd; if (end > INT_MAX) end = INT_MAX; if (start > end) return; for (long fd = start; fd <= end; fd++) { @@ -54,7 +56,7 @@ static bool close_proc_fds(void) { char *end = NULL; long fd = strtol(entry->d_name, &end, 10); if (end == entry->d_name || *end != '\0') continue; - if (fd < 3 || fd > (long)INT_MAX) continue; + if (fd < first_untrusted_fd || fd > (long)INT_MAX) continue; if ((int)fd == dir_fd) continue; /* don't close our iterator */ close((int)fd); } @@ -62,7 +64,8 @@ static bool close_proc_fds(void) { return true; } -/* Close every file descriptor >= 3 (preserve stdin/stdout/stderr) before +/* Close every unrelated descriptor (preserve stdin/stdout/stderr and, + * only with --tool-call-pipes, the validated pipe endpoints 3/4) before * the user command runs. The bug this prevents: * * (1) sandbox API / proxy / NsJail allocate FDs in the runner; @@ -86,7 +89,7 @@ static bool close_proc_fds(void) { */ static void close_inherited_fds(void) { #if defined(__NR_close_range) && !defined(SPEC_GUARD_TEST_DISABLE_CLOSE_RANGE) - long rc = syscall(__NR_close_range, (unsigned int)3, ~0U, 0u); + long rc = syscall(__NR_close_range, (unsigned int)first_untrusted_fd, ~0U, 0u); if (rc == 0) return; /* ENOSYS on pre-5.9 kernels, EINVAL on some odd builds. Fall through. */ #endif @@ -123,9 +126,24 @@ int main(int argc, char *argv[]) { umask(0077); + int command = 1; + if (strcmp(argv[1], "--tool-call-pipes") == 0) { + struct stat writer, reader; + int write_flags = fcntl(3, F_GETFL), read_flags = fcntl(4, F_GETFL); + if (argc < 3 || fstat(3, &writer) != 0 || fstat(4, &reader) != 0 + || !S_ISFIFO(writer.st_mode) || !S_ISFIFO(reader.st_mode) + || write_flags < 0 || read_flags < 0 + || (write_flags & O_ACCMODE) != O_WRONLY + || (read_flags & O_ACCMODE) != O_RDONLY) { + fprintf(stderr, "invalid tool-call pipes\n"); + return 1; + } + first_untrusted_fd = 5; + command = 2; + } close_inherited_fds(); - execvp(argv[1], &argv[1]); + execvp(argv[command], &argv[command]); perror("exec"); return 1; } diff --git a/api/src/supervisor-policy.c b/api/src/supervisor-policy.c new file mode 100644 index 00000000..b2091927 --- /dev/null +++ b/api/src/supervisor-policy.c @@ -0,0 +1,60 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#if defined(__x86_64__) +#define NATIVE_ARCH AUDIT_ARCH_X86_64 +#elif defined(__aarch64__) +#define NATIVE_ARCH AUDIT_ARCH_AARCH64 +#else +#error Unsupported supervisor architecture +#endif +#define DENY(call) BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_##call, 0, 1), BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM) +int main(int argc, char **argv) { + if (argc < 2) return 125; + struct sock_filter instructions[] = { + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, arch)), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, NATIVE_ARCH, 1, 0), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, nr)), +#ifdef __x86_64__ + BPF_JUMP(BPF_JMP | BPF_JSET | BPF_K, 0x40000000, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_KILL_PROCESS), +#endif + DENY(socketpair), DENY(sendmsg), DENY(sendmmsg), + DENY(io_uring_setup), DENY(io_uring_enter), DENY(io_uring_register), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, __NR_socket, 0, 3), + BPF_STMT(BPF_LD | BPF_W | BPF_ABS, offsetof(struct seccomp_data, args[0])), + BPF_JUMP(BPF_JMP | BPF_JEQ | BPF_K, AF_UNIX, 0, 1), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ERRNO | EPERM), + BPF_STMT(BPF_RET | BPF_K, SECCOMP_RET_ALLOW), + }; + struct sock_fprog program = { .len = sizeof(instructions) / sizeof(instructions[0]), .filter = instructions }; + if (prctl(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) != 0 + || prctl(PR_SET_SECCOMP, SECCOMP_MODE_FILTER, &program) != 0) { + perror("sandbox supervisor policy"); return 125; + } + /* glibc's resolver batches queries with sendmmsg. Use sequential sendto + * queries so hostname-based API/broker traffic needs no policy exception. */ + const char *existing = getenv("RES_OPTIONS"); + char *resolver_options = NULL; + if (asprintf(&resolver_options, "%s%ssingle-request", existing ? existing : "", + existing && *existing ? " " : "") < 0) { + perror("sandbox supervisor DNS options"); return 125; + } + if (setenv("RES_OPTIONS", resolver_options, 1) != 0) { + perror("sandbox supervisor DNS options"); free(resolver_options); return 125; + } + free(resolver_options); + execvp(argv[1], &argv[1]); + perror("sandbox supervisor exec"); return 125; +} diff --git a/api/src/tool-call-pipe-bridge.c b/api/src/tool-call-pipe-bridge.c new file mode 100644 index 00000000..af8c2af0 --- /dev/null +++ b/api/src/tool-call-pipe-bridge.c @@ -0,0 +1,131 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include +#ifdef __linux__ +#include +#endif + +/* Both broker-facing and job-facing transports are anonymous pipes. + * FD 3 writes requests; FD 4 reads responses. The relay retains bounded + * buffering and detects lost readers even with no queued response. */ +struct flow { + int source, destination; + unsigned char bytes[65536]; + size_t start, length; +}; + +static int nonblock(int fd) { + int flags = fcntl(fd, F_GETFL); + return flags < 0 ? -1 : fcntl(fd, F_SETFL, flags | O_NONBLOCK); +} + +static int transfer(struct flow *flow, short readable, short writable) { + if (writable && flow->length) { + ssize_t n = write(flow->destination, flow->bytes + flow->start, flow->length); + if (n > 0) { + flow->start += (size_t)n; + flow->length -= (size_t)n; + if (!flow->length) flow->start = 0; + } else if (n < 0 && errno != EINTR && errno != EAGAIN) return -1; + } + if (flow->source >= 0 && readable && !flow->length) { + ssize_t n = read(flow->source, flow->bytes, sizeof(flow->bytes)); + if (n > 0) flow->length = (size_t)n; + else if (n == 0) flow->source = -1; + else if (errno != EINTR && errno != EAGAIN) return -1; + } + return 0; +} + +static int parent_death(pid_t parent) { +#ifdef __linux__ + if (prctl(PR_SET_PDEATHSIG, SIGKILL) != 0) return -1; +#endif + return getppid() == parent ? 0 : -1; +} + +int main(int argc, char **argv) { + if (argc < 2) return 125; + pid_t broker = getppid(); + if (parent_death(broker) != 0) return 125; + /* The API also enters here to exec Node with a parent-death signal. + * Without this, an API crash could orphan a still-running broker. */ + if (strcmp(argv[1], "--broker") == 0) { + if (argc < 3) return 125; + execvp(argv[2], &argv[2]); + perror("tool-call broker exec"); + return 125; + } + if (strcmp(argv[1], "--job-pipes") != 0 || argc < 3) return 125; + int request_upstream = fcntl(3, F_DUPFD_CLOEXEC, 5); + int response_upstream = fcntl(4, F_DUPFD_CLOEXEC, 5); + if (request_upstream < 0 || response_upstream < 0) return 125; + close(3); close(4); + int requests[2], responses[2]; + if (pipe2(requests, O_CLOEXEC) != 0 || pipe2(responses, O_CLOEXEC) != 0) return 125; + pid_t relay = getpid(); + pid_t child = fork(); + if (child < 0) return 125; + if (child == 0) { + if (parent_death(relay) != 0 || dup2(requests[1], 3) < 0 + || dup2(responses[0], 4) < 0) _exit(125); + int fds[] = {request_upstream, response_upstream, requests[0], requests[1], responses[0], responses[1]}; + for (unsigned i = 0; i < sizeof(fds) / sizeof(fds[0]); i++) { + if (fds[i] > 4) close(fds[i]); + } + execvp(argv[2], &argv[2]); + perror("tool-call pipe exec"); + _exit(125); + } + close(requests[1]); + close(responses[0]); + signal(SIGPIPE, SIG_IGN); + if (nonblock(request_upstream) < 0 || nonblock(response_upstream) < 0 || nonblock(requests[0]) < 0 || nonblock(responses[1]) < 0) { + kill(child, SIGKILL); + waitpid(child, NULL, 0); + return 125; + } + struct flow request = {.source = requests[0], .destination = request_upstream}; + struct flow response = {.source = response_upstream, .destination = responses[1]}; + int status = 125 << 8; + for (;;) { + pid_t done = waitpid(child, &status, WNOHANG); + if (done == child) break; + if (done < 0 && errno != EINTR) return 125; + struct pollfd pollfds[] = { + /* HUP is reported even with events=0. Omit buffered sources + * until their destinations drain, or a closed writer spins poll. */ + {.fd = response.length ? -1 : response_upstream, .events = POLLIN}, + {.fd = request.length ? -1 : request.source, .events = POLLIN}, + {.fd = responses[1], .events = response.length ? POLLOUT : 0}, + {.fd = request_upstream, .events = request.length ? POLLOUT : 0}, + }; + int result = poll(pollfds, 4, 100); + if (result < 0 && errno == EINTR) continue; + if (result < 0 || (pollfds[0].revents & (POLLERR | POLLNVAL)) + || (pollfds[1].revents & (POLLERR | POLLNVAL)) + || (pollfds[2].revents & (POLLERR | POLLHUP | POLLNVAL)) + || (pollfds[3].revents & (POLLERR | POLLHUP | POLLNVAL)) + || transfer(&request, pollfds[1].revents & (POLLIN | POLLHUP), pollfds[3].revents & POLLOUT) < 0 + || transfer(&response, pollfds[0].revents & (POLLIN | POLLHUP), pollfds[2].revents & POLLOUT) < 0 || response.source < 0) { + /* EOF/error terminates the entire invocation, including an + * abandoned tool call. Never leave an orphaned NsJail monitor. */ + if (waitpid(child, &status, WNOHANG) == child) break; + kill(child, SIGKILL); + while (waitpid(child, &status, 0) < 0 && errno == EINTR) {} + break; + } + } + close(request_upstream); + close(response_upstream); + close(requests[0]); + close(responses[1]); + return WIFEXITED(status) ? WEXITSTATUS(status) : 128 + WTERMSIG(status); +} diff --git a/api/src/tool-call-pipe-broker.test.ts b/api/src/tool-call-pipe-broker.test.ts new file mode 100644 index 00000000..a55c6c33 --- /dev/null +++ b/api/src/tool-call-pipe-broker.test.ts @@ -0,0 +1,158 @@ +import { afterEach, describe, expect, test, spyOn } from 'bun:test'; +import * as http from 'node:http'; +import * as net from 'node:net'; +import { once } from 'node:events'; +import { MAX_FRAME_BYTES, serveToolCallPipe } from './tool-call-pipe-broker'; + +const cleanups: (() => void)[] = []; +afterEach(() => { for (const close of cleanups.splice(0).reverse()) close(); }); + +async function setup(handler: http.RequestListener, options = {}, target?: string): Promise<{ + client: net.Socket; replies: () => Promise; send: (value: unknown) => void; +}> { + const upstream = http.createServer(handler); + upstream.listen(0, '127.0.0.1'); + await once(upstream, 'listening'); + cleanups.push(() => { upstream.closeAllConnections(); upstream.close(); }); + const server = net.createServer(channel => { + cleanups.push(serveToolCallPipe(channel, target ?? `http://127.0.0.1:${(upstream.address() as net.AddressInfo).port}`, options)); + }); + server.listen(0, '127.0.0.1'); + await once(server, 'listening'); + cleanups.push(() => server.close()); + const client = net.createConnection((server.address() as net.AddressInfo).port, '127.0.0.1'); + await once(client, 'connect'); + cleanups.push(() => client.destroy()); + let buffered = Buffer.alloc(0); + const queued: any[] = []; + const waiters: ((response: any) => void)[] = []; + client.on('data', bytes => { + buffered = Buffer.concat([buffered, typeof bytes === 'string' ? Buffer.from(bytes) : bytes]); + while (buffered.length >= 4 && buffered.length >= 4 + buffered.readUInt32BE()) { + const size = buffered.readUInt32BE(); + const value = JSON.parse(buffered.subarray(4, 4 + size).toString()); + buffered = buffered.subarray(4 + size); + const waiter = waiters.shift(); + if (waiter) waiter(value); else queued.push(value); + } + }); + return { client, replies: () => queued.length ? Promise.resolve(queued.shift()) : new Promise(resolve => waiters.push(resolve)), + send: value => { const frame = Buffer.from(JSON.stringify(value)); const prefix = Buffer.alloc(4); prefix.writeUInt32BE(frame.length); client.write(Buffer.concat([prefix, frame])); } }; +} + +const claims = { 'x-execution-id': 'exec', 'x-callback-token': 'token', 'x-tool-call-id': 'call' }; +const frame = (id: number) => ({ id, headers: claims, body: '{"tool_name":"echo","input":{}}' }); + +describe('per-invocation tool-call pipe broker', () => { + test('forwards only its fixed route and whitelisted claims, preserving JSON errors', async () => { + let observed = false; + const harness = await setup((req, res) => { + observed = true; + expect(req.url).toBe('/tool-call'); + expect(req.method).toBe('POST'); + expect(req.headers['x-callback-token']).toBe('token'); + expect(req.headers['x-forged']).toBeUndefined(); + req.resume(); + res.writeHead(404).end('{"success":false,"error":"Session not found"}'); + }); + harness.send({ ...frame(7), path: '/admin', headers: { ...claims, 'x-forged': 'secret' } }); + expect(await harness.replies()).toEqual({ id: 7, status: 404, body: '{"success":false,"error":"Session not found"}' }); + expect(observed).toBe(true); + }); + + test('isolates identical frame IDs across invocation channels', async () => { + const first = await setup((_req, res) => res.end('first')); + const second = await setup((_req, res) => res.end('second')); + first.send(frame(1)); + second.send(frame(1)); + const replies = await Promise.all([first.replies(), second.replies()]); + expect(replies.map(reply => reply.body)).toEqual(['first', 'second']); + }); + + test('rejects missing claims before opening an upstream connection', async () => { + let count = 0; + const harness = await setup((_req, res) => { count++; res.end(); }); + harness.send({ ...frame(1), headers: { ...claims, 'x-callback-token': ', , ' } }); + expect((await harness.replies()).status).toBe(400); + expect(count).toBe(0); + }); + + test('returns redirects without following them', async () => { + let count = 0; + const harness = await setup((_req, res) => { count++; res.writeHead(302, { Location: '/credential-sink' }).end('redirect'); }); + harness.send(frame(1)); + expect((await harness.replies()).status).toBe(302); + expect(count).toBe(1); + }); + + test('bounds outstanding upstream calls and aborts them when the channel closes', async () => { + let upstreamClosed = false; + let received: () => void; + const started = new Promise(resolve => { received = resolve; }); + // Observe the actual TCP peer: Bun 1.3.14's HTTP IncomingMessage socket + // does not emit close when an unfinished response loses its connection. + const upstream = net.createServer(socket => { + socket.once('close', () => { upstreamClosed = true; }); + socket.once('data', () => received()); + socket.resume(); + }); + upstream.listen(0, '127.0.0.1'); + await once(upstream, 'listening'); + cleanups.push(() => upstream.close()); + const harness = await setup(() => {}, { maxActiveRequests: 1 }, + `http://127.0.0.1:${(upstream.address() as net.AddressInfo).port}`); + harness.send(frame(1)); + await started; + harness.send(frame(2)); + expect((await harness.replies()).status).toBe(429); + harness.client.destroy(); + for (let n = 0; n < 100 && !upstreamClosed; n++) await Bun.sleep(5); + expect(upstreamClosed).toBe(true); + }); + + test.each(['invalid claims', 'invalid body', 'active limit'])('closes a drained channel when %s exhausts the frame budget', async kind => { + const clock = spyOn(Date, 'now').mockReturnValue(1_000); + try { + let count = 0; + const harness = await setup(req => { count++; req.resume(); }, { maxActiveRequests: 1 }); + const ended = once(harness.client, 'close'); + for (let id = 1; id <= 65; id++) { + harness.send(kind === 'invalid claims' ? { ...frame(id), headers: {} } + : kind === 'invalid body' ? { ...frame(id), body: null } : frame(id)); + } + await ended; // setup drains replies continuously, so backpressure cannot hide the bypass. + expect(count).toBeLessThanOrEqual(kind === 'active limit' ? 1 : 0); + } finally { clock.mockRestore(); } + }); + + test('enforces an absolute deadline on a stalled upstream', async () => { + const harness = await setup(req => req.resume(), { requestTimeoutMs: 30 }); + harness.send(frame(1)); + expect((await harness.replies()).status).toBe(504); + }); + + test('rejects oversized upstream responses without buffering an unbounded body', async () => { + const harness = await setup((_req, res) => res.end(Buffer.alloc(MAX_FRAME_BYTES + 1, 'x'))); + harness.send(frame(1)); + const response = await harness.replies(); + expect(response.status).toBe(502); + expect(response.body).toContain('too large'); + }); + + test('closes an oversized frame before any upstream request', async () => { + let count = 0; + const harness = await setup((_req, res) => { count++; res.end(); }); + const ended = once(harness.client, 'close'); + const prefix = Buffer.alloc(4); prefix.writeUInt32BE(MAX_FRAME_BYTES + 1); + harness.client.write(prefix); + await ended; + expect(count).toBe(0); + }); + + test('bounds a partial-header slow writer', async () => { + const harness = await setup((_req, res) => res.end(), { frameTimeoutMs: 30 }); + const ended = once(harness.client, 'close'); + harness.client.write(Buffer.from([0])); + await ended; + }); +}); diff --git a/api/src/tool-call-pipe-broker.ts b/api/src/tool-call-pipe-broker.ts new file mode 100644 index 00000000..d81524c2 --- /dev/null +++ b/api/src/tool-call-pipe-broker.ts @@ -0,0 +1,172 @@ +import { spawnPipeProcess as spawn, toolCallPipes } from './pipe-process'; +import { Duplex, Readable, Writable } from 'node:stream'; +import * as http from 'node:http'; +import * as https from 'node:https'; + +export const MAX_FRAME_BYTES = 1024 * 1024; +const HEADER_NAMES = ['x-execution-id', 'x-callback-token', 'x-tool-call-id'] as const; + +/** One broker/channel per authorized invocation. The target and route come + * from trusted runner configuration; the job supplies only PTC claims/body. + * The upstream still authenticates those claims and enforces tool budgets. */ +export function serveToolCallPipe(channel: Duplex, rawTarget: string, options: { + requestTimeoutMs?: number; frameTimeoutMs?: number; maxActiveRequests?: number; + onFailure?: () => void; +} = {}): () => void { + const target = new URL(rawTarget.includes('://') ? rawTarget : `http://${rawTarget}`); + if (!['http:', 'https:'].includes(target.protocol) || target.username || target.password) { + throw new Error('invalid tool-call broker target'); + } + const transport = target.protocol === 'https:' ? https : http; + const requests = new Map(); + const timeout = options.requestTimeoutMs ?? 300_000; + const frameTimeout = options.frameTimeoutMs ?? 5_000; + const maxActive = options.maxActiveRequests ?? 16; + let closed = false; + let frameTimer: ReturnType | undefined; + let drainTimer: ReturnType | undefined; + let header = Buffer.alloc(4); + let offset = 0; + let frame: Buffer | undefined; + let tokens = 64; + let lastRefill = Date.now(); + const close = (): void => { + if (closed) return; + closed = true; + clearTimeout(frameTimer); + clearTimeout(drainTimer); + for (const request of requests.values()) request.destroy(); + requests.clear(); + channel.destroy(); + }; + const fail = (): void => { close(); options.onFailure?.(); }; + const reply = (id: number, status: number, body: string): void => { + if (closed) return; + let bytes = Buffer.from(JSON.stringify({ id, status, body })); + if (bytes.length > MAX_FRAME_BYTES) { + bytes = Buffer.from(JSON.stringify({ id, status: 502, body: JSON.stringify({ success: false, error: 'tool-call response too large' }) })); + } + const prefix = Buffer.alloc(4); + prefix.writeUInt32BE(bytes.length); + if (!channel.write(Buffer.concat([prefix, bytes]))) { + channel.pause(); + drainTimer ??= setTimeout(fail, frameTimeout); + } + }; + const reject = (id: number, status: number, error: string): void => reply(id, status, JSON.stringify({ success: false, error })); + const accept = (bytes: Buffer): void => { + // Charge every complete frame before JSON/claim validation, including + // rejected concurrency requests. Exhaustion closes the capability rather + // than generating an unlimited stream of cheap 429/400 replies. + const now = Date.now(); + tokens = Math.min(64, tokens + Math.max(0, now - lastRefill) * 0.02); + lastRefill = now; + if (tokens < 1) { fail(); return; } + tokens -= 1; + let parsed: { id?: unknown; headers?: unknown; body?: unknown }; + try { parsed = JSON.parse(bytes.toString('utf8')); } catch { fail(); return; } + if (!parsed || typeof parsed !== 'object' || !Number.isSafeInteger(parsed.id) + || (parsed.id as number) < 1 || requests.has(parsed.id as number)) { fail(); return; } + const id = parsed.id as number; + if (!parsed.headers || typeof parsed.headers !== 'object' || Array.isArray(parsed.headers) || typeof parsed.body !== 'string') { + reject(id, 400, 'invalid tool-call frame'); return; + } + const headers: http.OutgoingHttpHeaders = { 'content-type': 'application/json', connection: 'close' }; + for (const name of HEADER_NAMES) { + const value = (parsed.headers as Record)[name]; + if (typeof value !== 'string' || !value.replace(/[,\s]/g, '') || value.length > 8192 || /[^\x20-\x7e]/.test(value)) { + reject(id, 400, 'missing or invalid tool-call claims'); return; + } + headers[name] = value; + } + if (requests.size >= maxActive) { reject(id, 429, 'tool-call request budget exceeded'); return; } + const body = Buffer.from(parsed.body); + headers['content-length'] = body.length; + let deadline: ReturnType; + let finished = false; + const finish = (status: number, response: string): void => { + if (finished) return; + finished = true; + clearTimeout(deadline); + requests.delete(id); + reply(id, status, response); + }; + const upstream = transport.request({ protocol: target.protocol, hostname: target.hostname, + port: target.port || undefined, method: 'POST', path: '/tool-call', headers, agent: false }, response => { + const chunks: Buffer[] = []; + let size = 0; + response.on('data', (chunk: Buffer) => { + size += chunk.length; + if (size > MAX_FRAME_BYTES) { + finish(502, JSON.stringify({ success: false, error: 'tool-call response too large' })); + upstream.destroy(); + } else chunks.push(chunk); + }); + response.on('end', () => finish(response.statusCode ?? 502, Buffer.concat(chunks).toString('utf8'))); + response.on('error', () => finish(502, JSON.stringify({ success: false, error: 'tool-call upstream disconnected' }))); + response.on('aborted', () => finish(502, JSON.stringify({ success: false, error: 'tool-call upstream disconnected' }))); + }); + requests.set(id, upstream); + deadline = setTimeout(() => { + finish(504, JSON.stringify({ success: false, error: 'tool-call upstream timeout' })); + upstream.destroy(); + }, timeout); + upstream.on('error', () => finish(502, JSON.stringify({ success: false, error: 'tool-call upstream unavailable' }))); + upstream.end(body); + }; + channel.on('data', (chunk: Buffer) => { + let cursor = 0; + while (cursor < chunk.length && !closed) { + frameTimer ??= setTimeout(fail, frameTimeout); + const buffer = frame ?? header; + const count = Math.min(buffer.length - offset, chunk.length - cursor); + chunk.copy(buffer, offset, cursor, cursor + count); + offset += count; + cursor += count; + if (offset !== buffer.length) continue; + offset = 0; + if (!frame) { + const length = header.readUInt32BE(); + if (length < 1 || length > MAX_FRAME_BYTES) { fail(); break; } + frame = Buffer.allocUnsafe(length); + } else { + const complete = frame; + frame = undefined; + clearTimeout(frameTimer); + frameTimer = undefined; + accept(complete); + } + } + }); + channel.on('drain', () => { clearTimeout(drainTimer); drainTimer = undefined; channel.resume(); }); + channel.on('error', fail); + channel.on('end', fail); + channel.on('close', close); + return close; +} + +// Node starts only after restore, on an authenticated job's execution path. +// Never create a persistent listener or snapshot Node's TLS process state. +if (require.main === module) { + const target = process.env.SANDBOX_FORWARD_TARGET?.trim(); + if (!target || !process.argv[2]) throw new Error('tool-call pipe broker is not configured'); + const pipes = toolCallPipes(); + let child: ReturnType; + try { + child = spawn(process.env.TCS_PIPE_BRIDGE || '/usr/local/bin/tool-call-pipe-bridge', ['--job-pipes', ...process.argv.slice(2)], { + stdio: [0, 1, 2, ...pipes.child], + }); + } catch (error) { + pipes.reader.destroy(); pipes.writer.destroy(); + throw error; + } finally { pipes.closeChild(); } + const channel = Duplex.fromWeb({ readable: Readable.toWeb(pipes.reader), writable: Writable.toWeb(pipes.writer) }); + const stop = serveToolCallPipe(channel, target, { + requestTimeoutMs: Math.min(600_000, Math.max(5_000, Number(process.env.TCS_REQUEST_TIMEOUT_MS) || 300_000)), + onFailure: () => child.kill('SIGKILL'), + }); + child.on('error', () => { stop(); process.exitCode = 125; }); + child.on('exit', (code, signal) => { stop(); process.exitCode = code ?? (signal === 'SIGKILL' ? 137 : 125); }); + process.once('SIGTERM', () => { stop(); child.kill('SIGKILL'); }); + process.once('SIGINT', () => { stop(); child.kill('SIGKILL'); }); +} diff --git a/apparmor/README.md b/apparmor/README.md index f0921d16..934c3b89 100644 --- a/apparmor/README.md +++ b/apparmor/README.md @@ -29,11 +29,11 @@ the sandbox-runner service and grants the explicit capability set instead: ```bash # Baseline (privileged: true) docker compose -f docker-compose.local-dev.yml up -d -./test-sandbox.sh +bash ./test-sandbox.sh # Capability-restricted (no privileged, explicit caps) docker compose -f docker-compose.local-dev.yml -f your-capability-overlay.yml up -d -./test-sandbox.sh +bash ./test-sandbox.sh ``` **Note:** AppArmor profiles require native Linux with AppArmor enabled. WSL2 and Docker Desktop do not support AppArmor. The capability-restricted mode works without AppArmor. @@ -49,7 +49,7 @@ minikube start --cpus=4 --memory=8192 # Port forward and test kubectl port-forward deploy/codeapi-worker-sandbox 2000:2000 -./test-sandbox.sh +bash ./test-sandbox.sh ``` ## Kubernetes Deployment with AppArmor diff --git a/build-packages.sh b/build-packages.sh old mode 100755 new mode 100644 index 2ae04d02..1d635297 --- a/build-packages.sh +++ b/build-packages.sh @@ -10,7 +10,7 @@ # Environment Variables: # PYTHON_VERSION=3.14.4 # Python version to install # NODE_VERSION=24.15.0 # Node.js version to install -# BUN_VERSION=1.3.14 # Bun version to install +# BUN_VERSION=1.4.2 # Bun version to install # BASH_PACKAGE_VERSION=5.2.0 # Bash package registration version (semver-like x.y.z) # SKIP_PYTHON_PACKAGES=1 # Skip Python pip packages # SKIP_JS_PACKAGES=1 # Skip JavaScript npm packages for Node/Bun @@ -28,7 +28,7 @@ cd "$SCRIPT_DIR" PYTHON_VERSION="${PYTHON_VERSION:-3.14.4}" NODE_VERSION="${NODE_VERSION:-24.15.0}" -BUN_VERSION="${BUN_VERSION:-1.3.14}" +BUN_VERSION="${BUN_VERSION:-1.4.2}" PACKAGES_DIR="./data/pkgs" JS_PACKAGE_MANIFEST="${JS_PACKAGE_MANIFEST:-${SCRIPT_DIR}/javascript-packages.txt}" @@ -118,6 +118,9 @@ install_python() { rm -rf /tmp/Python-${PYTHON_VERSION}* " + docker cp "$SCRIPT_DIR/docker/python-runtime" "$CONTAINER_NAME:/tmp/python-runtime" + docker exec "$CONTAINER_NAME" "${pkg_dest}/bin/python3" /tmp/python-runtime/install.py + docker exec "$CONTAINER_NAME" bash -c "cat > ${pkg_dest}/pkg-info.json << 'EOF' { \"language\": \"python\", diff --git a/docker-compose.yaml b/docker-compose.yaml index ab37fd24..db517b4c 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -14,7 +14,14 @@ services: - CODEAPI_BRIDGE_AUTH_MODE=${CODEAPI_BRIDGE_AUTH_MODE:-paired} - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} + - CODEAPI_BRIDGE_LANE_GIT=${CODEAPI_BRIDGE_LANE_GIT:-false} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} + - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-} + - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0} + - CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=${CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS:-60} + - CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE:-12} + - CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE:-30} + - CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE:-240} - CODEAPI_AUTH_PROVIDER=${CODEAPI_AUTH_PROVIDER:-} - CODEAPI_ALLOW_AUTH_PROVIDER_NONE=${CODEAPI_ALLOW_AUTH_PROVIDER_NONE:-} - CODEAPI_JWT_ISSUER=${CODEAPI_JWT_ISSUER:-} @@ -63,7 +70,14 @@ services: - CODEAPI_BRIDGE_AUTH_MODE=${CODEAPI_BRIDGE_AUTH_MODE:-paired} - CODEAPI_BRIDGE_DYNAMIC_WORKERS=${CODEAPI_BRIDGE_DYNAMIC_WORKERS:-true} - CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS=${CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS:-1} + - CODEAPI_BRIDGE_LANE_GIT=${CODEAPI_BRIDGE_LANE_GIT:-false} - CODEAPI_BRIDGE_WORKER_ID=${CODEAPI_BRIDGE_WORKER_ID:-} + - CODEAPI_BRIDGE_RECOVERY_SERVER_ID=${CODEAPI_BRIDGE_RECOVERY_SERVER_ID:-} + - CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=${CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS:-0} + - CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=${CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS:-60} + - CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE:-12} + - CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE:-30} + - CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE=${CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE:-240} - CODEAPI_AUTH_PROVIDER=${CODEAPI_AUTH_PROVIDER:-} - CODEAPI_JWT_SINGLE_TENANT_ID=${CODEAPI_JWT_SINGLE_TENANT_ID:-} - CODEAPI_TENANT_ISOLATION_STRICT=${CODEAPI_TENANT_ISOLATION_STRICT:-} @@ -213,9 +227,11 @@ services: redis: image: redis:7-alpine container_name: redis - command: redis-server --requirepass localdev + command: redis-server --requirepass localdev --appendonly yes ports: - ${CODEAPI_REDIS_PORT:-16379}:6379 + volumes: + - redis_data:/data minio: image: quay.io/minio/minio @@ -232,3 +248,4 @@ services: volumes: minio_data: + redis_data: diff --git a/docker/Dockerfile.package-init b/docker/Dockerfile.package-init index c94cb8af..745346f8 100644 --- a/docker/Dockerfile.package-init +++ b/docker/Dockerfile.package-init @@ -23,6 +23,7 @@ RUN apt-install \ RUN mkdir -p /pkgs +COPY docker/python-runtime /python-runtime COPY docker/package-init.sh /package-init.sh COPY javascript-packages.txt /javascript-packages.txt RUN chmod +x /package-init.sh diff --git a/docker/Dockerfile.worker-sandbox b/docker/Dockerfile.worker-sandbox index 7156719c..673dd42f 100644 --- a/docker/Dockerfile.worker-sandbox +++ b/docker/Dockerfile.worker-sandbox @@ -39,11 +39,21 @@ RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ && git checkout 66ad78dc361721512fbf9470488e584364c75489 \ && git submodule update --init --recursive -RUN make -j$(nproc) +COPY docker/nsjail-anonymous-control.patch /tmp/nsjail-anonymous-control.patch +# Network-isolated jobs need no libnl transport; removing it avoids sendmsg +# in the supervisor and fails closed for interface/traffic-rule configuration. +RUN git apply /tmp/nsjail-anonymous-control.patch && make -j$(nproc) NL3_EXISTS=no COPY api/src/spec-guard.c /tmp/spec-guard.c +COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c +COPY api/src/anonymous-pipes.c /tmp/anonymous-pipes.c +COPY api/src/supervisor-policy.c /tmp/supervisor-policy.c +COPY --from=node:22-slim /usr/local/include/node /usr/local/include/node COPY docker/rootfs-setup.c /tmp/rootfs-setup.c -RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ +RUN gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I/usr/local/include/node -o /usr/local/lib/anonymous-pipes.node /tmp/anonymous-pipes.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/sandbox-supervisor-policy /tmp/supervisor-policy.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ + && gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup @@ -58,7 +68,7 @@ FROM buildpack-deps:bookworm AS package-builder ENV DEBIAN_FRONTEND=noninteractive ENV PYTHON_VERSION=3.14.4 -ENV BUN_VERSION=1.3.14 +ENV BUN_VERSION=1.4.2 COPY docker/apt-install.sh /usr/local/bin/apt-install RUN chmod +x /usr/local/bin/apt-install @@ -73,6 +83,7 @@ RUN apt-install \ RUN mkdir -p /pkgs +COPY docker/python-runtime /python-runtime COPY docker/package-init.sh /package-init.sh COPY javascript-packages.txt /javascript-packages.txt RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh @@ -80,7 +91,7 @@ RUN chmod +x /package-init.sh && FORCE_REBUILD=true /package-init.sh # ============================================================================ # Stage 2: Build worker # ============================================================================ -FROM oven/bun:1.3.14 AS worker-builder +FROM oven/bun:1.4.2 AS worker-builder WORKDIR /app COPY service/package.json service/bun.lock ./ RUN bun install --frozen-lockfile @@ -93,19 +104,20 @@ RUN bun build ./src/worker-server.ts --minify --outdir .build --target bun --ext # ============================================================================ # Stage 3: Build sandbox API # ============================================================================ -FROM oven/bun:1.3.14 AS sandbox-builder +FROM oven/bun:1.4.2 AS sandbox-builder WORKDIR /app COPY api/package.json ./ RUN bun install --frozen-lockfile COPY api/src ./src COPY shared /shared COPY api/tsconfig.json ./ -RUN bun build ./src/index.ts --minify --outdir .build --target bun --external '@opentelemetry/*' +RUN bun build ./src/index.ts --minify --outdir .build --target bun --external '@opentelemetry/*' \ + && bun build ./src/tool-call-pipe-broker.ts --target=node --format=cjs --outfile=.build/tool-call-pipe-broker.cjs # ============================================================================ # Stage 4: Build sandbox rootfs (full OS layer for the microVM guest) # ============================================================================ -FROM oven/bun:1.3.14-debian AS sandbox-rootfs +FROM oven/bun:1.4.2-debian AS sandbox-rootfs ENV DEBIAN_FRONTEND=noninteractive @@ -143,7 +155,12 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard +COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/tool-call-pipe-bridge +COPY --from=nsjail-builder /usr/local/bin/sandbox-supervisor-policy /usr/local/bin/sandbox-supervisor-policy +COPY --from=nsjail-builder /usr/local/lib/anonymous-pipes.node /usr/local/lib/anonymous-pipes.node +ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node RUN chmod +x /usr/sbin/nsjail +COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node WORKDIR /sandbox_api diff --git a/docker/nsjail-anonymous-control.patch b/docker/nsjail-anonymous-control.patch new file mode 100644 index 00000000..bfce44bf --- /dev/null +++ b/docker/nsjail-anonymous-control.patch @@ -0,0 +1,82 @@ +--- a/subproc.cc ++++ b/subproc.cc +@@ -510,6 +510,12 @@ + } + + pid_t runChild(nsj_t* nsj, int netfd, int fd_in, int fd_out, int fd_err) { ++ // This runner uses anonymous control pipes. Descriptor-passing user-mode ++ // networking and seccomp notification transports are intentionally unsupported. ++ if (nsj->njc.seccomp_unotify() || nsj->njc.has_user_net()) { ++ LOG_E("Descriptor-passing NsJail modes are disabled by runner hardening"); ++ return -1; ++ } + if (!net::limitConns(nsj, netfd)) { + return 0; + } +@@ -535,26 +541,34 @@ + LOG_D("Creating new process with clone flags:%s and exit_signal:SIGCHLD", + cloneFlagsToStr(flags).c_str()); + +- int sv[2]; +- if (socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC, 0, sv) == -1) { +- PLOG_E("socketpair(AF_UNIX, SOCK_STREAM | SOCK_CLOEXEC) failed"); ++ int ready[2], errors[2]; ++ if (pipe2(ready, O_CLOEXEC) == -1) { ++ PLOG_E("pipe2(ready) failed"); + return -1; + } +- int child_fd = sv[0]; +- int parent_fd = sv[1]; ++ if (pipe2(errors, O_CLOEXEC) == -1) { ++ close(ready[0]); close(ready[1]); ++ PLOG_E("pipe2(errors) failed"); ++ return -1; ++ } ++ int child_fd = ready[0]; ++ int parent_fd = ready[1]; + + pid_t pid = cloneProc(flags, SIGCHLD); + if (pid == 0) { + close(parent_fd); ++ close(errors[0]); + newProc(nsj, netfd, fd_in, fd_out, fd_err, child_fd); +- util::writeToFd(child_fd, &kSubprocErrorChar, sizeof(kSubprocErrorChar)); ++ util::writeToFd(errors[1], &kSubprocErrorChar, sizeof(kSubprocErrorChar)); + LOG_F("Launching child process failed"); + } + close(child_fd); ++ close(errors[1]); + if (pid == -1) { + auto saved_errno = errno; + PLOG_W("clone(flags=%s) failed", cloneFlagsToStr(flags).c_str()); + close(parent_fd); ++ close(errors[0]); + errno = saved_errno; + return pid; + } +@@ -562,20 +576,22 @@ + + if (!initParent(nsj, pid, parent_fd)) { + close(parent_fd); ++ close(errors[0]); + return -1; + } ++ close(parent_fd); + + char rcvChar; +- while (util::readFromFd(parent_fd, &rcvChar, sizeof(rcvChar)) == sizeof(rcvChar)) { ++ while (util::readFromFd(errors[0], &rcvChar, sizeof(rcvChar)) == sizeof(rcvChar)) { + if (rcvChar == kSubprocErrorChar) { + LOG_W("Received error message from the child process before it has been " + "executed"); +- close(parent_fd); ++ close(errors[0]); + return -1; + } + } + +- close(parent_fd); ++ close(errors[0]); + return pid; + } + diff --git a/docker/package-init.sh b/docker/package-init.sh index ee8bc548..5110a494 100644 --- a/docker/package-init.sh +++ b/docker/package-init.sh @@ -21,7 +21,7 @@ PYTHON_ALIAS="python${PYTHON_SITE_VERSION}" PYTHON_PACKAGE_INSTALLER="${PYTHON_PACKAGE_INSTALLER:-uv}" UV_VERSION="${UV_VERSION:-0.11.26}" NODE_VERSION="${NODE_VERSION:-24.15.0}" -BUN_VERSION="${BUN_VERSION:-1.3.14}" +BUN_VERSION="${BUN_VERSION:-1.4.2}" BASH_PACKAGE_VERSION="${BASH_PACKAGE_VERSION:-5.2.0}" INSTALL_FAILED=false JS_PACKAGE_MANIFEST="${JS_PACKAGE_MANIFEST:-${SCRIPT_DIR}/javascript-packages.txt}" @@ -87,6 +87,7 @@ echo "==============================================" echo "" packages_ready() { + [ -f "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/.sandbox-pipe-runtime-v1" ] && [ -f "/pkgs/python/${PYTHON_VERSION}/.package-installed" ] && [ -d "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/site-packages/PIL" ] && [ -d "/pkgs/python/${PYTHON_VERSION}/lib/python${PYTHON_SITE_VERSION}/site-packages/markitdown" ] && @@ -138,6 +139,7 @@ cd "Python-${PYTHON_VERSION}" ./configure --prefix="$PKG_DEST" --enable-optimizations 2>/dev/null make -j$(nproc) make install +"${PKG_DEST}/bin/python3" "${SCRIPT_DIR}/python-runtime/install.py" cd /tmp rm -rf /tmp/Python-${PYTHON_VERSION}* diff --git a/docker/python-runtime/_sandbox_multiprocessing.py b/docker/python-runtime/_sandbox_multiprocessing.py new file mode 100644 index 00000000..62fc2362 --- /dev/null +++ b/docker/python-runtime/_sandbox_multiprocessing.py @@ -0,0 +1,47 @@ +"""Duplex multiprocessing connections backed by two anonymous pipes.""" +import os +from multiprocessing.connection import Connection +from multiprocessing.context import reduction + + +class DuplexConnection(Connection): + def __init__(self, reader, writer): + super().__init__(reader) + self._writer = writer + + def _send(self, data): + remaining = memoryview(data) + while remaining: + written = os.write(self._writer, remaining) + remaining = remaining[written:] + + def _close(self): + try: + os.close(self._handle) + finally: + os.close(self._writer) + + +def rebuild(reader, writer): + return DuplexConnection(reader.detach(), writer.detach()) + + +def reduce_connection(connection): + connection._check_closed() + return rebuild, (reduction.DupFd(connection.fileno()), + reduction.DupFd(connection._writer)) + + +reduction.register(DuplexConnection, reduce_connection) + + +def duplex_pipe(): + first_reader, first_writer = os.pipe() + try: + second_reader, second_writer = os.pipe() + except BaseException: + os.close(first_reader) + os.close(first_writer) + raise + return (DuplexConnection(first_reader, second_writer), + DuplexConnection(second_reader, first_writer)) diff --git a/docker/python-runtime/_sandbox_pipe.py b/docker/python-runtime/_sandbox_pipe.py new file mode 100644 index 00000000..645dbc2a --- /dev/null +++ b/docker/python-runtime/_sandbox_pipe.py @@ -0,0 +1,32 @@ +"""Pipe-only Linux runtime IPC; seccomp still enforces socket denial.""" +import os + + +class WakeupPipe: + def __init__(self, fd): + self._fd = fd + + def __del__(self): + self.close() + + def fileno(self): + return self._fd + + def setblocking(self, blocking): + os.set_blocking(self._fd, blocking) + + def recv(self, size): + return os.read(self._fd, size) + + def send(self, data): + return os.write(self._fd, data) + + def close(self): + fd, self._fd = self._fd, -1 + if fd >= 0: + os.close(fd) + + +def wakeup_pipe(): + reader, writer = os.pipe2(os.O_CLOEXEC | os.O_NONBLOCK) + return WakeupPipe(reader), WakeupPipe(writer) diff --git a/docker/python-runtime/install.py b/docker/python-runtime/install.py new file mode 100644 index 00000000..61194cbb --- /dev/null +++ b/docker/python-runtime/install.py @@ -0,0 +1,41 @@ +"""Apply checked stdlib edits to the interpreter running this installer.""" +from pathlib import Path +import shutil +import sysconfig + +root = Path(sysconfig.get_path('stdlib')) +source = Path(__file__).resolve().parent +edits = [ + ('asyncio/selector_events.py', + ' self._ssock, self._csock = socket.socketpair()', + ' from _sandbox_pipe import wakeup_pipe\n self._ssock, self._csock = wakeup_pipe()'), + ('multiprocessing/connection.py', + ''' if duplex: + s1, s2 = socket.socketpair() + s1.setblocking(True) + s2.setblocking(True) + c1 = Connection(s1.detach()) + c2 = Connection(s2.detach())''', + ''' if duplex: + from _sandbox_multiprocessing import duplex_pipe + return duplex_pipe()'''), +] +prepared = [] +for name, old, new in edits: + path = root / name + text = path.read_text() + if text.count(new) == 1: + continue + if text.count(old) != 1: + raise RuntimeError(f'unsupported Python runtime layout: {path}') + prepared.append((path, text.replace(old, new))) +for name in ('_sandbox_pipe.py', '_sandbox_multiprocessing.py'): + shutil.copyfile(source / name, root / name) + for cached in (root / '__pycache__').glob(Path(name).stem + '.*.pyc'): + cached.unlink() +for path, text in prepared: + path.write_text(text) + # Avoid reusing timestamp-based bytecode after editing installed sources. + for cached in (path.parent / '__pycache__').glob(path.stem + '.*.pyc'): + cached.unlink() +(root / '.sandbox-pipe-runtime-v1').write_text('1\n') diff --git a/docs/SECURITY-MAINTENANCE.md b/docs/SECURITY-MAINTENANCE.md new file mode 100644 index 00000000..f33e509f --- /dev/null +++ b/docs/SECURITY-MAINTENANCE.md @@ -0,0 +1,194 @@ +# Sandbox security maintenance + +The default boundary is NsJail inside a libkrun guest. Direct NsJail mode +shares the host kernel and is for trusted development, not hostile workloads. + +## Verify deployed patches + +Repository dependencies do not establish production kernel patch status. +Inventory each runner node and deployed runner image: + +- Host OS and running kernel release (`uname -r` on the node). +- Container runtime version and runner image digest. +- Installed `libkrun` and `libkrunfw` packages + (`rpm -q libkrun libkrunfw` inside the runner image). +- Bundled guest kernel release, from an operator-controlled guest inspection. +- Guest-visible nested virtualization features and exposed virtual devices. + +Match kernel packages against distro security advisories, including backports. +Patch both the host kernel and bundled guest kernel. Rebuild and replace runner +images after guest-kernel updates; reboot or use the vendor-supported livepatch +procedure for host fixes. A container rebuild does not update host KVM. + +Priority 2026 checks: + +| Boundary | Advisory | +|---|---| +| Guest kernel; host kernel in direct mode | Bad Epoll, CVE-2026-46242 | +| Guest and host kernels | GhostLock, CVE-2026-43499 | +| Guest kernel; host kernel in direct mode | AF_UNIX garbage collection, CVE-2026-80521 | +| x86 host KVM | CVE-2026-46113, CVE-2026-53359, CVE-2026-64561 | +| ARM host KVM, when virtual ITS is exposed | CVE-2026-46316 | + +PI-futex filtering reduces the GhostLock surface but does not replace patches. +Do not disable ordinary futex or epoll operations: language runtimes use them. +Do not expose nested virtualization to untrusted guests. Confirm the installed +VMM's behavior rather than assuming current upstream defaults match its build. + +The October 2026 Vercel KVM report has no disclosed root cause or affected-version +matrix at the time of this change. These controls are not a verified fix for it. + +## Unix-socket descriptor passing + +The NsJail policy returns EPERM for `sendmsg` and `sendmmsg` to prevent +`SCM_RIGHTS` descriptor passing into the AF_UNIX garbage collector affected by +[CVE-2026-80521](https://www.cve.org/CVERecord?id=CVE-2026-80521). +Seccomp cannot inspect ancillary data behind a userspace pointer, so both calls +are denied even for messages without descriptors and for inherited sockets. +Keep `io_uring_setup`, `io_uring_enter`, and `io_uring_register` denied: ring +operations can submit sends without passing through these syscall filters. + +The job policy denies every `socket()` family (VSOCK retains its KILL action). +It also denies every `socketpair()`. The packaged Python runtime replaces +asyncio's wakeup socketpair with a nonblocking anonymous pipe and duplex +multiprocessing socketpairs with two anonymous pipes. Native tests cover +thread wakeups, signals, cancellation, subprocesses, duplex spawn/fork IPC, +queues and pools. Named Unix listeners, `multiprocessing.Manager`, forkserver, +and descriptor-sharing APIs remain unsupported. Keep the send and io_uring +denials for inherited descriptors and defense in depth. + +This mitigation is separate from PI-futex filtering. Verify vendor patches for +the deployed guest and node kernels; a policy update does not establish patch +status. Rebuild and replace sandbox runner images to deploy the policy change. + +## Tool-call pipe transport + +Authorized blocking Python invocations receive two anonymous pipe endpoints: +FD 3 writes requests and FD 4 reads responses. No tool-call socket path is +mounted in the jail. The signed, body-bound `tool_call_socket` boolean remains +the wire capability name for compatibility with manifest verification; it now +grants pipes. Ordinary executions, compile steps, and replay/Bash tool calls +receive no endpoints. The Python client makes both ends non-inheritable before +user code runs and refuses tool calls from forked children. + +A Node broker starts per invocation after MicroVM restore and forwards only +POST `/tool-call` to the configured `SANDBOX_FORWARD_TARGET`. It forwards only +the three PTC claim headers; upstream session/token/tool authorization remains +in force. It never follows redirects. Each channel bounds frames and upstream +responses to 1 MiB, active upstream requests to 16, admission to a 64-request +burst refilling at 20 requests/second, partial-frame and response-drain stalls +to 5 seconds, and upstream requests to the job timeout (5–600 seconds). +Malformed framing closes the invocation; disconnect/exit aborts upstream work. +The Python client correlates concurrent responses by frame ID. + +Node/Bun extra stdio descriptors are socketpairs. A small trusted C relay +converts the broker's channel into real anonymous pipes before execing NsJail. +NsJail explicitly passes only FDs 3/4; `spec-guard --tool-call-pipes` checks their +FIFO type and access direction, retains those two ends, and closes unrelated +FDs. Its ordinary mode still closes every descriptor above 2. API/broker/relay +parent-death handling kills the invocation if either supervisor disappears. +Each invocation adds a Node broker and C relay outside the job cgroup; size +runner CPU/memory and PID limits for the configured simultaneous-job count. + +The amd64/arm64 `NsJail IPC Filter` CI jobs compile the policy with pinned Kafel, +exercise socket/send denials plus asyncio/multiprocessing, check guard FD +cleanup, and run the actual broker/relay/NsJail/guard/generated Python client +with concurrent replies. These checks do not boot libkrun or establish deployed +kernel patch status. + +Every complete frame consumes the admission budget before JSON or claim +validation, including frames rejected by the active-request limit. Exhausting +that budget closes the channel and terminates the invocation instead of +producing an unlimited stream of rejection replies. Losing the response pipe's +reader also terminates the invocation, even when no reply is buffered. + +The standalone launcher guest, API runner and combined worker runner all +package Node, the broker bundle and the compiled pipe bridge. Native CI builds +the standalone launcher guest on both architectures and exercises a round trip +using that image's own binaries. + +### Local Mac/Docker canary + +`tests/tool_call_runner.cjs` exercises signed API authorization, the normal +NsJail mount/user/PID/network/IPC/UTS/cgroup namespaces, the generated Python +client, concurrent tool calls, descriptor cleanup, spawn/fork pipe IPC, ordinary +Python jobs and basic Bun JavaScript/TypeScript execution. Spawned workers can +import the preamble but cannot issue tool calls. Python 3.14's default forkserver +requires a named Unix listener and is denied; select an explicit spawn or fork +context. Semaphore-based pools/queues require `/dev/shm`, which the default jail +does not mount. The namespace-disabled native policy tests cover those APIs +when shared memory exists, not their availability in the default jail. + +With runtime packages built by `build-packages.sh` (or a compatible `/pkgs` +tree containing Python 3.14.4 and Bun 1.4.2), run from the repository root: + +```sh +bun service/scripts/dump-pipe-preamble.ts > /tmp/codeapi-pipe-client.py +docker build -f api/Dockerfile --target sandbox-build -t codeapi-pipe-canary . +docker run --rm --init \ + --cap-add SYS_ADMIN --cap-add SYS_CHROOT --cap-add SETUID \ + --cap-add SETGID --cap-add NET_ADMIN \ + --security-opt seccomp=./seccomp/nsjail.json \ + --mount "type=bind,source=$PWD/data/pkgs,target=/pkgs,readonly" \ + --mount "type=bind,source=/tmp/codeapi-pipe-client.py,target=/client.py,readonly" \ + --mount "type=bind,source=$PWD/tests,target=/tests,readonly" \ + --entrypoint /usr/local/bin/node \ + codeapi-pipe-canary /tests/tool_call_runner.cjs /client.py +``` + +The canary disables resource cgroup enforcement as the Mac compose override +does, while retaining the cgroup namespace and all remaining NsJail isolation. +It exercises Docker's Linux VM, not a libkrun guest. It does not establish +production kernel patch status or third-party language-package compatibility. + +### Coordinated rollout + +Deploy the service's new blocking preamble together with rebuilt runner images +containing the broker, relay and updated guard/policy. Rebuild Python packages +with the pipe-runtime patch before enabling socketpair denial. Package init +checks the `.sandbox-pipe-runtime-v1` marker and rebuilds unpatched packages. +Drain or version-route execution while replacing packages; rollback must pair +the policy with its compatible runtime packages. Drain old blocking jobs +and route new blocking requests only to matching images during rollout. Old +socket preambles fail on pipe-only runners; new pipe preambles fail immediately +on old runners. There is no TCP or Unix-socket fallback. Replay mode is unchanged. +Rollback must restore both service and runner versions; retain #309's send and +io_uring denials. See [ADR 0011](adr/0011-tool-call-pipes.md) for the design. + +## Preserve host-side confinement + +- Run sandbox runners on isolated nodes, separate from credential-bearing + control-plane workloads, with least-privilege node IAM. +- Keep hardened startup, signed manifests, egress controls, network policies, + and disabled runner service-account token mounting. +- The launcher refuses to boot if its host-side seccomp filter cannot install. +- Internal authenticated HTTP requests reject redirects. Configure their final + endpoints directly; do not depend on HTTP-to-HTTPS redirects. + +## Dependency checks + +Run `bun audit --cwd api` and `bun audit --cwd service` against the lockfiles. +Distinguish development dependencies from production paths and establish the +advisory's prerequisites before claiming an application exploit. Do not force +incompatible transitive major upgrades solely to obtain a clean audit. + +Remaining service audit findings at this change: + +- `stream-json` (GHSA-528h-pc64-c93x): MinIO 8 uses its JSON-lines parser, + not the affected path filters. Keep MinIO's supported major dependency; + reassess if notifications or path filters become reachable. +- `braces` (GHSA-vfj7-8cjw-p6xm): development-only dependency with no published + fix. Do not feed untrusted patterns to build tooling; track an upstream fix. + +Bun 1.4.2 subprocess spawning with piped stdio also uses socketpairs and is +denied by this policy. Basic JS/TS execution remains supported. Replacing Bun +subprocess IPC requires a separate runtime change; do not re-enable socketpairs +to provide that compatibility. + +For PVC-backed Helm deployments, pin the package-init image to the same immutable +release as the runner. The chart refreshes the mutable `latest` tag with `Always`, +including upgrades that reuse an old `IfNotPresent` value, so a cached old init +image cannot skip the runtime-marker check. Publish the patched init image +before upgrading the runner; refresh does not make an unpublished image current. +An explicit `Never` policy is preserved for locally loaded kind/minikube images; +rebuild and reload the patched package-init image before a local upgrade. diff --git a/docs/adr/001-stateful-code-environments.md b/docs/adr/001-stateful-code-environments.md index cfa44bf4..7992ded8 100644 --- a/docs/adr/001-stateful-code-environments.md +++ b/docs/adr/001-stateful-code-environments.md @@ -52,8 +52,11 @@ worker replacement; the UI and operator documentation must not imply otherwise. - The VM requires no inbound internet listener. - Code API, not the worker, authenticates LibreChat users and normalizes work. - A stolen short-lived credential is insufficient without the worker private - key; a stolen private key is insufficient after credential expiry or - revocation. + key. In the original pairing-only model, a stolen private key is insufficient + after credential expiry or revocation. With optional durable machine + enrollment and signed credential recovery, the private key itself remains + a revocable long-lived credential: access-credential expiry alone does not + protect against theft of that key. Revocation invalidates both. - Pairing codes and credentials are stored by digest where lookup permits. - One configured worker has at most one active fenced assignment. - Sandbox isolation and default-deny egress remain the mandatory default; diff --git a/docs/adr/0010-worktree-metadata-admission.md b/docs/adr/0010-worktree-metadata-admission.md new file mode 100644 index 00000000..4976adc3 --- /dev/null +++ b/docs/adr/0010-worktree-metadata-admission.md @@ -0,0 +1,101 @@ +# Target-Lane Worktree Lifecycle Admission + +Status: proposed. No scheduling or sandbox policy changes are enabled by this PR. + +## Failure + +A shell call to `git worktree add` currently uses the checkout isolation key. +`BridgeWorkspaceSlots.reserve` waits for every child lane; the worker's lane +family guard and native sandbox enforce the same exclusion. Repeated short +admission timeouts discard a waiting checkout request before lanes drain. +The reviewer in chat e2fedf18 (LibreChat PR #16722) could not create its lane. +Durable admission preserves FIFO, but does not remove the checkout-wide barrier. + +## Decision + +Add explicit `create_worktree` and `remove_worktree` workspace operations. +Never infer a weaker scope by parsing an arbitrary shell command. +Each lifecycle request reserves the target lane's ordinary isolation key, +including checkout exclusion and existing quarantine fencing. Sibling lanes +remain independent. Advertise support only when the service and worker agree. +Old clients keep using checkout-wide shell calls. + +Git metadata mutation gets a separate worker-owned mutex, scoped to the common +Git directory and held only while changing a worktree registration. The target +reservation covers the whole operation. It is not the metadata mutex. + +```text +create(name, frozenCommit) + target-lane reservation + validate root identity, target absence, commit and branch policy + metadata mutex → git worktree add --no-checkout --detach → unlock + target lane → git checkout frozenCommit → verify pointers and root identity + publish ready result, then release reservation + +remove(name) + target-lane reservation, require no mutation quarantine + verify pointers and identity, clean tracked/untracked state, no nested mounts + metadata mutex → rename lane to private tombstone + → git worktree remove original-path (now absent) → unlock + delete tombstone outside mutex; keep branch and reachable objects + persist outcome, then release reservation +``` + +A lifecycle journal outside untrusted roots records `reserved`, `registered`, +`ready`, `retiring`, `detached` and `removed`, with assignment owner and captured +identities. Worker recovery reconciles journal entries, never reruns uncertain +shell commands. Creation is not published until checkout completes. Failed +checkout retains the target fence until reconciled. Rename-before-detach must +roll back only while registration still points to the captured original target; +otherwise preserve the tombstone and require reconciliation. Never remove dirty +worktrees, force-remove, prune all registrations, or delete branches as recovery. + +## Boundaries + +- `service/src/bridge/slots.ts`: reserve the named target, not a new global slot + limit. A waiting root still prevents newer sibling admissions. A metadata + operation consumes an existing worker slot; fully occupied workers still wait. +- `packages/code/src/worker.ts`: treat lifecycle operations as mutations and + derive target-lane guards before execution. Keep assignment/identity fencing. +- `packages/code/src/linked-worktrees.ts`: target ownership includes absent + lanes during creation and excludes local retirement of that lane. +- `packages/code/src/linked-worktree-git-guard.ts`: ordinary lanes reject + registration-changing `git worktree` commands. Do not grant write access to + `.git/worktrees` or sibling checkouts to make shell operations work. +- Native sandbox: bulk checkout runs only in the target's admitted directory. + Run no repo hooks or credential helpers in the trusted metadata phase. + +## Alternatives + +1. Longer checkout waits: smallest fix and required for old servers, but still + drains all lanes and serializes bulk checkout/deletion. +2. Allow `git worktree` from arbitrary lane shells: rejected. A shell can remove + a busy sibling or repair foreign metadata; mutexing Git does not confine it. +3. Explicit lifecycle operations: recommended. More protocol/recovery surface, + but target ownership and metadata lifetime are independently testable. + +## Compatibility and rollout + +Deploy the worker journal/guard first, then service capability and matching +client tools. Never send target-scoped lifecycle work to an older worker. +Disable new submissions and drain lifecycle journals before rollback. Root +shell operations and destructive object maintenance remain checkout-exclusive. +Concurrent Git object/ref access still uses Git's own atomic locks; automatic +maintenance stays disabled in lanes. + +A read-only GitHub compare tool supplies merge-base and ancestry for public +repos without any worker call. This helps review orient while admission is busy, +but does not substitute for reading the frozen source or running tests. + +## Blocking verification + +- A long sibling lane remains running while another target is created/removed. +- Same-target/root exclusion and older-root FIFO barriers in Redis and worker. +- Short mutex lifetime verified while bulk checkout/deletion is deliberately gated. +- Cancellation/restart at each journal transition and lost acknowledgements. +- Dirty/untracked, symlink substitution, nested mount, quarantine and foreign-root denial. +- Lane Git guard rejects add/remove/move/repair/prune, including global options. +- Old/new service, worker and client combinations fail closed. + +`tests/worktree-metadata-phases.mjs` probes the local Git mechanism only. It does +not prove service scheduling, worker recovery, or live sandbox containment. diff --git a/docs/adr/0011-tool-call-pipes.md b/docs/adr/0011-tool-call-pipes.md new file mode 100644 index 00000000..c4b53f96 --- /dev/null +++ b/docs/adr/0011-tool-call-pipes.md @@ -0,0 +1,60 @@ +# 0011: Per-invocation anonymous pipes for blocking tool calls + +Status: Accepted + +## Context + +Blocking Python tool calls used an HTTP listener at `/tmp/tcs.sock` mounted into +selected NsJail jobs. CVE-2026-80521 motivated descriptor-passing denials in +#309. Keeping socket creation enabled nevertheless exposes named endpoints +and socket families to untrusted code. The runner is snapshotted before work: +Node and its TLS state must continue to start after MicroVM restore. + +The current stdio spawn APIs create socketpairs, not anonymous pipe endpoints. +The existing guard also intentionally removes unrelated inherited descriptors. +Unmodified Python asyncio and duplex multiprocessing pipes use Unix stream +socketpairs. ADR 0012 replaces those runtime internals with anonymous pipes. + +## Decision + +Use one post-restore Node broker per authorized blocking invocation. Keep the +existing signed/body-bound `tool_call_socket` capability name on the wire, but +grant two pipe endpoints instead of a socket mount. Use length-prefixed JSON +frames with response IDs, a fixed upstream route, whitelisted claim headers, +and explicit size, rate, concurrency and deadline limits. Keep upstream +cryptographic/session/tool authorization authoritative. + +A trusted C relay holds the spawn API's socket on its own side and passes real +anonymous pipe ends at FDs 3/4 into NsJail. The guard preserves them only under +an explicit option after validating FIFO type and write/read access; all +unrelated inherited descriptors are closed. API/broker parent death, channel failure and +job exit terminate the relay and its invocation and abort outstanding upstream +requests. The Python client clears descriptor inheritance and refuses calls +from forked or spawned children. Spawned workers may re-import tool definitions +without receiving or using the pipe capability. stdout/stderr and user stdin remain separate. + +Deny all socket and socketpair creation, retaining sendmsg/sendmmsg/io_uring +denials. The patched Python runtime preserves asyncio and duplex multiprocessing +using explicit spawn/fork contexts. No TCP/socket fallback is provided. + +## Consequences + +New service preambles and runner images require coordinated deployment; +blocking jobs must be drained or version-routed during upgrades and rollbacks. +Bash/replay mode has no pipe grant. Unix listeners/managers, non-stream pairs, +and descriptor-sharing APIs are unsupported. Python 3.14 defaults to forkserver, +which requires a named Unix listener and is denied; use an explicit spawn or +fork context. Semaphore-based queues/pools also require `/dev/shm`, which the +normal jail currently does not mount. Each active invocation adds two +trusted processes outside its job cgroup; runner sizing must account for them. + +A shared post-restore broker could reduce process overhead, but would require +an additional channel admission/multiplexing mechanism. Separate brokers keep +request state, failure and lifetime tied directly to one invocation. Named +FIFOs avoid inherited-FD changes but add filesystem names and open/unlink races; +anonymous pipes require neither path discovery nor shared listener state. + +Native CI verifies both CPU architectures, actual FIFO descriptors, guard +cleanup, socket/send denials, runtime compatibility and concurrent generated +client calls through the compiled policy. Production guest/node patches and +full libkrun rollout remain operator checks. diff --git a/docs/adr/0012-pipe-only-python-runtime.md b/docs/adr/0012-pipe-only-python-runtime.md new file mode 100644 index 00000000..ac1b844b --- /dev/null +++ b/docs/adr/0012-pipe-only-python-runtime.md @@ -0,0 +1,58 @@ +# 0012: Deny socketpairs with pipe-only Python runtime IPC + +Status: Proposed + +## Context + +ADR 0011 removed the named tool-call socket but retained Unix stream socketpairs +for Python runtime compatibility. Removing that exception reduces the kernel +surface available to untrusted code. A seccomp-only change breaks standard +asyncio initialization and duplex multiprocessing connections. + +## Decision + +Deny every socketpair in the job filter. Patch the packaged Linux Python stdlib +at build time, including the package-init and local package-builder paths: + +- Replace asyncio's self-socket with nonblocking CLOEXEC anonymous pipe endpoints. + Preserve thread wakeups, signal wakeup FDs, read callbacks and loop cleanup. +- Replace duplex multiprocessing connections with two anonymous pipes. Preserve + Connection framing, readable fileno/poll, close/EOF, and both descriptor ends + through the existing spawn reduction mechanism. +- Validate each stdlib edit before writing any edits. Unknown source layouts fail + the package build. Mark successful installation so package init cannot reuse an + unpatched package solely because its Python version matches. + +The trusted broker/relay may continue using sockets outside the jail. Untrusted +code receives only anonymous pipe capabilities. No socketpair emulation is +installed globally: direct socket APIs remain denied by seccomp. + +## Consequences + +Rebuild Python packages and deploy them with the new runner policy. Existing +unpatched packages are incompatible with the policy; drain or version-route +jobs during replacement. Snapshot/baked runners need rebuilt runtime contents. +The marker is build bookkeeping, not an authorization boundary. + +Other runtimes or third-party event loops that require socketpairs remain denied; +the patch only covers the standard Python runtime. Named listeners, forkserver, +managers and post-start descriptor sharing remain unsupported. Queues/pools +still require the jail to mount /dev/shm, independent of this change. + +Native regression tests exercise the compiled policy on both CPU architectures, +thread completion, signals, cancellation, async subprocesses, duplex spawn/fork +IPC and EOF, queues/pools, and generated concurrent tool calls. A local signed +runner canary additionally checks packaged Python and ordinary asyncio jobs. + +Bun 1.4.2 subprocess spawning with piped stdio also uses socketpairs and is +denied by this policy. Basic JS/TS execution remains supported. Replacing Bun +subprocess IPC requires a separate runtime change; do not re-enable socketpairs +to provide that compatibility. + +For PVC-backed Helm deployments, pin the package-init image to the same immutable +release as the runner. The chart refreshes the mutable `latest` tag with `Always`, +including upgrades that reuse an old `IfNotPresent` value, so a cached old init +image cannot skip the runtime-marker check. Publish the patched init image +before upgrading the runner; refresh does not make an unpublished image current. +An explicit `Never` policy is preserved for locally loaded kind/minikube images; +rebuild and reload the patched package-init image before a local upgrade. diff --git a/docs/adr/0013-supervisor-anonymous-pipes.md b/docs/adr/0013-supervisor-anonymous-pipes.md new file mode 100644 index 00000000..0a905455 --- /dev/null +++ b/docs/adr/0013-supervisor-anonymous-pipes.md @@ -0,0 +1,65 @@ +# 0013: Remove Unix sockets from runner supervisors + +Status: Proposed + +## Context + +ADRs 0011 and 0012 deny sockets in untrusted jobs. The trusted API, broker, +relay, and NsJail monitor still process job-controlled data. Node/Bun's piped +subprocess stdio and NsJail's startup handshake create Unix socketpairs outside +the job filter. Supervisor hardening must replace those transports before it +can deny the same kernel entry points. + +## Decision + +Install an inherited seccomp filter before starting the API. Deny AF_UNIX +socket creation, every socketpair, sendmsg/sendmmsg (including inherited +sockets), and all io_uring entry points. Validate the native audit architecture +and reject x32 syscall encodings. TCP remains available for API requests, +hosted previews, and the broker's fixed upstream. This is a targeted deny filter, +not a complete supervisor syscall allowlist. Startup fails if installation fails. Force sequential resolver queries with +`RES_OPTIONS=single-request` (preserving existing options), so glibc DNS uses +sendto instead of its blocked sendmmsg batching. + +A small N-API addon creates CLOEXEC anonymous pipes. Linux subprocess launches +pass integer pipe ends to Node/Bun rather than requesting runtime-created pipes. +The adapter preserves output draining, backpressure, child close ordering, and +cleanup on exit, failed spawn, and cancellation. macOS development retains native +spawn; Linux container tests on the Mac verify the real filter and pipe adapter. + +The tool-call broker uses separate request/response pipes to its relay. The relay +retains bounded buffers, parent-death signals, and immediate termination when a +response reader disappears, even without a pending response. Buffered sources +are omitted from poll until drained so a closed writer cannot cause a CPU spin. +Untrusted jobs now +receive anonymous pipes for stdin/stdout/stderr as well as tool-call FDs 3/4. + +Patch the pinned NsJail build to replace its supervisor/child handshake with +separate readiness and error pipes. Reject optional seccomp notification and +user-mode networking modes that require descriptor passing. Build without libnl: +network-isolated jobs do not need interface movement or traffic-rule programming, +and this avoids a gratuitous netlink enumeration during namespace setup. +Unsupported interface/traffic-rule requests fail closed. + +Hosted-app egress uses the packaged iptables-legacy and ip6tables-legacy binaries, +whose transport does not require sendmsg. Both chains must install before any +app is admitted. IPv4/IPv6 legacy netfilter support is consequently required on +the runner kernel; missing support rejects hosted-app admission. + +## Consequences and validation + +All three runner image paths package the matching addon, policy binary, broker, +relay, and patched NsJail. Deploy rebuilt runner images and regenerate snapshots; +these internal processes and binaries are an atomic image upgrade. The external +service protocol and previously patched Python runtime are unchanged. + +The filter also applies to hosted apps and their descendants. Runtimes and +third-party libraries that require Unix sockets/socketpairs remain unsupported. +The trusted API retains its existing privileges and other syscalls; reducing +those privileges requires another containment boundary. Kernel patching remains +necessary. + +CI runs native filter, anonymous stdio, cleanup/backpressure, packaged broker, +and hosted-app egress/preview tests on amd64 and arm64. A local signed runner +canary additionally exercises normal namespaces with Python 3.14.4 asyncio, +spawn/fork multiprocessing, concurrent tool calls, and Bun 1.4.2 JS/TS execution. diff --git a/docs/byom-worker-admission.md b/docs/byom-worker-admission.md index 2c2b354b..0df8f17c 100644 --- a/docs/byom-worker-admission.md +++ b/docs/byom-worker-admission.md @@ -5,14 +5,22 @@ The limit is 32 admitted requests per worker, including the active request. When the limit is reached, the workspace endpoint returns HTTP 429 with `WORKER_QUEUE_FULL`. A different worker has an independent admission queue. -The workspace HTTP endpoint allows at most 30 seconds for admission. After -admission and worker validation, a separate execution deadline starts. Commands -receive their requested timeout (30 seconds by default, up to five minutes), -capped by the operator's `JOB_TIMEOUT`, plus five seconds to settle the result. -Read/search/list operations receive up to 30 seconds, also capped by `JOB_TIMEOUT`. -Disconnecting or cancelling removes the waiting -request without cancelling the active assignment. Expired entries are pruned; -Redis key expiry also bounds state left by a crashed API process. +The Code API workspace HTTP endpoint allows 30 seconds for admission when the +request has no `X-LibreChat-Workspace-Queue-Wait-Ms` header. A caller can +advertise a positive integer allowance in milliseconds through that header, +up to five minutes and any configured server queue ceiling. An invalid or +out-of-range value is rejected before dispatch. This queue budget is +independent of `JOB_TIMEOUT`; a shorter client or proxy deadline still ends +the wait. After admission +and worker validation, a separate execution deadline starts. Commands receive +their requested timeout (30 seconds by default, up to five minutes), capped by +the operator's `JOB_TIMEOUT`, plus five seconds to settle the result. Other +operations receive up to 30 seconds, also capped by `JOB_TIMEOUT`. +Disconnecting or cancelling removes a waiting request without cancelling the +active assignment. Expired entries are pruned; Redis key expiry also bounds +state left by a crashed API process. Reservations derive their TTL at acquisition +from the remaining absolute deadline or a fresh execution budget; enqueued +assignment records use the final execution deadline, not the elapsed queue budget. After admission, the API revalidates the worker incarnation, identity, tenant binding and workspace operation. A waiting request cannot migrate to a replacement @@ -26,13 +34,29 @@ Existing workers still execute one assignment at a time. Parallel execution acro workspaces requires separate lease claims and isolated native sandbox contexts; this admission change does not advertise that capability. -LibreChat must allow queue time plus execution/settlement time and five seconds -for HTTP delivery: 65 seconds for reads, 70 seconds for default commands, and -340 seconds for five-minute commands. Either side can be upgraded first. Older -clients still cancel at their earlier deadline; newer clients preserve errors from -older servers without retrying mutations. Both updates are needed for the full -waiting budget. Any reverse proxy request timeout must accommodate these totals. -The worker package does not need an update for the deadline change. +Clients and reverse proxies must allow the admitted queue budget plus +execution/settlement time and five seconds for HTTP delivery. With the default +five-minute `JOB_TIMEOUT` and no queue header, that is at least 65 seconds for +non-command tools, 70 seconds for default commands, and 340 seconds for +five-minute commands. At the maximum advertised five-minute queue allowance, +those totals become 335, 340, and 610 seconds respectively. With a smaller +`JOB_TIMEOUT`, use the advertised allowance (or 30 seconds without a header), +bounded by the server queue ceiling, plus `min(JOB_TIMEOUT, 30s)` for other +operations or `min(JOB_TIMEOUT, requested command timeout) + 5s` for commands, +plus five seconds for delivery. The caller should advertise only the queue time +left after reserving execution, settlement, and delivery under its own HTTP +deadline; Code API does not receive that absolute deadline. -Focused regression coverage lives in `service/src/bridge/admission.test.ts` and -`service/src/bridge/worker-admission.test.ts`. +LibreChat's `maxQueueWaitMs` is a retry horizon after a typed capacity +rejection, **not** a per-attempt HTTP timeout. Without its opt-in +`maxRequestTimeoutMs`, LibreChat keeps a 30-second admission allowance per +attempt. Enabling a longer client budget requires LibreChat's header support on +every API replica and a timed canary through each intermediary; changing Code +API alone does not guarantee the full wait. An earlier client, tool, or proxy +timeout disconnects the request; if work was already admitted, a mutation may +have run and must not be blindly retried. Existing workers do not need an update. + +Focused regression coverage lives in `service/src/bridge/admission.test.ts`, +`service/src/bridge/worker-admission.test.ts`, +`service/src/bridge/concurrent-store.test.ts`, and +`service/src/workspace-tools/router.test.ts`. diff --git a/docs/lambda-microvm/README.md b/docs/lambda-microvm/README.md index 80e5e468..b8fbbcc5 100644 --- a/docs/lambda-microvm/README.md +++ b/docs/lambda-microvm/README.md @@ -511,7 +511,7 @@ appear in `api/src/config.ts`. | `EGRESS_GATEWAY_FILE_SERVER_URL` / `_TOOL_CALL_SERVER_URL` | Gateway only | Private upstream origins. Do not expose them to the runner. | | `FILE_SERVER_URL` | Worker only | Direct source used to fetch authorized by-reference inputs before pushing them into the runner cache. Do not bake it into the runner. | | `CODEAPI_EGRESS_LEDGER_REQUIRED=true` / `REDIS_*` | Gateway | Makes grant replay/revocation state fail closed in Redis. | -| `SANDBOX_ALLOWED_LOCAL_NETWORK_PORT` / `SANDBOX_FORWARD_TARGET` | Runner image | Configures the narrow tool-call socket proxy; it does not start or expose the socket by itself. The target host/port must match `EGRESS_GATEWAY_URL`. | +| `SANDBOX_ALLOWED_LOCAL_NETWORK_PORT` / `SANDBOX_FORWARD_TARGET` | Runner image | Configures the per-invocation tool-call pipe broker; Node starts only on an authorized post-restore execution. The target host/port must match `EGRESS_GATEWAY_URL`. | | `CODEAPI_EXECUTION_MANIFEST_PRIVATE_KEY` | Worker only | Signs the manifest after the gateway returns the scoped grant. | | `SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY` / `SANDBOX_REQUIRE_EGRESS_MANIFEST=true` | Runner image | Verifies and requires the signed per-execution manifest. | diff --git a/docs/openshell/README.md b/docs/openshell/README.md new file mode 100644 index 00000000..880225b0 --- /dev/null +++ b/docs/openshell/README.md @@ -0,0 +1,189 @@ +# OpenShell backend evaluation + +Status: Evaluation. No OpenShell backend is registered or enabled. + +Goal: add an opt-in, operator-hosted OpenShell backend behind Code API's existing +execution contract. Keep LibreChat clients, the default HTTP/libkrun backend, +Lambda MicroVMs, and outbound BYOM workers unchanged. + +## Baseline and upstream pin + +- Code API baseline: `836d001319015072f9493df5d7710fc371403962`. +- OpenShell evaluation release: `v0.1.2`, commit + `6648bd0c290efbc41ba131ee9831ee45cd431f94`. +- Pin the CLI, gateway, supervisor, and compute driver to that release. Record + image digests, effective policy, runtime/kernel, and resource limits with each + live result. The probe checks the CLI version, not the gateway's version. +- Revalidate this track against each proposed upgrade. Do not use rolling `dev` + tags for acceptance results. + +Upstream references at the evaluation commit: +[license](https://github.com/NVIDIA/OpenShell/blob/6648bd0c290efbc41ba131ee9831ee45cd431f94/LICENSE), +[third-party notices](https://github.com/NVIDIA/OpenShell/blob/6648bd0c290efbc41ba131ee9831ee45cd431f94/THIRD-PARTY-NOTICES), +[SDK](https://github.com/NVIDIA/OpenShell/blob/6648bd0c290efbc41ba131ee9831ee45cd431f94/sdk/typescript/README.md), +[policy schema](https://github.com/NVIDIA/OpenShell/blob/6648bd0c290efbc41ba131ee9831ee45cd431f94/docs/how-it-works/policies/schema.mdx). + +## Integration boundary + +Current flow: + +```text +LibreChat -> Code API auth/authorization -> Redis job + -> worker -> signed execute request -> SandboxBackend.execute + -> runner -> Code API egress gateway -> artifacts/tools + -> worker result finalization -> client +``` + +Proposed flow changes only the sandbox backend: + +```text +worker -> OpenShell adapter -> private OpenShell gateway + -> isolated workload + compatible Code API runner + -> unchanged signed execute contract and Code API egress gateway +``` + +The backend seam is +[`SandboxBackend`](../../service/src/sandbox-backend/types.ts). Its existing +consumers already own deadlines, queue/backend fencing, artifact restoration, +and result mapping. OpenShell's lifecycle API is not a substitute for the +runner's `/api/v2/execute` protocol. + +### Options + +| Approach | Trade-off | Decision | +| -------------------------------------------- | ------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------- | +| Existing backends only | No new control plane; no OpenShell policy/lifecycle integration | Keep as defaults and rollback targets | +| Adapter retaining the signed runner contract | Reuses Code API authorization, artifacts, tools, and finalization; runner compatibility is unproven | Evaluate first | +| Execute code directly with OpenShell exec | Avoids nested NsJail; requires rebuilding file/tool delivery, manifest validation, and result semantics | Defer unless the runner approach cannot retain its guarantees | + +Do not put the LibreChat application, the BYOM identity process, or Code API +control-plane credentials inside an agent workload. The gateway remains private; +Code API authenticates users and authorizes tenant/session access. + +## Invariants and owners + +| Invariant | Owner | +| ------------------------------------------------------------------------------ | ------------------------------------------------------------ | +| Tenant/user/session authorization and quotas | Code API | +| Signed body is immutable; `inputDelivery` storage IDs never reach the workload | Worker and adapter | +| Only scoped egress grants authorize artifact and tool access | Code API egress gateway and runner | +| Queue wait, provisioning, and execution share the original job deadline | Worker and adapter | +| Abort cannot settle as success; remote execution must terminate | Adapter and compute runtime | +| Retried jobs cannot create duplicate active execution or repeat mutations | Adapter, queue fencing, and runtime registry | +| Stateful workspace reuse waits for successful `sessionResultFinalizer` | Worker and adapter | +| Filesystem, process identity, network, and resource restrictions fail closed | OpenShell policy and selected runtime, plus runner hardening | +| Cleanup is confirmed, not merely requested; orphans remain observable/reapable | Adapter and lifecycle reconciler | + +OpenShell Docker/Podman policy rejects root workloads. The current runner +entrypoint performs mounts, ownership changes, cgroup setup, and an NsJail smoke +test. Compatibility is a blocker to resolve, not permission to run privileged, +disable manifests, relax Landlock, or bypass hardened startup. OpenShell policy +alone does not establish a separate-kernel tenant boundary. + +## Work sequence + +### 1. Lifecycle spike (this slice) + +- [x] Pin an upstream release and identify the existing backend seam. +- [x] Add an explicit, dependency-free CLI probe with strict filesystem policy, + no network allowances, no attached providers, resource limits, and cleanup. +- [x] Test validation, cancellation, partial failure, redaction, and nonterminal + cleanup without a live gateway. +- [ ] Run the probe against an isolated operator-owned gateway. Record results. + +This probe runs only a fixed shell/file round trip. It does **not** exercise Code +API, verify effective policy enforcement, or prove tenant isolation. A gateway +administrator's global policy can override the supplied sandbox policy. + +### 2. Hardened runner compatibility + +- [ ] Identify the supported compute/runtime boundary and prove startup without + weakening the existing security invariants. +- [ ] Forward the private runner listener through authenticated OpenShell + transport. Never expose its execute endpoint publicly. +- [ ] Execute a real worker-built signed request unchanged. Test Python/Bash, + input/output artifacts, tool-call replay, and error/result parity. +- [ ] Reject tampered/expired manifests, unauthorized storage handles, access to + supervisor/control-plane credentials, forbidden destinations, metadata + endpoints, and cross-tenant workspace reuse. +- [ ] Verify host/kernel filesystem protection and resource enforcement with + negative tests, not only effective-policy inspection. + +If nesting cannot work securely, stop and design the direct-exec replacement's +full authorization and delivery contract before implementation. + +### 3. Opt-in adapter + +- [ ] Use the same-release SDK behind lazy backend loading. Verify package + availability and locking; this release's TypeScript SDK uses GitHub Packages. + The CLI is a spike dependency, not a per-job production transport. +- [ ] Add an explicit backend config gate, isolated queues, and producer/consumer + backend checks. Old consumers must never pick up OpenShell jobs, including + legacy jobs without a backend marker. +- [ ] Start stateless. Reject affinity/strict mode until tenant-bound session + locks, fencing, finalization, and persistence are implemented and tested. +- [ ] Cover enqueue-anchored deadlines, cancellation while provisioning/running, + retry idempotency, gateway failure, ambiguous create responses, accepted/pending + deletion, worker crashes, orphan reconciliation, and shutdown. +- [ ] Keep auth tokens and provider credentials out of logs, argv, and workload + environment. Do not silently fall back to a different backend. + +### 4. Acceptance and rollout + +- [ ] Measure concurrency, queue/provision/execute/cleanup latency, resources, + artifact throughput, and failure rates against the existing backend using the + same workloads. Record measurements; define targets before promotion. +- [ ] Exercise gateway restart, worker restart, disconnects, upgrades, cleanup + backlogs, and quota pressure. Test database/runtime topology before claiming HA. +- [ ] Pin deployment artifacts, collect required notices/SBOMs, and document + private gateway authentication, backups, upgrades, and orphan operations. +- [ ] Canary a separate endpoint and worker pool only after compatibility and + security gates pass. Roll back routing, then drain the new queues. No automatic + file/workspace migration or production-default change is part of this track. + +## Run the lifecycle probe + +Requirements: Node 22.21+, an OpenShell `0.1.2` CLI, a registered named evaluation +gateway using the matching release, and a digest-pinned shell/coreutils image +pullable by that gateway. Install and configure these separately on dedicated +infrastructure. This repository does not install or start a gateway. + +```bash +node --test scripts/openshell/probe.test.mjs + +node scripts/openshell/probe.mjs \ + --gateway codeapi-evaluation \ + --workspace default \ + --image 'REGISTRY/IMAGE@sha256:REPLACE_WITH_64_HEX_DIGEST' +``` + +The live command creates and deletes one uniquely named `codeapi-probe-*` +sandbox. It uses the bundled policy, one CPU, 256 MiB, manual approvals, no +credential auto-discovery, and a fixed ten-second exec timeout. Each CLI phase +has a 120-second timeout (`--timeout-ms`, maximum 300000); this is not the +production job deadline. SIGINT/SIGTERM abort work but allow an independent +bounded cleanup attempt. SIGKILL or host/process failure cannot run cleanup. + +Only the pinned CLI's terminal `Deleted` or `already deleted` acknowledgement +confirms cleanup. Accepted/pending, unfamiliar, failed, and timed-out deletion +results fail the probe. CLI stdout/stderr are not emitted, except sanitized +phase/version/timing results. The live probe's overall success requires both +the file round trip and confirmed cleanup. + +On failure, use the emitted sandbox name and the **same gateway/workspace** to +inspect and reconcile it. A timed-out create can finish after cleanup; even an +`already deleted` acknowledgement does not resolve that race. Review the +gateway's resources after any ambiguous create or process failure. Do not retry +blindly or interpret `cleanup=confirmed` on a failed run as an orphan-free +fleet. Automated reconciliation is an adapter acceptance gate. + +## License implications + +OpenShell is Apache-2.0 at the pinned commit. That permits commercial hosted use +and modification without a network-use source-disclosure obligation. It does +not grant NVIDIA trademark rights or a support/warranty commitment. + +For distributed images or on-prem packages, include the license, preserve +applicable attribution/notices, mark modified upstream files, and audit bundled +runtime/SDK/image/model dependencies separately. This slice distributes no +OpenShell binaries or SDK and introduces no paid NVIDIA service dependency. diff --git a/docs/remote-bridge/README.md b/docs/remote-bridge/README.md index efe84b84..dc7cb98b 100644 --- a/docs/remote-bridge/README.md +++ b/docs/remote-bridge/README.md @@ -31,6 +31,42 @@ CODEAPI_BRIDGE_TOKEN= CODEAPI_BRIDGE_AUTH_MODE=paired ``` +To opt in to durable machine authorization on every Code API replica, set a +single stable public **Code API** origin (not the LibreChat URL): + +```dotenv +CODEAPI_BRIDGE_RECOVERY_SERVER_ID=https://code.example.com +# 0 (default): enrolled machine keys remain authorized until revoked. +# CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS=0 +# CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS=60 +# CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE=12 +# CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE=30 +# CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE=240 +``` + +Omitting the server ID retains the existing pairing and refresh behavior and +hides the recovery routes. Deploy the compatible Code API version to **all** +replicas before setting this value and enrolling workers again. Older Code API +replicas can still pair or refresh a worker but do not write durable enrollment; +they must not serve device login or recovery requests. Only a pairing redeemed +after this option is enabled has a recoverable key. Updating Code API alone +does not make old workers reconnect automatically: the CLI must also implement +this recovery protocol in the later worker release. + +Store Redis state durably across restarts. The primary `docker-compose.yaml` +now uses Redis AOF and a named `/data` volume; preserve that volume when +recreating the stack. If upgrading a running stack with an in-memory Redis, +migrate its state before recreating the container: mounting an empty volume +does **not** preserve active assignments, fences, or earlier revocations. Other +deployments must provide equivalent durable Redis (for example, a managed +persistent Redis service and backups). Revocation and +machine enrollment share that state across replicas; do not configure eviction +of authorization keys. If enrollment state is missing, credentials minted under +that enrollment fail closed, and the worker must be explicitly enrolled again. +Restoring a backup from *before* a revocation can revive trust; reconcile +revocations after recovery from backup. Use a distinct server ID for each Code +API deployment and keep it stable when the endpoint changes behind a proxy. + Use `strict` instead of `affinity` if every request must include a runtime session hint. In hardened mode, startup requires the bridge token to be at least 32 bytes. `PTC_MODE=blocking` is rejected; replay mode is required because a @@ -129,7 +165,10 @@ backslashes, symlink escapes, unexpected fields, and host roots are rejected. Workspace mutation remains disabled unless the operator starts the worker with `--allow-workspace-writes` (or `LIBRECHAT_CODE_ALLOW_WORKSPACE_WRITES=true`). That adds bounded `write_file` -and exact-match `edit_file` operations. Writes are limited to 1 MiB of UTF-8 +and `edit_file` operations. `edit_file` matches exactly by default; the +negotiated `tolerant_match` and `replace_all` edit features add +whitespace-tolerant matching and multi-location replacement (see +`packages/code/README.md`). Writes are limited to 1 MiB of UTF-8 text, require an existing in-workspace parent directory, reject symlinks, and commit atomically. The worker capability is an enforcement boundary; LibreChat should still route every mutation through its configurable tool-approval hooks. @@ -226,7 +265,40 @@ execution. atomically on their first redemption attempt. - Worker credentials expire after fifteen minutes and are bound to an Ed25519 public key. Exact-request signatures include the HTTP method, path, body - digest, timestamp, nonce, and credential. + digest, timestamp, nonce, and credential. With recovery enabled, redeeming a + pairing also persists a separate machine authorization and its public key in + Redis without a TTL by default; an operator can instead set a bounded + enrollment lifetime. +- `POST /v1/bridge/workers/:workerId/credentials/challenge` does not require + an administrator token or an existing access credential, but **does** require + the enrolled key. Its JSON body contains `protocolVersion: 1`, + `operation: "credential.challenge"`, the configured `serverId`, the matching + `workerId`, a fresh UTC ISO `timestamp`, a random 32-byte base64url `nonce`, + and `signature` computed with `signBridgeRecoveryStart(privateKey, fields)` + from `@librechat/code/identity`. Code API verifies the signed fields and + consumes the nonce once before charging the machine's shared challenge + budget; a fabricated request cannot exhaust another worker's budget. +- The response is a short-lived, single-use challenge with the server ID, + worker ID, enrollment generation, operation and expiry. Sign those fields + with `signBridgeRecovery(privateKey, challenge)` and send the fields plus + `signature` to `POST .../credentials/recover` to obtain a new short-lived + credential. Invalid proofs are limited per high-entropy challenge; only + successfully verified, unused proofs consume the machine's shared recovery + budget. Separately, both recovery endpoints limit all incoming requests per + connection peer *before* key verification, including well-formed JSON with + malformed or forged proofs; forged headers and worker IDs cannot bypass + that limit or consume the signed machine budget. All limits live in shared + Redis; HTTP 429 means back off. When a reverse proxy connects to Code API, + its clients share that peer's limit. Restrict direct backend access and apply + client-IP and global + abuse limits at the trusted ingress to keep one proxy peer from becoming a + shared bottleneck; do not trust an arbitrary `X-Forwarded-For` on Code API. +- Recovery and revocation are atomic Redis transitions across API replicas. + A missing, revoked, expired or superseded enrollment never creates new + credentials. Recovery only restores transport authentication. It does not + clear assignment fences, worker or workspace quarantine, or uncertain + execution state. The worker private key is a durable, revocable credential; + expiry of an access credential alone does **not** protect against key theft. - Accepted proof nonces cannot be replayed, credentials rotate before expiry, and an administrator can revoke the active worker identity immediately. - Assignment leases bind to a stable paired identity rather than an individual @@ -238,7 +310,18 @@ execution. - Code API negotiates a bounded number of active workspace assignments per worker. The lower API or worker slot ceiling wins, and assignments sharing the same workspace isolation key remain serialized while independent - conversation worktrees may run concurrently. + conversation worktrees may run concurrently. A linked-worktree lane + (`worktree: `) nests beneath its checkout's key: sibling lanes run + concurrently, while a lane and its checkout exclude each other, and a lane + cannot start while its checkout is quarantined. +- Workspace tool admission waits for capacity up to 30 seconds without a + `X-LibreChat-Workspace-Queue-Wait-Ms` header. A caller can advertise a longer + per-request allowance, bounded by five minutes and any server queue ceiling. + Disconnects cancel waiting, and admitted work receives a separate execution + budget capped by `JOB_TIMEOUT`. A shorter client or proxy timeout can end the + wait sooner; Code API does not receive an absolute caller deadline. See + [BYOM worker admission](../byom-worker-admission.md) for the total-request + and proxy timeout requirements. - Dynamic workers are fenced to their server-issued tenant before assignment. - Each assignment has an absolute deadline, generation, and random lease token. - Settlements with the wrong worker, generation, token, or expired deadline are @@ -301,3 +384,58 @@ capabilities. LibreChat's owner-scoped environment registry can issue these principal-bound pairings without changing the worker execution protocol or moving code tools into the Agents SDK. + +### Durable workspace requests + +New clients can probe authenticated `GET /v1/workspace-tools/capabilities` for +`durableWorkspaceRequests: 1`. Older servers return 404 or advertise 0. Keep the +legacy synchronous `/workspace-tools/execute` path for those servers. Do not +fall back to synchronous execution after submitting durable work. + +Submit the same workspace-tool body to `POST /v1/workspace-tools/requests`, with +`X-LibreChat-Workspace-Request-Id` set to a unique 16–128 character identifier +(letters, digits, `_`, `-`). The queue-wait header and command timeout retain their +existing limits. HTTP 202 acknowledges durable acceptance, not execution. +Disconnecting does not cancel accepted work. + +- Repeat an identical submission with the same ID to recover a lost response. + An ID reused for different work returns 409 `REQUEST_CONFLICT`. +- Poll `GET /v1/workspace-tools/requests/:requestId`. States are `queued`, + `admitted`, `completed`, `failed`, and `cancelled`. Lookup is tenant/user scoped; + unknown or another principal's IDs return 404. Results are non-consuming reads. +- Cancel with `DELETE /v1/workspace-tools/requests/:requestId`. Cancellation is a + request, not proof of termination. All operations notify the worker. A committed + result can win the race. Without a confirmed settlement, read-only calls retain + `cancelled`; mutations retain an unknown-outcome failure and their safety fence. + `ASSIGNMENT_EXPIRED` means execution may have occurred. Never replay it. +- Status includes worker/workspace/lane metadata, queue position when available, + measured queue wait, and the execution deadline after admission. Queue position + is advisory: negotiated independent roots may run concurrently. + +Requests and results are retained for 24 hours from acceptance. Idempotency is +bounded by that retention window. A client must not resubmit an expired ID after +404; its outcome is unknown. Automatic result claims and wake-ups remain the +client's responsibility. + +Bridge-enabled API replicas reconcile accepted work from Redis within a shared +bridge-policy scope. Disabled bridges and different scheduling policies never +claim that work. Keep backend/profile, bridge auth mode, worker selection, slot +ceiling and command-timeout policy identical on replicas behind one endpoint. +Drain durable requests before changing those settings. FIFO admission position +survives API restarts and temporary worker registration/readiness loss until the +queue deadline. Capacity acquisition and assignment enqueue atomically check the coordinator +claim, queued state, cancellation and admission deadline. Delayed callbacks +cannot reserve capacity after their claim ends. Assignment enqueue also checks +capacity ownership and atomically commits the durable `admitted` transition. +Recovery resumes only queued work and observes admitted assignments without +creating another command. Worker identity/incarnation changes fail pending work +rather than transferring it to another machine. Execution gets its full budget +after admission. Existing lease fencing, cancellation and workspace quarantine +continue to apply. + +This capability covers workspace tools, including `execute_command`, not +programmatic `/execute` jobs. It requires Redis state to survive the API restart; +Redis data loss is not recoverable. All API replicas behind an endpoint must +support the capability before a client enables it. Roll back clients to the legacy +path only for new work; drain durable requests before rolling back servers. +Worker protocol, worker concurrency and deployed rate limits are unchanged. diff --git a/docs/remote-bridge/projects.md b/docs/remote-bridge/projects.md index 987ce112..26887ee9 100644 --- a/docs/remote-bridge/projects.md +++ b/docs/remote-bridge/projects.md @@ -41,7 +41,11 @@ workspace registration remains available for independent project directories. filesystem boundary and coordination for the common Git directory. - A worktree beneath its parent checkout overlaps that checkout. Either use disjoint execution roots under a discovery grant or explicitly exclude and - coordinate descendant worktrees before relaxing root exclusion. + coordinate descendant worktrees before relaxing root exclusion. Linked + worktree lanes (`--linked-worktree-lanes`) take the second approach for + `.worktrees/`: hierarchical admission keeps a lane and its checkout + exclusive, and each lane's sandbox can write only its worktree, shared + object and ref storage, and its own worktree metadata. - Setup and dependency links must remain within the execution policy. Sharing writable dependency directories between supposedly isolated worktrees reintroduces overlap and requires an explicit operator decision. diff --git a/docs/remote-bridge/worker-runbook.md b/docs/remote-bridge/worker-runbook.md index 3043c541..99f19992 100644 --- a/docs/remote-bridge/worker-runbook.md +++ b/docs/remote-bridge/worker-runbook.md @@ -119,7 +119,7 @@ a separate machine. Install: -- Node.js 20.11 or newer (Node.js 24 is supported); +- Node.js 22.21 or newer (Node.js 24 is supported); - Git; - `bubblewrap`, `socat`, and `ripgrep` on Linux; - Bash 5.2 or newer and `jq` when Bash Programmatic Tool Calling is enabled; @@ -255,7 +255,10 @@ Important semantics: existing ID such as `primary` to preserve agent/conversation bindings. - `repo` and `ref` are labels. They do not clone, fetch, or check out anything. - `root` must already exist. Relative roots resolve from the definition file. -- Setup runs before registration on every worker start. It must be idempotent. +- Setup runs before registration on every worker start by default. It must be + idempotent. Optional `setup.reuse` declares fingerprint inputs and a sandboxed + readiness check to avoid reinstalling an unchanged, still-ready checkout. See + [preparation reuse](../../packages/code/README.md#reusing-a-prepared-checkout). - A setup failure or timeout prevents registration and leaves a durable quarantine marker for operator inspection. - Actions are fixed operator commands. The model selects only the action name @@ -338,6 +341,15 @@ filesystem root, but anyone able to alter a checkout's remote can select any repository where the App is installed; keep the App's installation scope narrow. Pass the checkout as the command working directory; changing directories only inside the shell cannot change the token chosen before command launch. +For a trusted VM that needs to switch among repositories in the same installed +account or organization inside one command, set +`LIBRECHAT_CODE_GITHUB_TOKEN_SCOPE=installation`. The resolved installation +token covers only repositories and permissions GitHub granted to that App +installation. It refreshes after two minutes so newly approved permissions +become available without a worker restart. The default is `repository`. +Commands spanning different accounts or organizations must start in a checkout +from the target account or organization; a shell `cd` cannot switch the +installation chosen at command launch. Set `LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` only as a legacy fixed-installation fallback; it cannot be combined with checkout routing. @@ -479,11 +491,16 @@ it still advertises named environments. ### Expired bridge credential -A running worker refreshes its short-lived credential automatically. If a -machine is offline long enough that refresh can no longer authenticate, issue -a fresh one-time pairing for the same worker ID and redeem it with a newly -generated keypair. Reusing the worker ID preserves the LibreChat environment -record and its agent assignments; creating a new ID creates a new environment. +A running worker refreshes its short-lived credential automatically. With +Code API durable enrollment enabled, a worker that still has its enrolled +private key can request a short-lived challenge and recover a new access +credential without manual re-pairing. The current CLI does **not** yet invoke +that endpoint automatically; update it when worker reconnect support ships. +Until then, or if enrollment is missing or revoked, use the one-time operator +pairing fallback. A new pairing replaces the Code API worker identity and may +require LibreChat environment reauthorization; reusing a worker ID alone does +not guarantee preservation of its LibreChat environment or agent assignments. +Never clear quarantine or workspace fences as part of credential recovery. ### Failed environment setup or uncertain mutation @@ -504,6 +521,35 @@ replace the local clear command. Never clear quarantine merely to make the worker start. It represents a setup, command, cancellation, or settlement whose effects may be incomplete. +### Disk filled by task worktrees + +Agents that create one linked worktree per task under `/.worktrees/` +leave each one behind with its own dependencies and build output. A worker with +`--linked-worktree-lanes` retires stale ones automatically: two minutes after +startup, every six hours, and before managed setup would be deferred for low +space. Only verified linked worktrees in writable roots that are idle for seven +days, clean, unlocked, outside any merge or rebase, unquarantined, and whose +`HEAD` is on a remote-tracking ref or already in the default branch by content +(a squash- or rebase-merged branch whose remote branch was deleted) are +removed, with `git worktree remove` without `--force`. Merged content is judged +against the checkout's last fetch of the default branch; the worker never +fetches. Branches are kept; restore one with +`git worktree add .worktrees/ `. The +[worker package guide](../../packages/code/README.md#retiring-stale-linked-worktrees) +lists every condition. + +Each pass logs `worktree retirement: retired N, kept M (reason counts)`. To see +why a particular worktree is kept, restart once with +`LIBRECHAT_CODE_LOG_LEVEL=debug`. Worktrees kept as `dirty` or `unpushed` hold +work nobody has published; push or discard it from the checkout before removing +them by hand, never with `rm -rf`. If the disk is already full and setup keeps +failing, free space by hand first: retirement runs only after the worker starts, +or during setup that has a `storage` floor configured. + +To keep every worktree, add `--no-worktree-retirement` (or +`LIBRECHAT_CODE_WORKTREE_RETIREMENT=false`). To keep them longer, set +`--worktree-idle-days ` (or `LIBRECHAT_CODE_WORKTREE_IDLE_DAYS`). + ## 15. Common failures - **`--environment cannot be combined...`:** remove old workspace flags and @@ -516,7 +562,26 @@ command, cancellation, or settlement whose effects may be incomplete. - **Worker online but not ready:** check native sandbox preparation, definition validation, setup, quarantine, and readiness logs. - **Setup repeats on restart:** setup is intentionally per-start; make it - idempotent or remove it. + idempotent, remove it, or opt in to the bounded `setup.reuse` readiness + contract in the [worker package guide](../../packages/code/README.md#reusing-a-prepared-checkout). +- **Dependency copies fill the disk:** declare shared npm/uv/browser resources + instead of per-worktree downloads. On a verified clone-capable filesystem, + configure private copy-on-write snapshots and their lifecycle budget. Do not + symlink another branch's mutable `node_modules` or hardlink writable installs. +- **Managed preparation deferred for low space:** `storage.minFreeBytes` plus + `setupReserveBytes` is a soft pre-setup floor, not a hard quota. A worker + with linked worktree lanes first retires stale task worktrees and measures + again. If space is still short, expand the volume or clean reproducible + artifacts; do not clear quarantine as a disk fix. +- **Many `.worktrees/*` directories remain:** see + [Disk filled by task worktrees](#disk-filled-by-task-worktrees). Count the + kept reasons in the retirement summary before removing anything by hand. +- **Snapshot maintenance:** preview with `prune-environment-storage + --environment ` and use `--apply` only after reviewing its JSON. Active, + unknown and unmarked data stays intact. Include all environment definitions + for root-isolation checks. This does not archive source worktrees or prune + mutable tool caches. Keep control state on separate storage when hard + protection from arbitrary build writes is required. - **Git works on the host but not in tools:** verify the App installation, permissions, private-key mode/owner, and allowed GitHub domains. - **Repository label is present but files are absent:** `repo`/`ref` are diff --git a/helm/codeapi/Chart.yaml b/helm/codeapi/Chart.yaml index 4e122bbf..dfcdd45b 100644 --- a/helm/codeapi/Chart.yaml +++ b/helm/codeapi/Chart.yaml @@ -3,7 +3,7 @@ apiVersion: v2 name: codeapi description: A Helm chart for Code Interpreter API - scalable code execution service type: application -version: 0.3.1 # Chart version (bump this when you change the chart) +version: 0.3.2 # Chart version (bump this when you change the chart) appVersion: "2.0.0" # App version (bump this when you change the app) # Keywords for searching diff --git a/helm/codeapi/README.md b/helm/codeapi/README.md index a063e597..f63950ae 100644 --- a/helm/codeapi/README.md +++ b/helm/codeapi/README.md @@ -55,6 +55,43 @@ platform rather than templated here: external ingress/service mesh, KEDA-style queue-depth autoscaling, and cloud-IAM secret delivery (the env hooks below cover all of them). +**Prometheus scraping.** With `metrics.enabled=true` and +`workerSandbox.enabled=true`, separate PodMonitors scrape the service-worker's `health` port and the sandbox-runner's +`sandbox` port at `/metrics`. Only the latter exports +`codeapi_sandbox_post_cleanup_*` metrics. Both inherit `metrics.interval` and +`metrics.scrapeTimeout`. Install the Prometheus Operator PodMonitor CRD and +configure Prometheus to discover this release's PodMonitors and namespace. + +The default NetworkPolicy allows runner ingress only from service-worker pods. +To permit scraping with `networkPolicy.enabled=true`, explicitly match your +trusted Prometheus namespace **and** pod labels in one peer, for example: + +```yaml +metrics: + enabled: true + sandboxRunner: + ingressFrom: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: observability + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus +``` + +Replace the example labels with those of your scraper. The default empty list +adds no access, even when metrics are enabled. This rule is rendered only while +metrics and the worker/sandbox tier are enabled. It uses the configured runner +port and does not change sandbox egress. Any policies on the Prometheus side +must also allow the scrape connection. When chart network policies are disabled, +allow scraping through your platform's policies instead. + +NetworkPolicy cannot restrict access to an HTTP path. These peers can reach the +runner's execution endpoint as well, so do not use unrestricted peers such as +`{}` or disable execution-manifest verification to enable scraping. No extra +Service or externally exposed port is created. Chart upgrades do not modify +runner images or require changes to stored session state for this scrape wiring. + **Pairing-fence rollbacks.** Do not use a direct `helm rollback` from a chart revision containing the bridge pairing fence to an older revision. Helm runs rollback hooks from the target revision, so a pre-fence target cannot stop its diff --git a/helm/codeapi/templates/network-policy.yaml b/helm/codeapi/templates/network-policy.yaml index 2184e9f9..fe24123f 100644 --- a/helm/codeapi/templates/network-policy.yaml +++ b/helm/codeapi/templates/network-policy.yaml @@ -3,6 +3,7 @@ Network Policies for CodeAPI. The important boundary is sandbox-runner: - service-worker may call sandbox-runner. + - explicitly configured metrics peers may reach the same runner port. - sandbox-runner may call only egress-gateway (plus DNS if required). - egress-gateway delegates only to file-server and tool-call-server. */}} @@ -70,6 +71,15 @@ spec: ports: - protocol: TCP port: {{ .Values.workerSandbox.sandbox.port }} + {{- if and .Values.metrics.enabled .Values.workerSandbox.enabled }} + {{- with (.Values.metrics.sandboxRunner | default dict).ingressFrom }} + - from: + {{- toYaml . | nindent 6 }} + ports: + - protocol: TCP + port: {{ $.Values.workerSandbox.sandbox.port }} + {{- end }} + {{- end }} egress: - to: - namespaceSelector: {} diff --git a/helm/codeapi/templates/package-init-job.yaml b/helm/codeapi/templates/package-init-job.yaml index 8d927503..baf81b17 100644 --- a/helm/codeapi/templates/package-init-job.yaml +++ b/helm/codeapi/templates/package-init-job.yaml @@ -40,7 +40,8 @@ spec: containers: - name: package-init image: "{{ .Values.workerSandbox.packages.initJob.image.repository }}:{{ .Values.workerSandbox.packages.initJob.image.tag }}" - imagePullPolicy: {{ .Values.workerSandbox.packages.initJob.image.pullPolicy }} + {{/* Refresh cached latest images; preserve explicit offline Never. */}} + imagePullPolicy: {{ if and (eq (.Values.workerSandbox.packages.initJob.image.tag | toString) "latest") (ne .Values.workerSandbox.packages.initJob.image.pullPolicy "Never") }}Always{{ else }}{{ .Values.workerSandbox.packages.initJob.image.pullPolicy }}{{ end }} env: - name: PYTHON_VERSION value: {{ .Values.workerSandbox.packages.initJob.pythonVersion | quote }} diff --git a/helm/codeapi/templates/podmonitor.yaml b/helm/codeapi/templates/podmonitor.yaml index 405e3bc3..aee33f44 100644 --- a/helm/codeapi/templates/podmonitor.yaml +++ b/helm/codeapi/templates/podmonitor.yaml @@ -43,6 +43,28 @@ spec: matchLabels: {{- include "codeapi.serviceWorker.selectorLabels" . | nindent 6 }} {{- end }} +{{- if .Values.workerSandbox.enabled }} +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: {{ include "codeapi.fullname" . }}-sandbox-runner + labels: + {{- include "codeapi.sandboxRunner.labels" . | nindent 4 }} +spec: + podMetricsEndpoints: + - path: /metrics + port: sandbox + {{- with .Values.metrics.interval }} + interval: {{ . }} + {{- end }} + {{- with .Values.metrics.scrapeTimeout }} + scrapeTimeout: {{ . }} + {{- end }} + selector: + matchLabels: + {{- include "codeapi.sandboxRunner.selectorLabels" . | nindent 6 }} +{{- end }} {{- if .Values.egressGateway.enabled }} --- apiVersion: monitoring.coreos.com/v1 diff --git a/helm/codeapi/values.yaml b/helm/codeapi/values.yaml index bafed305..4858fd71 100644 --- a/helm/codeapi/values.yaml +++ b/helm/codeapi/values.yaml @@ -306,11 +306,12 @@ workerSandbox: forceRebuild: false image: repository: codeapi-package-init + # Pin this tag to the same immutable release as the runner in production. tag: latest - pullPolicy: IfNotPresent + pullPolicy: Always pythonVersion: "3.14.4" nodeVersion: "24.15.0" - bunVersion: "1.3.14" + bunVersion: "1.4.2" bashPackageVersion: "5.2.0" backoffLimit: 2 ttlSecondsAfterFinished: 3600 @@ -512,6 +513,11 @@ metrics: enabled: false # Set to true to create PodMonitors for Prometheus scraping interval: "30s" scrapeTimeout: "10s" + sandboxRunner: + # Explicit NetworkPolicy peers allowed to scrape the runner when metrics and + # networkPolicy are enabled. Empty preserves worker-only ingress. This port + # also serves /execute, so select only trusted Prometheus pods (see README). + ingressFrom: [] # ============================================================================= # NETWORK POLICIES diff --git a/launcher/Dockerfile b/launcher/Dockerfile index b2b02386..bad5cd2f 100644 --- a/launcher/Dockerfile +++ b/launcher/Dockerfile @@ -28,17 +28,28 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ && rm -rf /var/lib/apt/lists/* RUN git clone -b master --single-branch https://github.com/google/nsjail.git . \ - && git checkout dccf911fd2659e7b08ce9507c25b2b38ec2c5800 + && git checkout 66ad78dc361721512fbf9470488e584364c75489 \ + && git submodule update --init --recursive -RUN make -j$(nproc) +COPY docker/nsjail-anonymous-control.patch /tmp/nsjail-anonymous-control.patch +# Network-isolated jobs need no libnl transport; removing it avoids sendmsg +# in the supervisor and fails closed for interface/traffic-rule configuration. +RUN git apply /tmp/nsjail-anonymous-control.patch && make -j$(nproc) NL3_EXISTS=no COPY api/src/spec-guard.c /tmp/spec-guard.c +COPY api/src/tool-call-pipe-bridge.c /tmp/tool-call-pipe-bridge.c +COPY api/src/anonymous-pipes.c /tmp/anonymous-pipes.c +COPY api/src/supervisor-policy.c /tmp/supervisor-policy.c +COPY --from=node:22-slim /usr/local/include/node /usr/local/include/node COPY docker/rootfs-setup.c /tmp/rootfs-setup.c -RUN gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ +RUN gcc -shared -fPIC -O2 -Wall -Wextra -Werror -I/usr/local/include/node -o /usr/local/lib/anonymous-pipes.node /tmp/anonymous-pipes.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/sandbox-supervisor-policy /tmp/supervisor-policy.c \ + && gcc -O2 -static -Wall -Wextra -Werror -o /usr/local/bin/tool-call-pipe-bridge /tmp/tool-call-pipe-bridge.c \ + && gcc -O2 -static -o /usr/local/bin/spec-guard /tmp/spec-guard.c \ && gcc -O2 -static -o /usr/local/bin/sandbox-rootfs-setup /tmp/rootfs-setup.c \ && chmod 0111 /usr/local/bin/spec-guard /usr/local/bin/sandbox-rootfs-setup -FROM oven/bun:1.3.14-debian AS sandbox-build +FROM oven/bun:1.4.2-debian AS sandbox-build ENV DEBIAN_FRONTEND=noninteractive @@ -64,12 +75,17 @@ RUN sed -i '/en_US.UTF-8/s/^# //g' /etc/locale.gen && locale-gen COPY --from=nsjail-builder /nsjail/nsjail /usr/sbin/nsjail COPY --from=nsjail-builder /usr/local/bin/spec-guard /usr/local/bin/spec-guard +COPY --from=nsjail-builder /usr/local/bin/tool-call-pipe-bridge /usr/local/bin/tool-call-pipe-bridge +COPY --from=nsjail-builder /usr/local/bin/sandbox-supervisor-policy /usr/local/bin/sandbox-supervisor-policy +COPY --from=nsjail-builder /usr/local/lib/anonymous-pipes.node /usr/local/lib/anonymous-pipes.node +ENV SANDBOX_PIPE_ADDON=/usr/local/lib/anonymous-pipes.node +COPY --from=node:22-slim /usr/local/bin/node /usr/local/bin/node RUN chmod +x /usr/sbin/nsjail WORKDIR /sandbox_api -COPY api/package.json ./ -RUN bun install +COPY api/package.json api/bun.lock ./ +RUN bun ci COPY api/tsconfig.json ./ COPY api/src ./src diff --git a/launcher/entrypoint.sh b/launcher/entrypoint.sh index 304858e4..47fba9e6 100644 --- a/launcher/entrypoint.sh +++ b/launcher/entrypoint.sh @@ -12,15 +12,129 @@ set -e # joined by a separator that api/src/guest-dns.sh expands back into lines. RESOLV_FIELD_SEPARATOR='|' +# The guest kernel keeps loopback traffic on its own loopback device instead of +# proxying it through TSI, so a loopback resolver such as Docker's embedded +# 127.0.0.11 never answers inside the guest. Relay it from this container's own +# address, which the guest reaches through TSI, and hand the guest that address. +# The relay forks per datagram and per connection, so bound how long an idle +# child may live, and how many TCP children may run at once. socat 1.8 fails +# every UDP-RECVFROM child given max-children, so the UDP relay relies on its +# short idle timeout. Only the runner network reaches the relay: NsJail jobs in +# the guest run in their own network namespace. +RESOLVER_RELAY_READY_ATTEMPTS=50 +RESOLVER_RELAY_MAX_CHILDREN=64 +RESOLVER_RELAY_UDP_IDLE_SECONDS=5 +RESOLVER_RELAY_TCP_IDLE_SECONDS=30 + +# One inet_aton(3) part: hex with 0x, octal with a leading 0, else decimal. +inet_aton_part() { + case "$1" in + 0x*) [[ "$1" =~ ^0x[0-9a-f]+$ ]] && echo $((16#${1#0x})) ;; + 0*) [[ "$1" =~ ^0[0-7]*$ ]] && echo $((8#$1)) ;; + *) [[ "$1" =~ ^[0-9]+$ ]] && echo $((10#$1)) ;; + esac +} + +# Numeric value of an IPv4 address in the inet_aton(3) forms glibc accepts in +# resolv.conf: one to four parts, the last filling the remaining bytes. +ipv4_value() { + local parts part value=0 index last + IFS=. read -ra parts <<< "$1" + [[ "$1" != *. ]] && (( ${#parts[@]} >= 1 && ${#parts[@]} <= 4 )) || return 1 + last=$(( ${#parts[@]} - 1 )) + for ((index = 0; index < last; index++)); do + part="$(inet_aton_part "${parts[index]}")" || return 1 + (( part <= 255 )) || return 1 + value=$(( value | part << (24 - 8 * index) )) + done + part="$(inet_aton_part "${parts[last]}")" || return 1 + (( part < 1 << (32 - 8 * last) )) || return 1 + echo $(( value | part )) +} + +# The eight 16-bit groups of an IPv6 address in decimal, accepting compressed +# zeros and a dotted IPv4 tail. +ipv6_groups() { + local address="$1" head tail octets octet group groups=() tail_groups=() fill + if [[ "$address" =~ ^(.*:)([0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)$ ]]; then + address="${BASH_REMATCH[1]}" + IFS=. read -ra octets <<< "${BASH_REMATCH[2]}" + for octet in "${octets[@]}"; do + [[ "$octet" =~ ^(0|[1-9][0-9]{0,2})$ ]] && (( octet <= 255 )) || return 1 + done + address+="$(printf '%x:%x' $(( octets[0] << 8 | octets[1] )) $(( octets[2] << 8 | octets[3] )))" + fi + [[ "$address" =~ ^[0-9a-f:]+$ && "$address" != :[!:]* && "$address" != *[!:]: && "$address" != *:::* ]] || return 1 + if [[ "$address" == *::* ]]; then + head="${address%%::*}" + tail="${address#*::}" + [[ "$tail" != *::* ]] || return 1 + [ -z "$head" ] || IFS=: read -ra groups <<< "$head" + [ -z "$tail" ] || IFS=: read -ra tail_groups <<< "$tail" + fill=$(( 8 - ${#groups[@]} - ${#tail_groups[@]} )) + (( fill >= 1 )) || return 1 + for ((; fill > 0; fill--)); do groups+=(0); done + groups+=("${tail_groups[@]}") + else + IFS=: read -ra groups <<< "$address" + fi + (( ${#groups[@]} == 8 )) || return 1 + for group in "${groups[@]}"; do + [[ "$group" =~ ^[0-9a-f]{1,4}$ ]] || return 1 + printf '%d ' $((16#$group)) + done +} + +# Canonical form of a loopback nameserver in any spelling glibc accepts: IPv4 +# 127.0.0.0/8, IPv6 ::1, or IPv4-mapped 127.0.0.0/8 (relayed as IPv4). Fails for +# every other address. +canonical_loopback() { + local address="${1,,}" text groups value + address="${address%%\%*}" + if [[ "$address" == *:* ]]; then + text="$(ipv6_groups "$address")" || return 1 + read -ra groups <<< "$text" + if [ "${groups[*]}" = '0 0 0 0 0 0 0 1' ]; then + echo ::1 + return 0 + fi + [ "${groups[*]:0:6}" = '0 0 0 0 0 65535' ] || return 1 + value=$(( groups[6] << 16 | groups[7] )) + else + value="$(ipv4_value "$address")" || return 1 + fi + (( value >> 24 == 127 )) || return 1 + printf '%d.%d.%d.%d' $(( value >> 24 )) $(( value >> 16 & 255 )) $(( value >> 8 & 255 )) $(( value & 255 )) +} + +loopback_nameserver() { + local line words + while IFS= read -r line || [ -n "$line" ]; do + read -ra words <<< "${line%$'\r'}" + if [ "${words[0]:-}" = nameserver ] && canonical_loopback "${words[1]:-}"; then + return 0 + fi + done < "$1" +} + +runner_address() { + getent ahostsv4 "$(cat /proc/sys/kernel/hostname)" | awk '$1 !~ /^127\./ { print $1; exit }' +} + encode_resolv_conf() { + local loopback="${1:-}" relay="${2:-}" local LC_ALL=C - local line words encoded='' + local line words canonical encoded='' while IFS= read -r line || [ -n "$line" ]; do line="${line%$'\r'}" if [[ ! "$line" =~ ^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]] ]]; then continue fi read -ra words <<< "$line" + if [ "${words[0]}" = nameserver ] && canonical="$(canonical_loopback "${words[1]}")"; then + [ "$canonical" = "$loopback" ] || continue + words[1]="$relay" + fi line="${words[*]}" if [[ "$line" == *[!' '-'~']* || "$line" == *[\"$RESOLV_FIELD_SEPARATOR]* ]]; then echo "ERROR: runner /etc/resolv.conf line cannot cross the kernel command line: $line" >&2 @@ -31,13 +145,62 @@ encode_resolv_conf() { printf '%s' "$encoded" } -SANDBOX_RESOLV_CONF="$(encode_resolv_conf < /etc/resolv.conf)" +proc_net_address() { + local a b c d + IFS=. read -r a b c d <<< "$1" + printf '%02X%02X%02X%02X:0035' "$d" "$c" "$b" "$a" +} + +start_resolver_relay() { + local nameserver="$1" address="$2" local_address udp_pid tcp_pid attempt + local udp_target="UDP4-SENDTO:$nameserver:53" tcp_target="TCP4:$nameserver:53" + if [ "$nameserver" = ::1 ]; then + udp_target='UDP6-SENDTO:[::1]:53' + tcp_target='TCP6:[::1]:53' + fi + socat -T "$RESOLVER_RELAY_UDP_IDLE_SECONDS" \ + "UDP4-RECVFROM:53,bind=$address,fork" "$udp_target" & + udp_pid=$! + socat -T "$RESOLVER_RELAY_TCP_IDLE_SECONDS" \ + "TCP4-LISTEN:53,bind=$address,reuseaddr,fork,max-children=$RESOLVER_RELAY_MAX_CHILDREN" "$tcp_target" & + tcp_pid=$! + local_address="$(proc_net_address "$address")" + for ((attempt = 0; attempt < RESOLVER_RELAY_READY_ATTEMPTS; attempt++)); do + if grep -q " $local_address " /proc/net/udp && grep -q " $local_address 00000000:0000 0A " /proc/net/tcp; then + echo "Relaying guest DNS from $address:53 to loopback resolver $nameserver" + return 0 + fi + if ! kill -0 "$udp_pid" "$tcp_pid" 2>/dev/null; then + break + fi + sleep 0.1 + done + kill "$udp_pid" "$tcp_pid" 2>/dev/null || true + echo "ERROR: guest DNS relay could not listen on $address:53 for loopback resolver $nameserver" >&2 + return 1 +} + +LOOPBACK_NAMESERVER="$(loopback_nameserver /etc/resolv.conf)" +RESOLVER_RELAY_ADDRESS='' +if [ -n "$LOOPBACK_NAMESERVER" ]; then + RESOLVER_RELAY_ADDRESS="$(runner_address)" + if [ -z "$RESOLVER_RELAY_ADDRESS" ]; then + echo "ERROR: runner has no non-loopback IPv4 address to relay loopback resolver $LOOPBACK_NAMESERVER" >&2 + exit 1 + fi +fi + +SANDBOX_RESOLV_CONF="$(encode_resolv_conf "$LOOPBACK_NAMESERVER" "$RESOLVER_RELAY_ADDRESS" < /etc/resolv.conf)" export SANDBOX_RESOLV_CONF if [[ "$RESOLV_FIELD_SEPARATOR$SANDBOX_RESOLV_CONF" != *"${RESOLV_FIELD_SEPARATOR}nameserver "[!\#]* ]]; then echo 'ERROR: runner /etc/resolv.conf has no nameserver' >&2 exit 1 fi +if [ -n "$LOOPBACK_NAMESERVER" ]; then + start_resolver_relay "$LOOPBACK_NAMESERVER" "$RESOLVER_RELAY_ADDRESS" +fi + if [ "${LAUNCHER_FILTER_VSOCK_ENOTCONN:-true}" = "true" ]; then # libkrun can emit this benign TSI/vsock teardown line after the guest has # already closed its side of the socket. It contains the word "error", so diff --git a/launcher/src/main.rs b/launcher/src/main.rs index 27f08f92..84e48347 100644 --- a/launcher/src/main.rs +++ b/launcher/src/main.rs @@ -570,6 +570,16 @@ fn guest_cmdline_problem(env: &[(String, String)], args: &[String]) -> Option, + enter: impl FnOnce() -> i32, +) -> Result { + let allowed = filter_result?; + eprintln!("[launcher] VMM seccomp filter applied ({allowed} syscalls allowed, ioctl restricted to KVM+terminal)"); + eprintln!("[launcher] Starting microVM..."); + Ok(enter()) +} + fn main() { let vcpus: u8 = env::var("LAUNCHER_VCPUS") .ok() @@ -701,16 +711,13 @@ fn main() { ffi::check("set_rlimits", ffi::krun_set_rlimits(ctx, rlimit_ptrs.as_ptr())); eprintln!("[launcher] Guest RLIMIT_NOFILE target: {nofile_target}"); - match seccomp::apply_vmm_filter() { - Ok(n) => eprintln!("[launcher] VMM seccomp filter applied ({n} syscalls allowed, ioctl restricted to KVM+terminal)"), - Err(e) => { - eprintln!("[launcher] WARNING: Failed to apply VMM seccomp: {e}"); + let ret = match enter_microvm(seccomp::apply_vmm_filter(), || ffi::krun_start_enter(ctx)) { + Ok(ret) => ret, + Err(error) => { + eprintln!("[launcher] ERROR: refusing to start microVM without VMM seccomp: {error}"); + process::exit(1); } - } - - eprintln!("[launcher] Starting microVM..."); - - let ret = ffi::krun_start_enter(ctx); + }; eprintln!("[launcher] krun_start_enter returned {ret} (should not return on success)"); process::exit(1); } @@ -719,7 +726,7 @@ fn main() { #[cfg(test)] mod tests { use super::{ - desired_nofile_soft_limit, guest_cmdline_problem, guest_nofile_rlimit, + desired_nofile_soft_limit, enter_microvm, guest_cmdline_problem, guest_nofile_rlimit, is_allowed_guest_env_key, GUEST_CMDLINE_LIMIT, LIBKRUN_CMDLINE_RESERVE, }; @@ -738,6 +745,25 @@ mod tests { .collect() } + #[test] + fn microvm_does_not_start_when_seccomp_installation_fails() { + let result = enter_microvm(Err("filter installation denied".into()), || { + panic!("microVM entry must not run without seccomp"); + }); + assert_eq!(result, Err("filter installation denied".into())); + } + + #[test] + fn microvm_starts_after_seccomp_installation_succeeds() { + let mut entered = false; + let result = enter_microvm(Ok(80), || { + entered = true; + -1 + }); + assert!(entered); + assert_eq!(result, Ok(-1)); + } + #[test] fn guest_cmdline_accepts_single_line_printable_env() { let env = env(&[ diff --git a/packages/code/README.md b/packages/code/README.md index 5a06b92a..0437ecfb 100644 --- a/packages/code/README.md +++ b/packages/code/README.md @@ -3,6 +3,9 @@ Provider-neutral protocol and worker CLI for attaching a stateful, sandboxed code environment to LibreChat Code API. +Requires Node.js 22.21 or newer. CI tests Node.js 22 and 24; Node.js 20 is +unsupported. + For a complete machine setup and operations guide, see the [self-hosted worker runbook](../../docs/remote-bridge/worker-runbook.md). @@ -213,12 +216,55 @@ SRT with: worker's private scratch directory; - read access denied to the worker's home directory except for that workspace; - paired identity and mutation-quarantine files explicitly denied; +- the registered workspace's own Git metadata denied writes — `.git/hooks` + and `.git/config` always, plus `.git/config.worktree`, `.git/commondir`, + and the equivalent files under `.git/modules/*` and `.git/worktrees/*` + where they already exist — so a sandboxed command cannot plant a hook or a + `filter`/`fsmonitor`/`diff` config entry that would run unsandboxed the + next time Git runs in the checkout; - `LIBRECHAT_CODE_*` and nonessential inherited environment variables removed; - network egress denied by default, local binding denied, and Unix sockets denied; and - bounded time and aggregate output, with best-effort process-group termination on cancellation, timeout, and completion. +Native command output keeps a prefix and rolling suffix for each stream, so late +summaries and errors survive truncation. Each stream stores at most +`maxOutputBytes` of copied raw bytes while the command runs, independent of output +volume or chunk count. When both streams are noisy they split the existing combined +response budget equally, with the odd byte reserved for stderr; a quiet stream gives +its unused allowance to the other based on rendered UTF-8 bytes, including replacement +characters for malformed input. Sandbox violation annotations enter the same +stderr window before rendering. UTF-8 boundaries and inline +`[... N bytes omitted ...]` markers count toward the combined byte limit. The count +reports omitted raw bytes for that stream, including annotation bytes. If a stream's +allowance cannot fit its marker, only the retained text and the existing `truncated` +flag are returned. Truncation does not stop execution. Exit codes, timeout/signal +fields, and cancellation errors keep their existing semantics, and no new request, +result, or capability keys are introduced. + +The Git-metadata denies are applied to the registered root directly rather +than relying on SRT's Linux mandatory denies, which are derived from the +worker process's own current directory — the worker home, not the workspace — +and so never covered the registered root; macOS already enforced the +equivalent through global Seatbelt patterns, and this keeps the guarantee +identical on both platforms regardless of the worker's cwd. The set is +recomputed before every command, so a repository, submodule, or linked +worktree created on the host after the worker starts is covered from the next +command, and metadata the worker cannot inspect fails the command closed. +`.git/commondir` and `.git/config.worktree` are denied only where they already +exist, because Git reads them strictly and SRT would otherwise mask an absent +one with a stub Git cannot open, failing every Git command. Because the whole +workspace stays writable, a sandboxed command can still stage Git +configuration Git will honor later by other means — for example replacing +the entire `.git` directory, writing a new `commondir` that redirects the +common directory, creating a missing `config.worktree` in a repository that +already enables the `worktreeConfig` extension, or initializing a fresh +nested repository. Denying those safely would require +making the workspace's Git storage structurally read-only, which the +personal-machine SRT trust model does not; use the Docker/NsJail backend or a +dedicated VM boundary when a workspace command must be treated as adversarial. + SRT restrictions remain inherited by descendants. Windows additionally uses a kill-on-close Job Object. Native macOS does not provide an equivalent hard process-lifetime boundary: a deliberately daemonized descendant can outlive @@ -308,6 +354,17 @@ installed**. Use this mode only where the machine operator trusts the VM and the App's installation scope; the default `admitted` mode keeps the startup binding. Checkout routing requires an App without a fixed installation ID. +On a trusted VM, `--github-token-scope installation` (or +`LIBRECHAT_CODE_GITHUB_TOKEN_SCOPE=installation`) mints one token for all +repositories GitHub grants to the resolved App installation. This lets a +command started in one checkout push to another repository in the same account +or organization, including through `cd` or `git -C`, and use organization +Projects. GitHub still enforces the installation's selected repositories and +permissions. Tokens are shared by installation, refreshed after two minutes, +and kept out of the sandbox's readable environment. The default remains +`repository`. A command crossing to another account or organization still +needs to start in a checkout belonging to that account or organization. + For compatibility with deployments that intentionally bind a worker to one installation, set the optional legacy `LIBRECHAT_CODE_GITHUB_INSTALLATION_ID` fallback. @@ -579,6 +636,59 @@ installed as one atomic mutation. Code API dispatches the batch form only after the worker and server negotiate `batch` in `editFileModes`. Revision-fenced edits likewise require the negotiated `expected_base_sha256` entry in `editFileFeatures`. + +Edits apply in order, each to the text the earlier ones produced. When any edit +fails, the worker still checks the rest and rejects the whole batch with one +`EDIT_CONFLICT` whose message lists every failing edit by position. Large +batch messages shorten reasons and source excerpts to stay within the bound, +but never omit failing edit positions. A missing edit names the nearest +candidate line and flags elided (`...`) or line-numbered `oldText`, a +whitespace-only difference, or CRLF line endings. When the file has a region +that `oldText` most likely meant (the alignment where the most of its +distinctive lines fall into place, or its unique first line or closest line), +the reason ends with `the current text at lines A-B (~ whitespace differs, ! +text differs) is ""`: a JSON string of at most 8 rows of the form +`|`, each line shortened to 160 characters, where the mark is +a space for a line identical to `oldText` at that position, `~` when only its +whitespace differs and `!` when its text differs. Excerpts are the first detail +dropped to fit the message bound, and never shorten another edit's reason. Both +the excerpt (1,600 bytes) and the message that carries it (3,800 bytes) are +measured as UTF-8 once JSON-encoded, so the Code API's error body stays within +the 4,096 bytes hosts such as LibreChat read. +An ambiguous edit gives its match count and line numbers. Overlapping +occurrences count as separate locations. Detailed source-line excerpts require +`read_file` or `preview_edit` on the same workspace; edit-only workers return a +generic failure and its error code without revealing file contents. A preview +itself exposes the resulting file text, so it is read-capable. + +Two optional features change matching, each negotiated in `editFileFeatures` +before Code API dispatches it: + +- `tolerant_match`: a request-level `matching: 'tolerant'` falls back from an + exact match to, in order, `line-trimmed` (ignores trailing whitespace and + CRLF), `indentation-flexible` (a uniformly shifted block, with `newText` + moved to the file's indentation) and `whitespace-normalized` (any whitespace + run between complete whitespace-delimited tokens, never a prefix or suffix + of another token). Without `replaceAll`, a match must still be unique; + replacements use the matched line's ending even in mixed-ending files. The + whitespace-normalized tier peels the complete shared newline-and-indentation + wrapper from `newText` even when CRLF/LF or nearby spaces differ, without + removing intentional extra line breaks or duplicating the source line ending. + Boundary whitespace claimed by `oldText`, including the number of line breaks, + must exist beside the matched tokens in the source; an attempt to remove it + with a token-only fallback fails rather than silently preserving it. Exact and + line-window matches can still replace terminators. + Excessively repetitive indentation candidates fail closed with a request + for more context rather than scanning every long window. Dense files reuse a + compact newline index for matching and diagnostics; overlapping exact matches + are counted without restarting a scan at each offset. +- `replace_all`: a batch edit's `replaceAll: true` replaces every + non-overlapping match instead of requiring exactly one, and still fails when + nothing matches. + +A request that sets `matching` or any `replaceAll` receives `matches`, one +`{ strategy, occurrences }` entry per edit. Requests that set neither receive +exactly the legacy result. Only IDs, names, protocol version, supported operations, and negotiated write modes appear in worker capabilities; absolute host paths remain local to the worker process. @@ -622,9 +732,10 @@ native SRT command backend. This operator switch controls availability; LibreChat tool approval hooks remain the user-facing allow/deny boundary for each invocation. -Reads reject absolute paths, traversal, escaping symlinks, non-regular files, -and files larger than 1 MiB. The opened file is checked against its canonical -in-workspace inode before it is read. Text search uses `rg` only to enumerate a +Reads reject absolute paths, traversal, escaping symlinks, and non-regular files. +The opened file is checked against its canonical in-workspace inode before it is +read. Ordinary reads stream bounded line windows even from files larger than 1 MiB; +see the [read contract](WORKSPACE-READS.md). Text search uses `rg` only to enumerate a bounded set of ignored-aware candidates with configuration and symlink following disabled. It then opens and verifies each candidate through the same confined 1 MiB read boundary before matching locally. File listing invokes `rg` without @@ -644,8 +755,10 @@ non-regular files, and commit through an owner-only temporary file followed by an atomic rename. The worker syncs the containing directory and verifies that the installed inode still contains the requested bytes before reporting success. Edits replace text only when the requested old text occurs exactly -once and reject if the file changes before commit. These operations do not -create directories or execute commands. +once (unless negotiated `replaceAll` selects every non-overlapping match) and +reject if the file changes before commit. Intermediate replacements are bounded +before construction, including `replaceAll`; these operations do not create +directories or execute commands. Register one directory already present on the worker machine with the worker-directory option: @@ -780,11 +893,149 @@ Legacy requests without a conversation identity continue to use the selected source root. Older Code API deployments do not negotiate the capability, so the worker omits it until every request path understands the isolation boundary. -Admission waits at most 30 seconds. A `WORKSPACE_QUEUE_TIMEOUT` response (HTTP -503, `Retry-After: 1`) means the operation was not assigned or started; wait for +#### Linked worktree lanes + +Agents that keep one checkout and give each task its own linked worktree +(`git worktree add .worktrees/`) can run those tasks concurrently: + +```sh +librechat-code run \ + --worker-dir /projects/LibreChat \ + --workspace-lease-slots 4 \ + --linked-worktree-lanes \ + --allow-workspace-writes \ + --allow-workspace-commands +``` + +`LIBRECHAT_CODE_LINKED_WORKTREE_LANES=true` is the environment equivalent. The +worker then advertises `workspaceScopes: ['git_linked_worktree']` for each +registered root, and a request that names `worktree: ` runs in its own +lane at `/.worktrees/`, with `cwd` and file paths relative to that +worktree. Sibling lanes run concurrently up to the negotiated slot count. A +lane and its checkout never run at the same time: requests without a +`worktree`, including `git worktree add` or `remove` run at the root, wait for +every lane beneath the checkout, and a waiting root request holds back newer +lanes so it cannot be starved. + +Before admission the worker verifies, without running Git, that the directory +is a real linked worktree of that checkout: no symlinks on the path, a `.git` +file pointing at `/.git/worktrees/`, and metadata whose `commondir` +and `gitdir` point back. Shared Git storage paths granted for writes must be +real directories, not symlinks into sibling metadata; optional log and LFS +paths may be absent. A lane's sandbox can write only its worktree, the +shared object and ref storage (`.git/objects`, `.git/refs`, `.git/logs/refs`, +`.git/lfs`) and its own `.git/worktrees/` metadata. Everything else in +`.git` stays read-only: configuration, hooks and `info`, the checkout's own +`HEAD`, index and merge or rebase state, and sibling metadata. Automatic `gc` +and maintenance are disabled, and `git gc` itself cannot run in a lane (it +needs to write `.git/gc.pid` and `packed-refs`). A lane's `PATH` starts with +a read-only Git wrapper that refuses `prune`, `gc`, `repack`, `prune-packed`, +`maintenance`, `multi-pack-index`, `for-each-repo` and `git lfs prune`, +including after Git options such as `-C` or `-c`. `for-each-repo` is refused +entirely because Git dispatches its child commands without re-entering the +wrapper. It rejects `git lfs fetch` or `pull` with +`--prune`/`-p`, and `git fetch` or `pull` with `--auto-maintenance`/`--auto-gc`. +It enforces `maintenance.auto=false`, `gc.auto=0` and `help.autocorrect=0` +after caller-supplied Git options, preventing those overrides from restoring +automatic maintenance or correcting a misspelled command to `prune`. +It also refuses **all configured Git aliases**, +including harmless ones: aliases can hide destructive maintenance through +local config, `-c`, `--config-env`, includes or shell commands. Call the +underlying Git command instead, or run the alias from the checkout. Git's +safe pathspec options (`--literal-pathspecs`, `--glob-pathspecs`, +`--noglob-pathspecs`, `--icase-pathspecs`) remain usable in lanes. Run storage +maintenance from the checkout. + +This is a guardrail against accidental commands, **not** a filesystem +boundary: invoking Git by an absolute path, resetting `PATH`, or using a +script that does either bypasses it. Such commands can still delete a sibling +lane's unpublished objects. Do not enable concurrent lanes for agents or +scripts that deliberately bypass the wrapper. The guard requires `/bin/bash` +and a trusted system Git executable; without either, lane commands are not +admitted. Deleting a branch or tag also needs the checkout, because Git locks +`packed-refs` for every ref deletion. Each lane has its own durable quarantine +guard. A lane cannot start while its checkout is quarantined, and a checkout +cannot start while any lane beneath it is. + +Lanes require native-srt commands and at least two lease slots, and cannot yet +be combined with conversation worktrees. Code API must advertise +`supportedWorkspaceScopes`; older deployments do not, and the worker omits the +scope for them. Deploy consumers that read worker status (such as LibreChat) +with support for `workspaceScopes` before enabling lanes on a worker. + +##### Retiring stale linked worktrees + +Nothing else removes a task's worktree once its work is pushed, and each one +keeps its own dependencies and build output. A worker with lanes therefore +retires stale ones itself, with no configuration: a pass runs two minutes after +startup and every six hours after that, sooner while a backlog remains, and +also before managed environment setup would be deferred for low disk space. +Passes run in the background and never overlap. Only an actual removal holds +back requests, and only those for that lane or its checkout. + +A worktree in a writable registered root is retired only when **all** of these +hold; otherwise it is kept and the reason is counted: + +- It verifies as a linked worktree of the checkout under `.worktrees/`, as for + lane admission. The checkout itself and worktrees elsewhere are never + touched. +- Neither the lane nor its checkout has a request in flight or ran one while + the worktree was being inspected, and both this + worker's last use of the lane and the newest on-disk activity (the worktree + directory and its Git `HEAD`, `index`, `logs/HEAD`, `ORIG_HEAD` and + `FETCH_HEAD`) are older than the idle threshold, seven days by default. +- It has no modified tracked files and no untracked files the repository does + not ignore; no merge, rebase, cherry-pick, revert or bisect in progress; no + `git worktree lock`; and no quarantine on the lane or its checkout. +- Its `HEAD` commit, including a detached one, is contained in at least one + remote-tracking ref (`refs/remotes/*`), or its work is already in the + remote's default branch by content, as after a squash or rebase merge whose + remote branch was deleted. Content counts only when `HEAD` has no live + upstream (it is detached, never pushed, or its upstream ref is gone) and + either every commit beyond the default branch is patch-equivalent to one in + it (`git cherry` shows only `-`, with no merge commits), or the default + branch has identical content at every path the branch changed since their + merge base. The default branch is the remote's `HEAD`, else `main` or + `master`, as last fetched; the worker never fetches or calls a hosting API. + Commits beyond a live upstream are never treated as merged. + +Removal is `git worktree remove` **without** `--force`, so Git re-checks for +changes itself and deletes only that worktree's metadata. No repository-wide +`git worktree prune` runs, so other registered worktrees that are temporarily +unavailable stay registered. Removal has no timeout, because a half-deleted +worktree could no longer be recognized. Ignored files such as `node_modules`, +build output and ignored `.env` files go with the worktree; that is the space +being reclaimed. The branch is kept, so +`git worktree add .worktrees/ ` restores the worktree. Lane and +checkout requests that arrive during a removal wait for it, or for their own +cancellation; a lane request then fails as an unknown worktree. Each pass reads +every `.worktrees` entry, inspects at most 128 idle worktrees and removes at +most 32, oldest first, rotating so that worktrees kept for lasting reasons +cannot hide the rest. Each pass logs one summary line, for example +`worktree retirement: retired 3, kept 12 (dirty 2, recent 8, unpushed 2), freed +about 4.1 GiB`; set `LIBRECHAT_CODE_LOG_LEVEL=debug` to log every kept worktree +and its reason. + +Pass `--no-worktree-retirement` or set `LIBRECHAT_CODE_WORKTREE_RETIREMENT=false` +to disable retirement. Change the idle threshold with `--worktree-idle-days ` +or `LIBRECHAT_CODE_WORKTREE_IDLE_DAYS=` (1 to 3650 days). + +On an updated Code API, admission waits up to 30 seconds without the +`X-LibreChat-Workspace-Queue-Wait-Ms` request header. A caller may advertise a +positive integer millisecond allowance up to five minutes, capped by any server +queue ceiling. This allowance is separate from the `JOB_TIMEOUT` execution +budget and cannot outlast a shorter client or proxy timeout. A +`WORKSPACE_QUEUE_TIMEOUT` response (HTTP 503, +`Retry-After: 1`) means the operation was not assigned or started; wait for capacity before submitting it again. This is distinct from `ASSIGNMENT_EXPIRED` or a transport timeout after dispatch, where execution may have occurred and mutations must not be blindly retried. No automatic retry is added by this policy. +Align the client's per-attempt timeout and every proxy with the queue **plus** +execution, settlement, and delivery budget before relying on a longer wait. +LibreChat keeps the 30-second admission allowance unless its longer total HTTP +budget is explicitly enabled and the live ingress path is verified. See the +[BYOM worker admission guide](../../docs/byom-worker-admission.md) for the +timeout calculations. Keep the existing URL, pairing/identity, and network policy configuration. The primary root keeps its configured workspace ID (default `primary`). Repeat @@ -828,6 +1079,11 @@ To recover a quarantined native root: 3. Run the normal worker command with all its root/slot options plus `--reset-workspace-quarantine second`. This verifies the local guard is cleared, resets the server fence, then exits. 4. Restart the normal worker command without the reset option. +A linked-worktree lane keeps its own guard and fence. Inspect or restore +`/.worktrees/`, clear its guard with +`--worker-dir /.worktrees/`, then add +`--reset-workspace-worktree ` to the reset command in step 3. + The workspace selector in LibreChat must preserve these registered IDs. Adding roots here does not grant a principal access or change an agent's selected root. # Named project environments @@ -866,7 +1122,7 @@ worker runs. This inspection happens at startup, not on the command hot path. Setup is an operator-authorized startup command under the configured native sandbox policy. It requires commands to be enabled, runs once per worker startup before -registration, and must be idempotent for restarts. Its timeout is bounded to five +registration by default, and must be idempotent for restarts. Its timeout is bounded to five minutes and captured output to 8 KiB. Setup failure prevents registration. A nonzero exit, timeout, crash or uncertain termination retains the workspace quarantine marker; inspect the workspace before running `librechat-code clear-workspace-quarantine @@ -874,8 +1130,203 @@ inspect the workspace before running `librechat-code clear-workspace-quarantine deployment and identity configuration. Only use the separate `--reset-workspace-quarantine ` run option afterward if a server fence also needs clearing. Only successful setup automatically clears its marker. + +## Reusing a prepared checkout + +Opt in to checkout-local preparation reuse when setup is an installation rather +than work that must run on every startup: + +```yaml +setup: + command: npm ci + timeoutMs: 300000 + reuse: + inputs: + - package.json + - package-lock.json + - packages/api/package.json + - packages/data-provider/package.json + checkCommand: test -f node_modules/.package-lock.json && test -x node_modules/.bin/tsc + checkTimeoutMs: 10000 +``` + +Declare **all** relevant manifests, installation configuration and lifecycle-script +inputs. There is no globbing or automatic monorepo discovery. Inputs must be +existing root-confined regular files: at most 32, 8 MiB per file and 32 MiB total. +Include an operator-maintained toolchain/version file if installation uses tools +other than the worker's Node runtime. Lockfile equality alone does not prove that +arbitrary postinstall scripts are reusable. + +The worker fingerprints declared file bytes, the setup recipe, checkout inode, +Node version/ABI, platform/architecture and native command policy. A matching +worker-owned receipt runs the readiness check instead of setup. A nonzero check +reruns setup; a timed-out, signalled or aborted check fails without starting a +replacement command. After successful setup, the check must pass and inputs must +remain unchanged before the worker publishes a receipt. Receipts are bounded, +owner-only files alongside the identity, outside all registered roots and denied +to native commands. Deleting one causes setup to run again; it never clears a +quarantine. Checks are operator commands under the same sandbox and mutation guard +as setup, not unsandboxed host scripts. Keep them cheap and non-mutating. + +Existing definitions without `reuse` retain the startup behavior. Fresh conversation +instances use the same preparation contract, with independent checkout receipts. +This does not attach another checkout's `node_modules`, provision linked lanes on +command admission, or recheck existing instances on every command. It does not +deduplicate installed dependencies between worktrees unless snapshots below are +configured, or enforce disk quotas. + +Shared tool cache grants below do not attach another checkout's installed dependency +tree. Keep monorepo links and mutable outputs checkout-local. Do not broaden the +sandbox root or symlink another branch's full installation. Shared download caches +alone do not reduce installed `node_modules` copies. + +## Copy-on-write installed dependencies + +For matching fresh checkouts on the **same clone-capable filesystem**, add a +private snapshot store to the readiness contract: + +```yaml +setup: + command: npm ci + timeoutMs: 300000 + reuse: + inputs: [package.json, package-lock.json] + checkCommand: test -x node_modules/.bin/tsc + snapshot: + store: /srv/lia-state/dependency-snapshots + paths: [node_modules] + maxBytes: 4294967296 + maxFiles: 200000 +``` + +Create the external store as the worker account with mode `0700`. It must not +overlap any source root, definition, credential or shared tool cache. Commands +cannot read or write it. Use a separate store per project/trust domain. The +portable key includes project identity, declared input bytes, recipe, policy, +Node ABI and platform, but not the checkout inode. Kernel locks serialize setup +for one key; different keys remain independent. Incomplete clones are never +published. Cancellation is checked during bounded traversal. + +Matching snapshots restore only when **every** declared `node_modules` directory +is missing. The sandboxed readiness check must pass before accepting the restore. +Existing directories are never replaced by restoration; ordinary setup handles +repair. Include nested workspace installations explicitly. Relative checkout-local +package links are preserved; absolute/escaping links, hard-linked files, links into +Git/worktree control paths and special files are rejected. Hardlink-based package +manager layouts need a different adapter; this snapshot mode targets npm copies. +Changing one restored installation cannot modify the snapshot or another checkout. + +This requires APFS clones or Linux reflinks (for example a suitably configured +XFS/Btrfs volume). The worker verifies cloning before installation and rejects +unsupported filesystems instead of silently making full copies or writable hard +links. Ordinary ext4 workers should leave snapshots disabled and can still use +checkout-local preparation receipts and shared downloads. This is **not** Python +virtualenv relocation, automatic preparation of manually created linked lanes, +or a hard quota on arbitrary commands. Validate all install/postinstall inputs +and path-independent artifacts before opting in. + +## Shared tool and download resources + +Explicit operator-managed stores can live outside the checkouts: + +```yaml +resources: + - kind: npm-cache + path: /srv/lia-resources/npm + access: read-write + - kind: uv-cache + path: /srv/lia-resources/uv + access: read-write + - kind: playwright-browsers + path: /srv/lia-resources/playwright + access: read-only +``` + +Create each directory as the worker service account with mode `0700`, then populate +browser binaries using the matching Playwright version and `PLAYWRIGHT_BROWSERS_PATH` +outside the coding session. Roots must exist and may not be symlinks or overlap any +registered workspace or worker control state. Linux mount-alias checks cover both +directions. Keep the mount namespace stable while the worker runs. + +The worker grants only these paths and injects `npm_config_cache`, `UV_CACHE_DIR` +and `PLAYWRIGHT_BROWSERS_PATH` from the loaded definition, including in linked lanes +and fresh conversation worktrees. An undeclared process environment variable never +grants filesystem access. Each store root is inode-bound and revalidated before +commands. Read-only stores stay read-only, and speculative programmatic probes +cannot write any shared store. + +Sharing is explicit within one worker's trust domain. Do not share mutable caches +between unrelated principals, store credentials in them, or treat their contents +as trusted worker code. Separate package caches from browsers and mutable browser +profiles, Redis data, build output and test state. Stores currently have no automatic +eviction. The snapshot lifecycle below does not delete these mutable stores. This shares downloads +and browser binaries, not installed `node_modules` trees. No setup output is sent to the model. +## Storage admission and snapshot lifecycle + +Use an explicit free-space floor before managed setup or restoration: + +```yaml +storage: + minFreeBytes: 5368709120 # 5 GiB left for the worker and other activity + setupReserveBytes: 2147483648 # estimated installation headroom +``` + +This works without dependency snapshots. Absent `storage`, preparation retains +today's behavior. A ready checkout can still pass its readiness check when disk +is low. When installation is needed, low space rejects it **before** starting the +setup command, with a specific remediation message. These are soft admission +checks, not allocated reservations: concurrent processes and arbitrary shell +writes can still consume space after admission. Put worker identity/quarantine +state on a separate small volume and use filesystem/project quotas for hard +containment. This option neither clears quarantine nor deletes source to recover. + +The private snapshot store can bound reproducible dependency versions: + +```yaml +setup: + # command and reuse inputs/readiness omitted here; retain the full contract above + reuse: + snapshot: + store: /srv/lia-state/dependency-snapshots + paths: [node_modules] + lifecycle: + maxStoreBytes: 21474836480 + maxEntries: 8 + retentionMs: 432000000 + scanLimit: 4096 +``` + +Cleanup runs during managed preparation and publication. Oldest inactive snapshots +are reclaimed by last-use age and logical payload/count budgets. Per-key kernel +locks protect installs, restores and publishers; maintenance never waits on or +deletes an active key. A short store-wide lock makes budget checks and publication +atomic, without serializing installations for different keys. Abandoned staging +directories require a worker ownership manifest and a free key lock before removal. +Unknown/unmarked entries are reported and preserved. Lock files are kept to avoid +splitting lock ownership. Scan limits fail closed instead of crawling unbounded data. + +The byte budget counts snapshot file lengths, not deduplicated physical blocks, +metadata overhead or working checkout copies. An otherwise valid prepared checkout +does not fail when the cache cannot fit another entry: publication is skipped with +a worker diagnostic, and later low-space admission still applies. + +Preview or explicitly apply maintenance without restarting the worker: + +```sh +librechat-code prune-environment-storage --environment /etc/librechat-code/app.yaml +librechat-code prune-environment-storage --environment /etc/librechat-code/app.yaml --apply +``` + +Pass all relevant definitions with repeated `--environment` flags so their roots +participate in isolation checks. Preview is the default and the JSON identifies +`dryRun`, proposed removals, active keys, unknown data and retained logical usage. +The command is host-operator-only, not an agent workspace action. It does not touch +source worktrees (including dirty or unpushed branches), `.verification`, build +outputs, arbitrary installations or mutable npm/uv caches. Source worktree archival +requires the separate worktree ownership/binding lifecycle, not an mtime heuristic. + Named actions are fixed commands without model-supplied substitution. The bridge advertises only their names and the definition fingerprint, never their shell source or host root. A command request can select `environmentAction: { name, fingerprint }`; diff --git a/packages/code/WORKSPACE-READS.md b/packages/code/WORKSPACE-READS.md new file mode 100644 index 00000000..f3edbe58 --- /dev/null +++ b/packages/code/WORKSPACE-READS.md @@ -0,0 +1,32 @@ +# Workspace file read contract + +Ordinary `read_file` streams complete LF-delimited line windows through the verified +file descriptor. It defaults to 200 lines, accepts at most 500, and returns at most +1 MiB of UTF-8 content, including inter-line separators, regardless of file size. +CR in CRLF is preserved; the final LF does not add a phantom line. An empty file +has one empty line. A start beyond EOF returns an empty, non-truncated window. + +Byte-limited windows stop before the first line that cannot fit and return +`nextStartLine = endLine + 1`. If the first requested line exceeds 1 MiB, the read +fails with `READ_LIMIT_EXCEEDED`, without a repeating continuation. Scanning and +collection check cancellation and share a 10-second deadline. Far-away starts +can fail with `READ_LIMIT_EXCEEDED`; use an earlier start or `search_text`. +Cancellation and timeout settle the request without waiting for queued I/O. +Node retains an interrupted read's descriptor and chunk until that I/O drains; +file and held-root cleanup are initiated immediately and finish asynchronously. +This releases the request lane, but does not cancel kernel I/O. + +Memory is bounded by 64 KiB chunks and the returned window. Reads stop at the +opened file's initial size, so concurrent growth cannot extend the scan. Truncation +ends the scan at observed EOF; pathname replacement never switches the open +handle. In-place writes can change observed content. Pagination is not a snapshot +across requests and assumes the file stays unchanged. + +UTF-8 and BOM-marked UTF-16LE/BE are decoded incrementally. One leading BOM is +removed; malformed sequences use replacement characters, as before. Binary bytes +are decoded as text, including NUL, not classified or rejected. Neither a successful +window nor continuation validates unread content. + +Request/result shapes, protocol version, and capabilities are unchanged. Search, +preview, edit, and write limits remain unchanged. Digest-checked repository +instruction snapshots retain their separate byte-bounded, newline-preserving path. diff --git a/packages/code/package-lock.json b/packages/code/package-lock.json index 426b950f..a27188a6 100644 --- a/packages/code/package-lock.json +++ b/packages/code/package-lock.json @@ -21,7 +21,7 @@ "typescript": "^5.5.4" }, "engines": { - "node": ">=20.11" + "node": ">=22.21" } }, "node_modules/@anthropic-ai/sandbox-runtime": { diff --git a/packages/code/package.json b/packages/code/package.json index b00ca807..95eea841 100644 --- a/packages/code/package.json +++ b/packages/code/package.json @@ -15,6 +15,10 @@ "types": "./dist/protocol.d.ts", "import": "./dist/protocol.js" }, + "./identity": { + "types": "./dist/identity.d.ts", + "import": "./dist/identity.js" + }, "./worker": { "types": "./dist/worker.d.ts", "import": "./dist/worker.js" @@ -61,7 +65,7 @@ "typescript": "^5.5.4" }, "engines": { - "node": ">=20.11" + "node": ">=22.21" }, "dependencies": { "@anthropic-ai/sandbox-runtime": "0.0.75", diff --git a/packages/code/src/cli.test.ts b/packages/code/src/cli.test.ts index f95d7eee..f6737d5f 100644 --- a/packages/code/src/cli.test.ts +++ b/packages/code/src/cli.test.ts @@ -358,6 +358,35 @@ test('CLI rejects checkout routing outside a trusted VM or without repository-sc assert.match(invalid.stderr, /must be admitted or checkout/); }); +test('CLI permits installation-scoped GitHub tokens only for a trusted VM with routed App auth', () => { + const cli = fileURLToPath(new URL('./cli.js', import.meta.url)); + const base = { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + LIBRECHAT_CODE_WORKER_DIR: process.cwd(), + LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS: 'true', + LIBRECHAT_CODE_GITHUB_TOKEN: undefined, + LIBRECHAT_CODE_GITHUB_APP_ID: '123', + LIBRECHAT_CODE_GITHUB_PRIVATE_KEY_FILE: '/does/not/matter', + LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: undefined, + LIBRECHAT_CODE_GITHUB_TOKEN_SCOPE: 'installation', + }; + const restricted = spawnSync(process.execPath, [cli], { encoding: 'utf8', env: base }); + assert.match(restricted.stderr, /Installation-scoped GitHub tokens require the trusted-vm/); + const trusted = spawnSync(process.execPath, [cli], { + encoding: 'utf8', + env: { ...base, LIBRECHAT_CODE_COMMAND_POLICY_PRESET: 'trusted-vm' }, + }); + assert.doesNotMatch(trusted.stderr, /Installation-scoped GitHub tokens require/); + const fixed = spawnSync(process.execPath, [cli], { + encoding: 'utf8', + env: { ...base, LIBRECHAT_CODE_GITHUB_INSTALLATION_ID: '456' }, + }); + assert.match(fixed.stderr, /without a fixed installation ID/); +}); + test('CLI requires a runtime image for Docker supervision', () => { const result = spawnSync( process.execPath, diff --git a/packages/code/src/cli.ts b/packages/code/src/cli.ts index 841164fb..85ce947e 100644 --- a/packages/code/src/cli.ts +++ b/packages/code/src/cli.ts @@ -1,8 +1,8 @@ #!/usr/bin/env node import { createHash, createHmac, randomBytes } from 'node:crypto'; import { readFileSync } from 'node:fs'; -import { realpath, stat } from 'node:fs/promises'; -import { basename, resolve, relative, isAbsolute, sep } from 'node:path'; +import { readdir, realpath, stat } from 'node:fs/promises'; +import { basename, dirname, join, resolve, relative, isAbsolute, sep } from 'node:path'; import { pairBridgeWorker } from './pairing.js'; import { discoverProjects } from './projects.js'; @@ -12,6 +12,9 @@ import { assertEnvironmentDefinitionsOutsideRoots, EnvironmentWorkspaceTools, } from './environment.js'; +import { prepareCodeEnvironment } from './environment-preparation.js'; +import { assertEnvironmentResourceIsolation } from './environment-resources.js'; +import { pruneDependencySnapshots } from './snapshot-lifecycle.js'; import { startFileRelay } from './relay.js'; import { DockerFileRelaySupervisor } from './relay-runtime.js'; import { @@ -26,6 +29,7 @@ import { loadWorkspaceMutationQuarantine, saveBridgeIdentity, saveWorkspaceMutationQuarantine, + prepareEnvironmentPreparationDirectory, } from './storage.js'; import { BridgeWorker } from './worker.js'; import { LocalWorkspaceTools, SandboxWorkspaceTools } from './workspace.js'; @@ -37,7 +41,13 @@ import { RuntimeWorkspaceCommandSandbox } from './workspace-runtime.js'; import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; import { NativeWorkspaceCommandPool } from './native-pool.js'; import { GitWorktreeWorkspaceTools, internalWorkspaceId } from './workspace-instances.js'; +import { LaneGitWorkspaceTools, ownsGitMetadata, ownsLinkedWorktreeMetadata } from './lane-git.js'; +import { LINKED_WORKTREE_DIRECTORY, LinkedWorktreeWorkspaceTools, verifyLinkedWorktree } from './linked-worktrees.js'; import { GitWorktreeManager } from './worktrees.js'; +import { + WorktreeRetirementScheduler, + worktreeRetirementSettings, +} from './worktree-retirement.js'; import { captureWorkspaceRootIdentity } from './root-identity.js'; import { resolveNativeSrtCommandPolicy, @@ -66,6 +76,7 @@ import { BridgeProtocolError, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, + isValidLinkedWorktreeName, workspaceIsolationKey, } from './protocol.js'; @@ -157,6 +168,7 @@ function githubCredentials(args: string[]): { mode?: 'app' | 'token'; repositoryRouting?: boolean; checkoutRouting?: boolean; + installationTokenScope?: boolean; policyIdentity: string; } { const token = nonEmpty(process.env.LIBRECHAT_CODE_GITHUB_TOKEN); @@ -191,6 +203,18 @@ function githubCredentials(args: string[]): { 'Checkout GitHub repository routing requires a GitHub App without a fixed installation ID', ); } + const tokenScope = + option(args, '--github-token-scope')?.trim().toLowerCase() ?? + process.env.LIBRECHAT_CODE_GITHUB_TOKEN_SCOPE?.trim().toLowerCase() ?? + 'repository'; + if (tokenScope !== 'repository' && tokenScope !== 'installation') { + throw new Error('GitHub token scope must be repository or installation'); + } + if (tokenScope === 'installation' && (!hasApp || installationId)) { + throw new Error( + 'Installation-scoped GitHub tokens require a GitHub App without a fixed installation ID', + ); + } const configuredHostValue = nonEmpty( process.env.LIBRECHAT_CODE_GITHUB_HOST, ); @@ -225,16 +249,19 @@ function githubCredentials(args: string[]): { mode: 'app', repositoryRouting: !installationId, checkoutRouting: routing === 'checkout', + installationTokenScope: tokenScope === 'installation', policyIdentity: gitHubAuthenticationPolicyIdentity({ mode: 'app', host, appId, installationId, - }) + (routing === 'checkout' ? ':routing:checkout' : ''), + }) + (routing === 'checkout' ? ':routing:checkout' : '') + + (tokenScope === 'installation' ? ':scope:installation' : ''), privateKeyPath, provider: new GitHubAppCredentialProvider({ appId: appId!, installationId, + tokenScope, privateKeyPath: privateKeyPath!, host, apiUrl, @@ -498,6 +525,11 @@ async function run( (args.includes('--allow-workspace-commands') || process.env.LIBRECHAT_CODE_ALLOW_WORKSPACE_COMMANDS?.trim().toLowerCase() === 'true'); + const linkedWorktreeLanes = + runtimeSessionId == null && + (args.includes('--linked-worktree-lanes') || + process.env.LIBRECHAT_CODE_LINKED_WORKTREE_LANES?.trim().toLowerCase() === + 'true'); const commandSandboxMode = option(args, '--command-sandbox') ?? process.env.LIBRECHAT_CODE_COMMAND_SANDBOX?.trim().toLowerCase() ?? @@ -568,6 +600,11 @@ async function run( 'Checkout GitHub repository routing requires the trusted-vm command policy', ); } + if (github.installationTokenScope && commandPolicy.preset !== 'trusted-vm') { + throw new Error( + 'Installation-scoped GitHub tokens require the trusted-vm command policy', + ); + } const githubDomains = github.provider ? github.host === 'github.com' ? [...GITHUB_ALLOWED_DOMAINS] @@ -762,6 +799,31 @@ async function run( 'Conversation worktrees require native-srt commands and at least two workspace lease slots', ); } + if ( + linkedWorktreeLanes && + (!allowWorkspaceCommands || + commandSandboxMode !== 'native-srt' || + workspaceLeaseSlots < 2) + ) { + throw new Error( + 'Linked worktree lanes require native-srt commands and at least two workspace lease slots', + ); + } + if (linkedWorktreeLanes && conversationWorktreeRoot) { + throw new Error( + 'Linked worktree lanes cannot be combined with conversation worktrees', + ); + } + if (linkedWorktreeLanes && process.platform === 'win32') { + throw new Error('Linked worktree Git guard requires a POSIX host'); + } + const worktreeRetirement = worktreeRetirementSettings({ + optOut: args.includes('--no-worktree-retirement'), + enabled: process.env.LIBRECHAT_CODE_WORKTREE_RETIREMENT, + idleDays: + option(args, '--worktree-idle-days') ?? + process.env.LIBRECHAT_CODE_WORKTREE_IDLE_DAYS, + }); if ( roots.length > 1 && process.env.LIBRECHAT_CODE_WORKSPACE_QUARANTINE_FILE?.trim() @@ -780,6 +842,34 @@ async function run( }), ]), ); + const preparationDirectory = join(dirname(identityPath ?? defaultBridgeIdentityPath(workerId)), 'environment-preparation'); + if (environments.some(environment => environment.definition.setup?.reuse)) { + const sourceParents = await prepareEnvironmentPreparationDirectory(preparationDirectory); + // Reuse the mount/ancestor isolation checks for this worker-owned state directory. + await assertEnvironmentDefinitionsOutsideRoots([{ + path: preparationDirectory, sourceParents, + definition: { name: 'preparation-state', root: preparationDirectory }, fingerprint: '', + }], roots); + } + // Cache contents may be shared explicitly; their grants must never authorize source or control state. + const resourceRoots = environments.flatMap(environment => (environment.resources ?? []).map(resource => ({ + path: resource.path, sourceParents: resource.controlPaths, + definition: { name: 'shared-resource', root: resource.path }, fingerprint: '', + }))); + await assertEnvironmentResourceIsolation(environments.flatMap(environment => environment.resources ?? []), roots.map(root => root.root), [ + identityPath, preparationDirectory, github.privateKeyPath, + ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), + ].filter((path): path is string => path != null)); + await assertEnvironmentDefinitionsOutsideRoots(resourceRoots, roots); + const snapshotStores = environments.flatMap(environment => environment.snapshotStore ? [environment.snapshotStore] : []); + await assertEnvironmentResourceIsolation(snapshotStores.map(store => ({ kind: 'npm-cache' as const, path: store.store, access: 'read-only' as const })), roots.map(root => root.root), [ + identityPath, preparationDirectory, github.privateKeyPath, ...environments.map(environment => environment.path), + ...rootQuarantinePaths.values(), ...environments.flatMap(environment => (environment.resources ?? []).map(resource => resource.path)), + ].filter((path): path is string => path != null)); + await assertEnvironmentDefinitionsOutsideRoots(snapshotStores.map(store => ({ path: store.store, sourceParents: store.controlPaths, + definition: { name: 'dependency-store', root: store.store }, fingerprint: '' })), roots); + const preparationReceipt = (root: string) => join(preparationDirectory, + `${createHash('sha256').update(JSON.stringify([codeApiUrl, workerId, root])).digest('hex')}.json`); // Keep an admission boundary even when trusted-VM checkout routing uses a // nested repository's remote for the current command. const admittedGitHubRepositories = github.provider && github.repositoryRouting @@ -1002,6 +1092,8 @@ async function run( commandPolicy, protectedPaths: [ identityPath, + ...(environments.some(environment => environment.definition.setup?.reuse) ? [preparationDirectory] : []), + ...snapshotStores.map(store => store.store), ...environments.map(environment => environment.path), ...rootQuarantinePaths.values(), github.privateKeyPath, @@ -1050,19 +1142,23 @@ async function run( } : {}), }; + const nativeOptionsForWorkspace = (workspaceId: string): NativeProcessSandboxOptions => ({ + ...nativeOptions, + resources: environments.find(environment => environment.definition.name === workspaceId)?.resources, + }); const nativeCommandSandbox = allowWorkspaceCommands && commandSandboxMode === 'native-srt' - ? roots.length > 1 || workspaceLeaseSlots > 1 || conversationWorktreeRoot + ? roots.length > 1 || workspaceLeaseSlots > 1 || conversationWorktreeRoot || linkedWorktreeLanes ? new NativeWorkspaceCommandPool( new Map( roots.map(root => [ root.id, - { ...nativeOptions, workspaceRoot: root.root, workspaceIdentity: root.identity }, + { ...nativeOptionsForWorkspace(root.id), workspaceRoot: root.root, workspaceIdentity: root.identity }, ]), ), workspaceLeaseSlots, ) - : new NativeProcessWorkspaceCommandSandbox(nativeOptions) + : new NativeProcessWorkspaceCommandSandbox(nativeOptionsForWorkspace(roots[0].id)) : undefined; if (allowWorkspaceCommands && workspaceTools) { workspaceTools = new SandboxWorkspaceTools({ @@ -1088,10 +1184,11 @@ async function run( ...(nativeCommandSandbox instanceof NativeWorkspaceCommandPool ? { prepareInstance: async (instance, signal) => { - const setup = environments.find( + const environment = environments.find( (environment) => environment.definition.name === instance.sourceWorkspaceId, - )?.definition.setup; + ); + const setup = environment?.definition.setup; if (!setup) return; if (admittedGitHubRepositories) { admittedGitHubRepositories.set( @@ -1101,26 +1198,28 @@ async function run( } const id = internalWorkspaceId(instance.sourceWorkspaceId, instance.id); await nativeCommandSandbox.registerRoot(id, { - ...nativeOptions, + ...nativeOptionsForWorkspace(instance.sourceWorkspaceId), workspaceIdentity: instance.identity, workspaceRoot: instance.root, }); - const result = await nativeCommandSandbox.execute( - { + await prepareCodeEnvironment({ + snapshotStore: environment!.snapshotStore, + storage: environment!.definition.storage, + snapshotScope: environment!.definition.repo ?? environment!.definition.name, + root: instance.root, identity: instance.identity, setup, + receiptPath: preparationReceipt(instance.root), + context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity, + nativeOptionsForWorkspace(instance.sourceWorkspaceId).resources]), + signal, + execute: (command, timeoutMs) => nativeCommandSandbox.execute({ protocolVersion: 1, operation: 'execute_command', workspaceId: id, - command: setup.command, - timeoutMs: setup.timeoutMs, + command, + timeoutMs, maxOutputBytes: 8192, - }, - signal, - ); - if (result.exitCode !== 0 || result.timedOut) { - throw new Error( - `Environment ${instance.sourceWorkspaceId} setup failed for its conversation worktree`, - ); - } + }, signal), + }); }, discardInstance: async (instance) => { await nativeCommandSandbox.unregisterRoot( @@ -1167,7 +1266,7 @@ async function run( root.id, { command: { - ...nativeOptions, + ...nativeOptionsForWorkspace(root.id), workspaceIdentity: root.identity, workspaceRoot: root.root, }, @@ -1179,6 +1278,98 @@ async function run( }); workspaceTools = conversationWorkspaceTools; } + let linkedWorktreeTools: LinkedWorktreeWorkspaceTools | undefined; + if (linkedWorktreeLanes && workspaceTools) { + if (!(nativeCommandSandbox instanceof NativeWorkspaceCommandPool)) { + throw new Error('Linked worktree lanes require a native command pool'); + } + linkedWorktreeTools = new LinkedWorktreeWorkspaceTools({ + commandPool: nativeCommandSandbox, + delegate: workspaceTools, + programmaticDelegate: conversationWorkspaceTools ?? nativeCommandSandbox, + onResolve(workspaceId, root) { + if (admittedGitHubRepositories) { + admittedGitHubRepositories.set( + root, + repositoriesByWorkspace?.get(workspaceId), + ); + } + }, + onRelease(root) { + admittedGitHubRepositories?.delete(root); + }, + sources: new Map( + roots.map((root) => [ + root.id, + { + root: root.root, + identity: root.identity, + command: nativeOptionsForWorkspace(root.id), + repositoryInstructions: args.includes('--repository-instructions'), + writable: root.writable ?? false, + }, + ]), + ), + }); + workspaceTools = linkedWorktreeTools; + } + // Probe Git only while the active registration advertises lane_git, which the worker decides. + let laneGitWorker: BridgeWorker | undefined; + if (workspaceTools?.capabilities.operations.includes('execute_command')) { + workspaceTools = new LaneGitWorkspaceTools({ + delegate: workspaceTools, + isEnabled: () => laneGitWorker?.commandResultFeatureActive('lane_git') === true, + async resolveRoot(request, signal) { + const source = roots.find((root) => root.id === request.workspaceId); + if (!source) return undefined; + if (request.workspaceInstanceId != null) { + const root = await conversationWorktrees?.plannedRoot(request.workspaceId, request.workspaceInstanceId); + return root != null && (await ownsGitMetadata(root, signal)) ? root : undefined; + } + if (request.worktree != null) { + signal?.throwIfAborted(); + const lane = await verifyLinkedWorktree(source.root, request.worktree, source.identity); + return (await ownsLinkedWorktreeMetadata(lane.commonGitDir, request.worktree, signal)) ? lane.root : undefined; + } + return (await ownsGitMetadata(source.root, signal)) ? source.root : undefined; + }, + }); + } + const retirementSources = roots.filter((root) => root.writable); + const debugLogs = + process.env.LIBRECHAT_CODE_LOG_LEVEL?.trim().toLowerCase() === 'debug'; + const worktreeRetirementScheduler = + linkedWorktreeTools && + worktreeRetirement.enabled && + retirementSources.length > 0 && + option(args, '--reset-workspace-quarantine') == null + ? new WorktreeRetirementScheduler({ + activity: linkedWorktreeTools, + idleMs: worktreeRetirement.idleMs, + sources: retirementSources.map((root) => ({ + workspaceId: root.id, + root: root.root, + identity: root.identity, + })), + async isQuarantined(selectedWorkspaceId, worktree) { + const source = roots.find((root) => root.id === selectedWorkspaceId); + if (!source) return true; + const path = + worktree == null + ? rootQuarantinePaths.get(selectedWorkspaceId)! + : defaultWorkspaceQuarantinePath({ + codeApiUrl, + workerId, + workspaceRoot: join(source.root, LINKED_WORKTREE_DIRECTORY, worktree), + }); + return (await loadWorkspaceMutationQuarantine(path)) != null; + }, + log(level, message) { + if (level === 'debug' && !debugLogs) return; + process.stdout.write(`librechat-code: ${message}\n`); + }, + }) + : undefined; if (workspaceTools && environments.length) { workspaceTools = new EnvironmentWorkspaceTools( workspaceTools, @@ -1242,33 +1433,55 @@ async function run( incarnationId, ); await guard.assertAvailable(); - await guard.arm('Environment setup did not settle', 'setup'); - const result = await nativeCommandSandbox.execute( - { + let armed = false; + const preparation = await prepareCodeEnvironment({ + snapshotStore: environment.snapshotStore, + storage: environment.definition.storage, + snapshotScope: environment.definition.repo ?? environment.definition.name, + beforeMutation: async () => { + if (!armed) { await guard.arm('Dependency restoration did not settle', 'setup'); armed = true; } + }, + root: environment.definition.root, + identity: roots.find(root => root.id === id)!.identity!, + setup, receiptPath: preparationReceipt(environment.definition.root), + reclaimSpace: worktreeRetirementScheduler + ? () => worktreeRetirementScheduler.runNow() + : undefined, + context: JSON.stringify([serializeNativeSrtCommandPolicy(commandPolicy), commandAllowedDomains, github.policyIdentity, + nativeOptionsForWorkspace(id).resources]), + signal: controller.signal, + execute: async (command, timeoutMs) => { + if (!armed) { + await guard.arm('Environment preparation did not settle', 'setup'); + armed = true; + } + return nativeCommandSandbox.execute({ protocolVersion: 1, operation: 'execute_command', workspaceId: id, - command: setup.command, - timeoutMs: setup.timeoutMs, + command, + timeoutMs, maxOutputBytes: 8192, + }, controller.signal); }, - controller.signal, - ); - if (result.exitCode !== 0 || result.timedOut) { + }).catch(error => { throw new Error( `Environment ${id} setup failed; inspect the workspace and use clear-workspace-quarantine with its root and workspace ID before restarting`, + { cause: error }, ); - } + }); await guard.clear('setup'); process.stdout.write( - `librechat-code: environment ${id} prepared\n`, + `librechat-code: environment ${id} ${preparation}\n`, ); } } catch (error) { + await worktreeRetirementScheduler?.stop().catch(() => undefined); await nativeCommandSandbox?.close().catch(() => undefined); await fileRelaySupervisor?.stop().catch(() => undefined); throw error; } + worktreeRetirementScheduler?.start(); try { const worker = new BridgeWorker({ instructionDescriptors: () => localWorkspaceTools?.instructionDescriptors() ?? Promise.resolve(undefined), @@ -1283,7 +1496,7 @@ async function run( ...(nativeProgrammaticEnabled && nativeCommandSandbox ? { workspaceProgrammatic: - conversationWorkspaceTools ?? nativeCommandSandbox, + linkedWorktreeTools ?? conversationWorkspaceTools ?? nativeCommandSandbox, } : {}), ...(conversationWorktrees @@ -1310,6 +1523,40 @@ async function run( ), } : {}), + ...(linkedWorktreeTools + ? { + linkedWorktreeQuarantineResolver: ( + selectedWorkspaceId: string, + worktree: string, + ) => { + const source = roots.find((root) => root.id === selectedWorkspaceId); + if (!source) { + throw new BridgeProtocolError('Linked worktree source is not registered'); + } + return workspaceMutationGuard( + defaultWorkspaceQuarantinePath({ + codeApiUrl, + workerId, + workspaceRoot: join(source.root, LINKED_WORKTREE_DIRECTORY, worktree), + }), + workerId, + workspaceIsolationKey(selectedWorkspaceId, undefined, worktree), + incarnationId, + ); + }, + linkedWorktreeNames: async (selectedWorkspaceId: string) => { + const source = roots.find((root) => root.id === selectedWorkspaceId); + if (!source) return []; + try { + const entries = await readdir(join(source.root, LINKED_WORKTREE_DIRECTORY)); + return entries.filter(isValidLinkedWorktreeName); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return []; + throw error; + } + }, + } + : {}), ...(workspaceLeaseSlots > 1 || roots.length > 1 ? { workspaceQuarantines: new Map( @@ -1409,6 +1656,7 @@ async function run( ); }, }); + laneGitWorker = worker; if (runtimeSessionId !== undefined) { await worker.refreshCredential(controller.signal); await worker.register(controller.signal); @@ -1424,21 +1672,27 @@ async function run( args, '--reset-workspace-instance', ); + const resetWorkspaceWorktree = option( + args, + '--reset-workspace-worktree', + ); await worker.refreshCredential(controller.signal); await worker.registerForMaintenance(controller.signal); await worker.resetNativeWorkspace( resetNativeRoot, controller.signal, resetWorkspaceInstance, + resetWorkspaceWorktree, ); process.stdout.write( - `librechat-code: reset acknowledged for native workspace ${resetNativeRoot}${resetWorkspaceInstance ? ` instance ${resetWorkspaceInstance}` : ''}\n`, + `librechat-code: reset acknowledged for native workspace ${resetNativeRoot}${resetWorkspaceInstance ? ` instance ${resetWorkspaceInstance}` : ''}${resetWorkspaceWorktree ? ` worktree ${resetWorkspaceWorktree}` : ''}\n`, ); return; } await worker.run(controller.signal); } finally { try { + await worktreeRetirementScheduler?.stop(); await nativeCommandSandbox?.close(); } finally { await fileRelaySupervisor?.stop(); @@ -1512,6 +1766,28 @@ async function clearMutationQuarantine(args: string[]): Promise { async function main(): Promise { const args = process.argv.slice(2); + if (args[0] === 'prune-environment-storage') { + const paths: string[] = []; + for (let i = 1; i < args.length; i++) { + if (args[i] === '--apply') continue; + if (args[i] !== '--environment' || !args[i + 1] || args[i + 1].startsWith('--')) + throw new Error('Usage: librechat-code prune-environment-storage --environment [--environment ...] [--apply]'); + paths.push(args[++i]); + } + if (!paths.length || paths.length > 32) throw new Error('Supply between one and 32 environment definitions'); + const loaded = await Promise.all(paths.map(loadCodeEnvironment)); + const roots = loaded.map(environment => ({ id: environment.definition.name, root: environment.definition.root })); + const stores = loaded.flatMap(environment => environment.snapshotStore ? [environment.snapshotStore] : []); + if (!stores.length) throw new Error('No dependency snapshot stores configured'); + await assertEnvironmentDefinitionsOutsideRoots(loaded, roots); + await assertEnvironmentDefinitionsOutsideRoots(stores.map(store => ({ path: store.store, sourceParents: store.controlPaths, + definition: { name: 'dependency-store', root: store.store }, fingerprint: '' })), roots); + await assertEnvironmentResourceIsolation(stores.map(store => ({ kind: 'npm-cache' as const, path: store.store, access: 'read-only' as const })), + roots.map(root => root.root), [...loaded.map(environment => environment.path), + ...loaded.flatMap(environment => (environment.resources ?? []).map(resource => resource.path))]); + for (const store of stores) process.stdout.write(`${JSON.stringify(await pruneDependencySnapshots(store, { dryRun: !args.includes('--apply') }))}\n`); + return; + } if (args[0] === 'projects') { const root = option(args, '--root'); if (!root || args.slice(1).some((arg, index, rest) => diff --git a/packages/code/src/dependency-snapshots.test.ts b/packages/code/src/dependency-snapshots.test.ts new file mode 100644 index 00000000..f7d59681 --- /dev/null +++ b/packages/code/src/dependency-snapshots.test.ts @@ -0,0 +1,284 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + mkdtemp, + mkdir, + writeFile, + readFile, + lstat, + link, + realpath, + rm, + symlink, + rename, + readdir, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { + parseDependencySnapshot, + loadDependencySnapshot, + publishDependencySnapshot, + restoreDependencySnapshot, + withDependencySnapshot, +} from './dependency-snapshots.js'; +import { prepareCodeEnvironment } from './environment-preparation.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; +import type { EnvironmentPreparationOptions } from './environment-preparation.js'; + +async function fixture(t: test.TestContext) { + const directory = await realpath( + await mkdtemp(join(tmpdir(), 'dependency-snapshot-')), + ); + t.after(() => rm(directory, { recursive: true, force: true })); + const a = join(directory, 'a'), + b = join(directory, 'b'), + storePath = join(directory, 'store'); + for (const path of [a, b, storePath]) await mkdir(path, { mode: 0o700 }); + const store = await loadDependencySnapshot( + parseDependencySnapshot({ store: storePath, paths: ['node_modules'] }), + ); + const ai = await captureWorkspaceRootIdentity(a), + bi = await captureWorkspaceRootIdentity(b); + return { directory, a, b, store, ai, bi, key: 'a'.repeat(64) }; +} + +test('snapshot schema restricts installed directories, overlapping paths and finite bounds', () => { + const valid = { + store: '/private/store', + paths: ['node_modules', 'packages/api/node_modules'], + }; + assert.equal(parseDependencySnapshot(valid).maxFiles, 200_000); + for (const paths of [ + ['.git'], + ['../node_modules'], + ['node_modules', 'node_modules/pkg/node_modules'], + ['node_modules', 'node_modules'], + ]) + assert.throws(() => parseDependencySnapshot({ ...valid, paths })); + assert.throws(() => + parseDependencySnapshot({ ...valid, maxBytes: Infinity }), + ); + assert.throws(() => + parseDependencySnapshot({ ...valid, store: 'relative' }), + ); +}); + +// These assertions exercise real filesystem cloning, not a mocked successful copy. +test( + 'real clone snapshots isolate writable files, preserve checkout-local links and never replace installations', + { + skip: + process.platform !== 'darwin' && + process.env.LIBRECHAT_CODE_LIVE_SNAPSHOT_TESTS !== '1', + }, + async t => { + const { a, b, store, ai, bi, key } = await fixture(t); + await mkdir(join(a, 'node_modules')); + await writeFile( + join(a, 'node_modules', 'package.js'), + 'module.exports = 42', + ); + await mkdir(join(a, 'packages')); + await mkdir(join(b, 'packages')); + await symlink('../packages', join(a, 'node_modules', 'local')); + await withDependencySnapshot(store, key, () => + publishDependencySnapshot(store, key, a, ai), + ); + assert.equal( + await withDependencySnapshot(store, key, () => + restoreDependencySnapshot(store, key, b, bi), + ), + true, + ); + assert.notEqual( + (await lstat(join(a, 'node_modules', 'package.js'))).ino, + (await lstat(join(b, 'node_modules', 'package.js'))).ino, + ); + await writeFile(join(b, 'node_modules', 'package.js'), 'changed'); + assert.equal( + await readFile(join(a, 'node_modules', 'package.js'), 'utf8'), + 'module.exports = 42', + ); + assert.equal( + await readFile(join(store.store, key, '0', 'package.js'), 'utf8'), + 'module.exports = 42', + ); + assert.equal(await restoreDependencySnapshot(store, key, b, bi), false); + assert.equal( + await readFile(join(b, 'node_modules', 'package.js'), 'utf8'), + 'changed', + ); + }, +); + +test('unsafe links and bounded traversal never publish a partial snapshot', async t => { + const { a, store, ai, key } = await fixture(t); + await mkdir(join(a, 'node_modules')); + await symlink('/etc/passwd', join(a, 'node_modules', 'escape')); + await assert.rejects( + publishDependencySnapshot(store, key, a, ai), + /checkout-local/, + ); + assert.deepEqual(await readdir(store.store), []); + await rm(join(a, 'node_modules', 'escape')); + await writeFile(join(a, 'node_modules', 'large'), 'large'); + await assert.rejects( + publishDependencySnapshot({ ...store, maxBytes: 1 }, key, a, ai), + /maxBytes/, + ); + assert.deepEqual(await readdir(store.store), []); +}); + +test('hardlinks cannot copy outside file contents into a readable dependency snapshot', async t => { + const { a, store, ai, key, directory } = await fixture(t); + await mkdir(join(a, 'node_modules')); + const outside = join(directory, 'private-input'); + await writeFile(outside, 'private'); + await link(outside, join(a, 'node_modules', 'alias')); + await assert.rejects( + publishDependencySnapshot(store, key, a, ai), + /hard-linked/, + ); + assert.deepEqual(await readdir(store.store), []); +}); + +test('kernel lock serializes publishers, supports cancellation and rejects a replaced store', async t => { + const { store, key } = await fixture(t); + let release!: () => void; + const blocked = new Promise(resolve => { + release = resolve; + }); + let entered!: () => void; + const started = new Promise(resolve => { + entered = resolve; + }); + const first = withDependencySnapshot(store, key, async () => { + entered(); + await blocked; + }); + await started; + const controller = new AbortController(); + const second = withDependencySnapshot( + store, + key, + async () => assert.fail('cancelled lock ran'), + controller.signal, + ); + controller.abort(); + await assert.rejects(second); + release(); + await first; + await rename(store.store, `${store.store}-old`); + await mkdir(store.store, { mode: 0o700 }); + await assert.rejects( + withDependencySnapshot(store, key, async () => {}), + /changed after admission/, + ); +}); + +test( + 'real native preparation shares dependencies between checkouts, invalidates changed inputs and denies store access', + { + skip: + process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1' || + (process.platform !== 'darwin' && + process.env.LIBRECHAT_CODE_LIVE_SNAPSHOT_TESTS !== '1'), + }, + async t => { + const { a, b, store, directory, ai, bi } = await fixture(t); + store.lifecycle = { + maxStoreBytes: 10000, + maxEntries: 1, + retentionMs: 5 * 24 * 60 * 60 * 1000, + scanLimit: 100, + }; + for (const root of [a, b]) + await writeFile(join(root, 'package-lock.json'), 'version-one'); + const commands: string[] = []; + const setup: EnvironmentPreparationOptions['setup'] = { + command: + "mkdir -p node_modules; printf 'module.exports = 42' > node_modules/package.js", + timeoutMs: 5000, + reuse: { + inputs: ['package-lock.json'], + checkCommand: 'test -f node_modules/package.js', + checkTimeoutMs: 3000, + snapshot: { + store: store.store, + paths: store.paths, + maxBytes: store.maxBytes, + maxFiles: store.maxFiles, + lifecycle: store.lifecycle, + }, + }, + }; + const sandboxes = [ai, bi].map( + identity => + new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: identity.path, + workspaceIdentity: identity, + homeDirectory: directory, + protectedPaths: [store.store], + allowedDomains: [], + }), + ); + t.after(async () => { + for (const sandbox of sandboxes) await sandbox.close(); + }); + const run = (index: number) => { + const identity = [ai, bi][index]; + return prepareCodeEnvironment({ + root: identity.path, + identity, + setup, + context: 'policy', + snapshotStore: store, + snapshotScope: 'project', + receiptPath: join(store.store, `receipt-${index}.json`), + execute: async (command, timeoutMs) => { + commands.push(command); + return sandboxes[index].execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command, + timeoutMs, + maxOutputBytes: 8192, + }); + }, + }); + }; + assert.equal(await run(0), 'prepared'); + assert.equal(await run(1), 'restored'); + assert.equal( + commands.filter(command => command === setup.command).length, + 1, + ); + await writeFile(join(b, 'package-lock.json'), 'version-two'); + assert.equal(await run(1), 'prepared'); + assert.equal( + commands.filter(command => command === setup.command).length, + 2, + ); + assert.equal( + (await readdir(store.store)).filter(name => + /^[a-f0-9]{64}$/.test(name), + ).length, + 1, + ); + assert.equal( + await readFile(join(a, 'node_modules', 'package.js'), 'utf8'), + 'module.exports = 42', + ); + const denied = await sandboxes[0].execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: `ls '${store.store}'`, + timeoutMs: 3000, + }); + assert.notEqual(denied.exitCode, 0); + }, +); diff --git a/packages/code/src/dependency-snapshots.ts b/packages/code/src/dependency-snapshots.ts new file mode 100644 index 00000000..89a76935 --- /dev/null +++ b/packages/code/src/dependency-snapshots.ts @@ -0,0 +1,538 @@ +import { randomUUID } from 'node:crypto'; +import { constants } from 'node:fs'; +import * as fs from 'node:fs/promises'; +import { + basename, + dirname, + isAbsolute, + join, + relative, + resolve, + sep, +} from 'node:path'; +import { createRequire } from 'node:module'; +import * as rooted from './root-access.js'; +import { withProcessLock } from './process-lock.js'; +import { + captureWorkspaceRootIdentity, + matchesWorkspaceRoot, +} from './root-identity.js'; +import { loadEnvironmentResource } from './environment-resources.js'; +import { isSafePortableRelativePath } from './protocol.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; +import { + parseSnapshotLifecycle, + pruneDependencySnapshots, + SNAPSHOT_MANIFEST, + STAGING_MANIFEST, + SnapshotBudgetFullError, +} from './snapshot-lifecycle.js'; +import type { SnapshotLifecyclePolicy } from './snapshot-lifecycle.js'; + +export interface DependencySnapshotConfig { + store: string; + paths: string[]; + maxBytes: number; + maxFiles: number; + lifecycle?: SnapshotLifecyclePolicy; +} +export interface DependencySnapshotStore extends DependencySnapshotConfig { + identity: WorkspaceRootIdentity; + controlPaths: string[]; +} +export function parseDependencySnapshot( + value: unknown, +): DependencySnapshotConfig { + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Invalid dependency snapshot'); + const v = value as Record; + const maxBytes = v.maxBytes ?? 4 * 1024 ** 3; + const maxFiles = v.maxFiles ?? 200_000; + if ( + Object.keys(v).some( + k => + ![ + 'store', + 'paths', + 'maxBytes', + 'maxFiles', + 'lifecycle', + ].includes(k), + ) || + typeof v.store !== 'string' || + !isAbsolute(v.store) || + v.store.length > 4096 || + /[\0\r\n]/.test(v.store) || + !Array.isArray(v.paths) || + !v.paths.length || + v.paths.length > 32 || + v.paths.some( + p => + typeof p !== 'string' || + !isSafePortableRelativePath(p) || + basename(p) !== 'node_modules' || + p.split('/').some(c => c === '.git' || c === '.worktrees'), + ) || + new Set(v.paths).size !== v.paths.length || + v.paths.some(p => + (v.paths as unknown[]).some( + q => p !== q && (p as string).startsWith(`${q}/`), + ), + ) || + !Number.isSafeInteger(maxBytes) || + (maxBytes as number) < 1 || + !Number.isSafeInteger(maxFiles) || + (maxFiles as number) < 1 + ) + throw new Error('Invalid dependency snapshot'); + return { + store: v.store, + paths: v.paths as string[], + maxBytes: maxBytes as number, + maxFiles: maxFiles as number, + ...(v.lifecycle !== undefined + ? { lifecycle: parseSnapshotLifecycle(v.lifecycle) } + : {}), + }; +} +export async function loadDependencySnapshot( + config: DependencySnapshotConfig, +): Promise { + const resource = await loadEnvironmentResource({ + kind: 'npm-cache', + path: config.store, + access: 'read-only', + }); + return { + ...config, + store: resource.path, + identity: resource.identity, + controlPaths: resource.controlPaths, + }; +} + +let native: + | { clone: (...args: any[]) => number; errno: () => number } + | undefined; +/** No copy fallback and no hardlinks: failure means the host cannot meet the contract. */ +async function cloneFile(source: fs.FileHandle, target: string): Promise { + if (!native) { + const koffi = createRequire(import.meta.url)('koffi'); + const library = koffi.load(null); + native = { + errno: () => koffi.errno(), + clone: + process.platform === 'darwin' + ? library.func( + 'int fclonefileat(int srcfd, int dstfd, const char *name, unsigned int flags)', + ) + : library.func( + 'int ioctl(int fd, unsigned long request, ...)', + ), + }; + } + if (process.platform === 'darwin') { + const parent = await rooted.open( + dirname(target), + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ); + try { + if (native.clone(source.fd, parent.fd, basename(target), 0) !== 0) + throw new Error( + `Copy-on-write dependency cloning unavailable (errno ${native.errno()})`, + ); + } finally { + await parent.close(); + } + } else if (process.platform === 'linux') { + const output = await rooted.open( + target, + constants.O_CREAT | + constants.O_EXCL | + constants.O_WRONLY | + constants.O_NOFOLLOW, + 0o600, + ); + try { + if (native.clone(output.fd, 0x40049409, 'int', source.fd) !== 0) + throw new Error( + `Copy-on-write dependency cloning unavailable (errno ${native.errno()})`, + ); + } finally { + await output.close(); + } + } else + throw new Error( + 'Dependency snapshots require a clone-capable POSIX filesystem', + ); +} + +function inside(root: string, path: string): boolean { + const p = relative(root, path); + return !isAbsolute(p) && p !== '..' && !p.startsWith(`..${sep}`); +} +async function removeTree(path: string): Promise { + const stat = await rooted.lstat(path).catch((e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return undefined; + throw e; + }); + if (!stat) return; + if (!stat.isDirectory()) return rooted.unlink(path); + for (const name of await rooted.readdir(path)) + await removeTree(join(path, name)); + await rooted.rmdir(path); +} +async function cloneTree( + sourceRoot: string, + sourceIdentity: WorkspaceRootIdentity, + targetRoot: string, + targetIdentity: WorkspaceRootIdentity, + paths: string[], + config: DependencySnapshotConfig, + checkout: string, + signal?: AbortSignal, +): Promise<{ bytes: number; files: number }> { + let bytes = 0, + files = 0; + const target = (operation: () => Promise) => + rooted.withWorkspaceRoot(targetRoot, targetIdentity, operation); + await rooted.withWorkspaceRoot(sourceRoot, sourceIdentity, async () => { + const visit = async ( + path: string, + out: string, + checkoutPath: string, + ): Promise => { + signal?.throwIfAborted(); + if (++files > config.maxFiles) + throw new Error('Dependency snapshot exceeds maxFiles'); + const before = await rooted.lstat(path); + if (before.isDirectory()) { + await target(() => rooted.mkdir(out)); + for (const name of await rooted.readdir( + path, + config.maxFiles - files, + )) + await visit( + join(path, name), + join(out, name), + join(checkoutPath, name), + ); + const after = await rooted.lstat(path); + if ( + before.ino !== after.ino || + before.dev !== after.dev || + before.mtimeMs !== after.mtimeMs || + before.ctimeMs !== after.ctimeMs + ) + throw new Error( + 'Dependencies changed during snapshot traversal', + ); + } else if (before.isSymbolicLink()) { + const link = await rooted.readlink(path); + const resolved = resolve(dirname(checkoutPath), link); + if ( + isAbsolute(link) || + !inside(checkout, resolved) || + relative(checkout, resolved).split(sep).includes('.git') || + relative(checkout, resolved) + .split(sep) + .includes('.worktrees') + ) + throw new Error( + 'Dependency snapshot link must remain checkout-local', + ); + await target(() => rooted.symlink(link, out)); + } else if (before.isFile()) { + if (before.nlink !== 1) + throw new Error( + 'Dependency snapshots reject hard-linked files', + ); + bytes += Number(before.size); + if (bytes > config.maxBytes) + throw new Error('Dependency snapshot exceeds maxBytes'); + const handle = await rooted.open( + path, + constants.O_RDONLY | + constants.O_NOFOLLOW | + constants.O_NONBLOCK, + ); + try { + const opened = await handle.stat(); + if ( + !opened.isFile() || + opened.nlink !== 1 || + opened.ino !== before.ino || + opened.dev !== before.dev + ) + throw new Error( + 'Dependency entry changed before cloning', + ); + await target(async () => { + await cloneFile(handle, out); + const output = await rooted.open( + out, + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + await output.chmod(opened.mode & 0o777); + } finally { + await output.close(); + } + }); + const after = await handle.stat(); + if ( + opened.size !== after.size || + after.nlink !== 1 || + opened.mtimeMs !== after.mtimeMs || + opened.ctimeMs !== after.ctimeMs + ) + throw new Error( + 'Dependency file changed during cloning', + ); + } finally { + await handle.close(); + } + } else + throw new Error( + 'Dependency snapshots accept regular files, directories and relative links only', + ); + }; + for (let i = 0; i < paths.length; i++) + await visit( + join(sourceRoot, paths[i]), + join(targetRoot, String(i)), + join(checkout, config.paths[i]), + ); + }); + return { bytes, files }; +} + +export async function withDependencySnapshot( + store: DependencySnapshotStore, + key: string, + operation: () => Promise, + signal?: AbortSignal, + verifyCloneSupport = false, +): Promise { + if (!/^[a-f0-9]{64}$/.test(key)) + throw new Error('Invalid dependency snapshot key'); + if (!(await matchesWorkspaceRoot(store.store, store.identity))) + throw new Error('Dependency snapshot store changed after admission'); + return withProcessLock( + join(store.store, `${key}.lock`), + async () => { + if (!(await matchesWorkspaceRoot(store.store, store.identity))) + throw new Error( + 'Dependency snapshot store changed while waiting', + ); + signal?.throwIfAborted(); + if (store.lifecycle) + await pruneDependencySnapshots(store, { + currentKey: key, + signal, + }).catch(error => { + if (!(error instanceof SnapshotBudgetFullError)) + throw error; + }); + // Fail before installation, rather than discovering unsupported reflinks after npm ci. + if (verifyCloneSupport) { + const probe = await fs.mkdtemp(join(store.store, '.probe-')); + try { + await fs.writeFile(join(probe, 'source'), 'clone-support', { + mode: 0o600, + }); + const input = await fs.open( + join(probe, 'source'), + constants.O_RDONLY | constants.O_NOFOLLOW, + ); + try { + await rooted.withWorkspaceRoot( + store.store, + store.identity, + () => cloneFile(input, join(probe, 'target')), + ); + } finally { + await input.close(); + } + } finally { + await fs.rm(probe, { recursive: true, force: true }); + } + } + signal?.throwIfAborted(); + const result = await operation(); + const manifest = await fs + .open( + join(store.store, key, SNAPSHOT_MANIFEST), + constants.O_RDONLY | constants.O_NOFOLLOW, + ) + .catch((e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return undefined; + throw e; + }); + if (manifest) { + try { + const now = new Date(); + await manifest.utimes(now, now); + } finally { + await manifest.close(); + } + } + return result; + }, + signal, + ); +} +/** Restores only when every declared installation is absent. Existing files are never replaced. */ +export async function restoreDependencySnapshot( + store: DependencySnapshotStore, + key: string, + checkout: string, + identity: WorkspaceRootIdentity, + signal?: AbortSignal, +): Promise { + return rooted.withWorkspaceRoot(checkout, identity, async () => { + for (const path of store.paths) { + const exists = await rooted.lstat(join(checkout, path)).then( + () => true, + (e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return false; + throw e; + }, + ); + if (exists) return false; + } + const source = join(store.store, key); + const metadata = await fs + .lstat(source) + .catch((e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return undefined; + throw e; + }); + if (!metadata) return false; + if (!metadata.isDirectory() || metadata.isSymbolicLink()) + throw new Error('Invalid dependency snapshot entry'); + const staging = join( + checkout, + `.librechat-dependencies-${randomUUID()}`, + ); + await rooted.mkdir(staging); + const installed: string[] = []; + try { + const stagingIdentity = await captureWorkspaceRootIdentity(staging); + await cloneTree( + source, + await captureWorkspaceRootIdentity(source), + staging, + stagingIdentity, + store.paths.map((_, i) => String(i)), + store, + checkout, + signal, + ); + for (let i = 0; i < store.paths.length; i++) { + signal?.throwIfAborted(); + const destination = join(checkout, store.paths[i]); + // rename is descriptor-anchored; reject a newly appeared entry instead of replacing it. + if ( + await rooted.lstat(destination).then( + () => true, + (e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return false; + throw e; + }, + ) + ) + throw new Error( + 'Dependency destination appeared during restore', + ); + await rooted.renameExclusive( + join(staging, String(i)), + destination, + ); + installed.push(destination); + } + return true; + } catch (error) { + // A partial restore is inspection-worthy; do not delete a destination another process may have changed. + if (installed.length) + throw new Error( + 'Dependency restore partially committed; inspect the checkout', + { cause: error }, + ); + throw error; + } finally { + await removeTree(staging); + } + }); +} +export async function publishDependencySnapshot( + store: DependencySnapshotStore, + key: string, + checkout: string, + identity: WorkspaceRootIdentity, + signal?: AbortSignal, +): Promise { + const destination = join(store.store, key); + if ( + await fs.lstat(destination).then( + () => true, + (e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT') return false; + throw e; + }, + ) + ) + return; + const staging = await fs.mkdtemp(join(store.store, `.staging-${key}-`)); + try { + await fs.writeFile( + join(staging, STAGING_MANIFEST), + JSON.stringify({ + version: 1, + key, + bytes: 0, + files: 1, + createdAt: Date.now(), + }), + { mode: 0o600 }, + ); + const measured = await cloneTree( + checkout, + identity, + staging, + await captureWorkspaceRootIdentity(staging), + store.paths, + store, + checkout, + signal, + ); + signal?.throwIfAborted(); + await fs.writeFile( + join(staging, SNAPSHOT_MANIFEST), + JSON.stringify({ + version: 1, + key, + ...measured, + createdAt: Date.now(), + }), + { mode: 0o600 }, + ); + const publish = () => fs.rename(staging, destination); + if (store.lifecycle) + await pruneDependencySnapshots(store, { + currentKey: key, + incomingBytes: measured.bytes, + incomingEntries: 1, + signal, + publish, + }).catch(error => { + if (!(error instanceof SnapshotBudgetFullError)) throw error; + process.stderr.write( + 'librechat-code: dependency snapshot not cached because the store budget is full; prepared checkout remains valid\n', + ); + }); + else await publish(); + } finally { + await fs.rm(staging, { recursive: true, force: true }); + } +} diff --git a/packages/code/src/edits.test.ts b/packages/code/src/edits.test.ts new file mode 100644 index 00000000..5bcabf49 --- /dev/null +++ b/packages/code/src/edits.test.ts @@ -0,0 +1,964 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; + +import { + applyTextEdits, + EDIT_DIAGNOSTIC_MAX_CHARS, + WorkspaceEditMatchError, + WorkspaceEditOutputLimitError, +} from './edits.js'; + +function rejection(run: () => unknown): WorkspaceEditMatchError { + try { + run(); + } catch (error) { + assert.ok(error instanceof WorkspaceEditMatchError); + return error; + } + assert.fail('expected the edits to be rejected'); +} + +test('exact matching keeps its single-location contract', () => { + const applied = applyTextEdits('alpha\nbeta\n', [{ oldText: 'beta', newText: 'gamma' }]); + assert.deepEqual(applied, { + text: 'alpha\ngamma\n', + matches: [{ strategy: 'exact', occurrences: 1 }], + }); +}); + +test('overlapping occurrences make an exact edit ambiguous', () => { + const error = rejection(() => applyTextEdits('aaa', [{ oldText: 'aa', newText: 'b' }])); + assert.match(error.message, /matched 2 locations/); + const applied = applyTextEdits('aaaa', [{ oldText: 'aa', newText: 'b', replaceAll: true }]); + assert.deepEqual(applied, { text: 'bb', matches: [{ strategy: 'exact', occurrences: 2 }] }); +}); + +test('later edits see the result of earlier ones', () => { + const applied = applyTextEdits('one', [ + { oldText: 'one', newText: 'two' }, + { oldText: 'two', newText: 'three' }, + ]); + assert.equal(applied.text, 'three'); +}); + +test('an ambiguous edit names every location instead of a bare conflict', () => { + const text = 'const a = 1;\nreturn a;\nconst b = 2;\nreturn a;\n'; + const error = rejection(() => + applyTextEdits(text, [{ oldText: 'return a;', newText: 'return b;' }]), + ); + assert.match(error.message, /matched 2 locations at lines 2, 4/); + assert.match(error.message, /include more surrounding lines/); + assert.match(error.message, /nothing was written/); +}); + +test('a batch reports every failing edit with its position in one rejection', () => { + const text = 'a\nb\nc\nb\n'; + const error = rejection(() => + applyTextEdits(text, [ + { oldText: 'a', newText: 'A' }, + { oldText: 'missing', newText: 'x' }, + { oldText: 'c', newText: 'C' }, + { oldText: 'b', newText: 'B' }, + ]), + ); + assert.deepEqual( + error.failures.map((failure) => failure.index), + [1, 3], + ); + assert.match(error.message, /^2 of 4 workspace edits did not apply/); + assert.match(error.message, /\nEdit 2: old_text was not found/); + assert.match(error.message, /\nEdit 4: old_text matched 2 locations at lines 2, 4/); +}); + +test('a missing edit points at the line it most likely meant', () => { + const text = 'function load(user) {\n return fetchUser(user.id);\n}\n'; + const error = rejection(() => + applyTextEdits(text, [ + { oldText: 'function load(user) {\n return fetchUser(user);\n}', newText: 'x' }, + ]), + ); + assert.match(error.message, /its first line appears at line 1, but the lines after it differ/); +}); + +test('a missing edit explains elided and line-numbered old_text', () => { + const text = 'start\nmiddle\nend\n'; + const elided = rejection(() => + applyTextEdits(text, [{ oldText: 'start\n// ...\nend', newText: 'x' }]), + ); + assert.match(elided.message, /elision placeholder/); + const numbered = rejection(() => + applyTextEdits(text, [{ oldText: '1 | start\n2 | middle', newText: 'x' }]), + ); + assert.match(numbered.message, /line-number prefixes/); +}); + +test('exact mode says when only the whitespace differs', () => { + const shifted = rejection(() => + applyTextEdits('class A {\n a();\n b();\n}\n', [{ oldText: 'a();\nb();', newText: 'x' }]), + ); + assert.match(shifted.message, /exists at line 2 with different whitespace \(indentation-flexible\)/); + const reflowed = rejection(() => + applyTextEdits('if (ready) {\n run();\n}\n', [ + { oldText: 'if (ready) {\n run();\n}', newText: 'x' }, + ]), + ); + assert.match(reflowed.message, /exists at line 1 with different whitespace \(whitespace-normalized\)/); +}); + +test('exact-mode diagnostics detect normalized whitespace with both boundary wrappers intact', () => { + for (const [source, oldText] of [ + ['foo bar\n', 'foo bar\n'], + ['header\n foo bar\n', '\nfoo bar'], + ['header\r\n foo bar\r\nnext', '\r\nfoo bar\r\n'], + [' foo bar \n', ' foo bar '], + ]) { + const error = rejection(() => applyTextEdits(source, [{ oldText, newText: '' }])); + assert.match(error.message, /exists at line \d+ with different whitespace \(whitespace-normalized\)/, + JSON.stringify({ source, oldText })); + } + const missingBoundary = rejection(() => applyTextEdits('foo bar', [ + { oldText: 'foo bar\n', newText: 'unchanged' }, + ])); + assert.doesNotMatch(missingBoundary.message, /exists at.*different whitespace/); +}); + +test('tolerant matching ignores trailing whitespace and keeps CRLF line endings', () => { + const text = 'first \r\nsecond\t\r\nthird\r\n'; + const applied = applyTextEdits( + text, + [{ oldText: 'first\nsecond', newText: 'one\ntwo' }], + 'tolerant', + ); + assert.equal(applied.text, 'one\r\ntwo\r\nthird\r\n'); + assert.deepEqual(applied.matches, [{ strategy: 'line-trimmed', occurrences: 1 }]); +}); + +test('a trailing newline in old_text matches through the line terminator', () => { + const text = 'keep\ndrop\nkeep too\n'; + const applied = applyTextEdits(text, [{ oldText: 'drop \n', newText: '' }], 'tolerant'); + assert.equal(applied.text, 'keep\nkeep too\n'); +}); + +test('indentation-flexible matches move new_text to the file indentation', () => { + const text = 'class A {\n method() {\n return 1;\n }\n}\n'; + const applied = applyTextEdits( + text, + [ + { + oldText: 'method() {\n return 1;\n}', + newText: 'method() {\n const value = 2;\n return value;\n}', + }, + ], + 'tolerant', + ); + assert.equal( + applied.text, + 'class A {\n method() {\n const value = 2;\n return value;\n }\n}\n', + ); + assert.deepEqual(applied.matches, [{ strategy: 'indentation-flexible', occurrences: 1 }]); +}); + +test('whitespace-normalized matches do not splice prefixes or suffixes of tokens', () => { + for (const [source, oldText] of [ + ['prereturn value;\n', 'return value;'], + ['return valueSuffix\n', 'return value'], + ['return value;postfix\n', 'return value;'], + ]) { + const error = rejection(() => applyTextEdits(source, [{ oldText, newText: 'changed' }], 'tolerant')); + assert.match(error.message, /old_text was not found/); + } + const valid = applyTextEdits('return value;\n', [ + { oldText: 'return value;', newText: 'return changed;' }, + ], 'tolerant'); + assert.equal(valid.text, 'return changed;\n'); + assert.deepEqual(valid.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('large whitespace-normalized edits match without compiling a request-sized regular expression', () => { + const oldText = `head ${'part '.repeat(16_000)}tail`; + const source = `before ${oldText.replace(/ /g, '\t')} after`; + const applied = applyTextEdits(source, [{ oldText, newText: 'result' }], 'tolerant'); + assert.equal(applied.text, 'before result after'); + assert.deepEqual(applied.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized matches count overlapping token sequences but replaceAll does not overlap', () => { + const source = 'a b a\tb a'; + const edit = { oldText: 'a b a', newText: 'x' }; + const error = rejection(() => applyTextEdits(source, [edit], 'tolerant')); + assert.match(error.message, /matched 2 locations/); + const replaced = applyTextEdits(source, [{ ...edit, replaceAll: true }], 'tolerant'); + assert.equal(replaced.text, 'x\tb a'); + assert.deepEqual(replaced.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +for (const fileEnding of ['\n', '\r\n'] as const) { + for (const [oldEnding, newEnding] of [['\r\n', '\n'], ['\n', '\r\n']] as const) { + test(`whitespace-normalized matching peels ${JSON.stringify(oldEnding)} to ${JSON.stringify(newEnding)} in ${JSON.stringify(fileEnding)} files`, () => { + const source = `foo bar${fileEnding}next`; + const result = applyTextEdits(source, [{ + oldText: `foo bar${oldEnding}`, + newText: `baz qux${newEnding}`, + }], 'tolerant'); + assert.equal(result.text, `baz qux${fileEnding}next`); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); + }); + } +} + +test('whitespace-normalized matching peels leading and trailing CRLF/LF wrappers in a batch', () => { + const result = applyTextEdits('header\r\n \tfoo bar\r\nnext\r\n', [ + { oldText: '\n foo bar\r\n', newText: '\r\n baz qux\n' }, + { oldText: 'next', newText: 'done' }, + ], 'tolerant'); + assert.equal(result.text, 'header\r\n \tbaz qux\r\ndone\r\n'); + assert.deepEqual(result.matches, [ + { strategy: 'whitespace-normalized', occurrences: 1 }, + { strategy: 'exact', occurrences: 1 }, + ]); +}); + +test('whitespace-normalized matching peels a trailing newline even if adjacent spaces differ', () => { + const result = applyTextEdits('foo bar \r\nnext', [{ + oldText: 'foo bar \r\n', newText: 'baz qux\n', + }], 'tolerant'); + assert.equal(result.text, 'baz qux \r\nnext'); +}); + +test('whitespace-normalized matching does not prepend new indentation beside preserved source indentation', () => { + const source = 'header\n foo bar\n'; + const result = applyTextEdits(source, [{ + oldText: '\n foo bar', newText: '\n\tbaz qux', + }], 'tolerant'); + assert.equal(result.text, 'header\n baz qux\n'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('newline-only boundary prefixes peel the complete equivalent indentation run', () => { + for (const newline of ['\n', '\r\n']) { + const result = applyTextEdits(`header${newline} foo bar${newline}next`, [{ + oldText: '\nfoo bar\n', newText: '\n\tbaz qux\t\n', + }], 'tolerant'); + assert.equal(result.text, `header${newline} baz qux${newline}next`); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); + } +}); + +test('wrapper peeling preserves source whitespace across complete prefixes and extra-newline combinations', () => { + for (const newline of ['\n', '\r\n']) { + for (const oldIndent of ['', ' ', '\t']) { + for (const newIndent of ['', ' ', '\t']) { + for (let extra = 0; extra < 3; extra++) { + const extraLines = `${newline}${newIndent}`.repeat(extra); + const leading = applyTextEdits(`header${newline} foo bar${newline}`, [{ + oldText: `${newline}${oldIndent}foo bar`, + newText: `${newline}${newIndent}${extraLines}baz qux`, + }], 'tolerant'); + assert.equal(leading.text, `header${newline} ${extraLines}baz qux${newline}`); + const trailing = applyTextEdits(`foo bar${newline} next`, [{ + oldText: `foo bar${newline}${oldIndent}`, + newText: `baz qux${extraLines}${newline}${newIndent}`, + }], 'tolerant'); + assert.equal(trailing.text, `baz qux${extraLines}${newline} next`); + } + } + } + } +}); + +test('horizontal boundary prefixes do not silently duplicate mismatched spaces', () => { + const error = rejection(() => applyTextEdits(' foo bar', [ + { oldText: ' foo bar', newText: ' baz qux' }, + ], 'tolerant')); + assert.match(error.message, /old_text was not found/); +}); + +test('whitespace-normalized matching preserves extra leading blank lines when indentation changes', () => { + const source = 'header\n foo bar\n'; + const result = applyTextEdits(source, [{ + oldText: '\n foo bar', newText: '\n\t\n\tbaz qux', + }], 'tolerant'); + assert.equal(result.text, 'header\n \n\tbaz qux\n'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized replaceAll preserves extra leading blank lines independently', () => { + const source = 'one\n foo bar\ntwo\n foo bar\n'; + const result = applyTextEdits(source, [{ + oldText: '\n foo bar', newText: '\n\t\n\tbaz qux', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'one\n \n\tbaz qux\ntwo\n \n\tbaz qux\n'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 2 }]); +}); + +test('whitespace-normalized matching preserves extra lines after an identical leading wrapper', () => { + const result = applyTextEdits('header\r\n foo bar', [{ + oldText: '\r\n foo bar', newText: '\r\n \r\n\tbaz qux', + }], 'tolerant'); + assert.equal(result.text, 'header\r\n \r\n\tbaz qux'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized matching preserves multiple extra leading blank lines', () => { + const source = 'header\r\n \r\n foo bar\r\n'; + const result = applyTextEdits(source, [{ + oldText: '\n \n foo bar', newText: '\r\n\t\r\n\t\r\n\t\r\n\tbaz qux', + }], 'tolerant'); + assert.equal(result.text, 'header\r\n \r\n \r\n\t\r\n\tbaz qux\r\n'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized matching preserves extra trailing blank lines when boundary indentation changes', () => { + const source = 'foo bar\r\n next'; + const result = applyTextEdits(source, [{ + oldText: 'foo bar\n ', newText: 'baz qux\n\t\n\t', + }], 'tolerant'); + assert.equal(result.text, 'baz qux\r\n\t\r\n next'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized matching leaves intentional indentation on an extra blank line', () => { + const result = applyTextEdits('foo bar\r\nnext', [{ + oldText: 'foo bar\n', newText: 'baz qux\n\t\n', + }], 'tolerant'); + assert.equal(result.text, 'baz qux\r\n\t\r\nnext'); +}); + +test('whitespace-normalized matching does not duplicate differing trailing spaces', () => { + const result = applyTextEdits('foo bar \nnext', [{ + oldText: 'foo bar \n', newText: 'baz qux\t\n', + }], 'tolerant'); + assert.equal(result.text, 'baz qux \nnext'); +}); + +test('whitespace-normalized matching peels a leading newline even if adjacent spaces differ', () => { + const result = applyTextEdits('header\r\n foo bar\r\nnext', [{ + oldText: '\n foo bar', newText: '\nchanged', + }], 'tolerant'); + assert.equal(result.text, 'header\r\n changed\r\nnext'); +}); + +test('whitespace-normalized matching rejects absent required boundary whitespace', () => { + const cases = [ + { source: 'foo bar', oldText: 'foo bar\r\n', newText: 'baz qux\n' }, + { source: 'foo bar next', oldText: 'foo bar\n', newText: 'baz qux\n' }, + { source: 'previous foo bar', oldText: '\nfoo bar', newText: '\nbaz qux' }, + { source: 'foo bar', oldText: ' foo bar', newText: ' baz qux' }, + { source: 'foo bar', oldText: 'foo bar ', newText: 'baz qux ' }, + { source: 'foo bar\nnext', oldText: 'foo bar\n\n', newText: 'baz qux\n\n' }, + { source: 'header\nfoo bar\nnext', oldText: '\n\nfoo bar', newText: '\n\nbaz qux' }, + { source: 'header\n \n foo bar', oldText: '\n \n foo bar', newText: '\n\tbaz qux' }, + ]; + for (const edit of cases) { + const { source, ...request } = edit; + const error = rejection(() => applyTextEdits(source, [request], 'tolerant')); + assert.match(error.message, /old_text was not found/, `unmatched boundary: ${JSON.stringify(edit)}`); + } +}); + +test('whitespace-normalized scanning skips boundary-invalid matches and keeps later valid ones', () => { + const source = 'header foo bar header\nfoo bar\nend'; + const result = applyTextEdits(source, [{ + oldText: '\nfoo bar\n', newText: '\nbaz qux\n', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'header foo bar header\nbaz qux\nend'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('whitespace-normalized matching rejects boundary removal outside its token range', () => { + const cases = [ + { source: 'foo bar\r\nnext', oldText: 'foo bar\r\n', newText: 'baz qux' }, + { source: 'header\r\n foo bar', oldText: '\n foo bar', newText: 'baz qux' }, + { source: ' foo bar', oldText: ' foo bar', newText: 'baz qux' }, + { source: 'foo bar ', oldText: 'foo bar ', newText: 'baz qux' }, + { source: 'header\nfoo bar\nnext', oldText: '\nfoo bar\n', newText: '\n' }, + { source: 'foo bar\n\nnext', oldText: 'foo bar\n\n', newText: 'baz qux\n' }, + ]; + for (const edit of cases) { + const { source, ...request } = edit; + const error = rejection(() => applyTextEdits(source, [request], 'tolerant')); + assert.match(error.message, /old_text was not found/, `unremovable boundary: ${JSON.stringify(edit)}`); + } + const exact = applyTextEdits('foo bar\r\nnext', [{ + oldText: 'foo bar\r\n', newText: 'baz qux', + }], 'tolerant'); + assert.equal(exact.text, 'baz quxnext'); + assert.deepEqual(exact.matches, [{ strategy: 'exact', occurrences: 1 }]); +}); + +test('whitespace-normalized matching preserves two present boundary line breaks', () => { + const result = applyTextEdits('foo bar\n\nnext', [{ + oldText: 'foo bar\r\n\r\n', newText: 'baz qux\n\n', + }], 'tolerant'); + assert.equal(result.text, 'baz qux\n\nnext'); +}); + +test('whitespace-normalized matching preserves an intentional extra line break', () => { + const result = applyTextEdits('foo bar\r\nnext', [{ + oldText: 'foo bar\r\n', newText: 'baz qux\n\n', + }], 'tolerant'); + assert.equal(result.text, 'baz qux\r\n\r\nnext'); +}); + +test('line-trimmed replacements adopt the matched line ending in mixed files', () => { + const source = 'header\r\nfoo \nbar \nnext\n'; + const result = applyTextEdits(source, [{ oldText: 'foo\nbar', newText: 'baz\nqux' }], 'tolerant'); + assert.equal(result.text, 'header\r\nbaz\nqux\nnext\n'); + assert.deepEqual(result.matches, [{ strategy: 'line-trimmed', occurrences: 1 }]); +}); + +test('indentation-flexible replacements keep the selected line ending in mixed files', () => { + const source = 'header\r\n method() {\n return 1;\n }\nend\n'; + const result = applyTextEdits(source, [{ + oldText: 'method() {\n return 1;\n}', + newText: 'method() {\r\n return 2;\r\n}', + }], 'tolerant'); + assert.equal(result.text, 'header\r\n method() {\n return 2;\n }\nend\n'); + assert.deepEqual(result.matches, [{ strategy: 'indentation-flexible', occurrences: 1 }]); +}); + +test('line-trimmed replaceAll keeps each local ending in mixed files', () => { + const source = 'foo \r\nbar \r\nfoo \nbar \nend'; + const result = applyTextEdits(source, [{ + oldText: 'foo\nbar', newText: 'baz\nqux', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'baz\r\nqux\r\nbaz\nqux\nend'); + assert.deepEqual(result.matches, [{ strategy: 'line-trimmed', occurrences: 2 }]); +}); + +test('exact replacements preserve caller line endings rather than normalizing them', () => { + const source = 'foo bar\r\nnext\r\n'; + const result = applyTextEdits(source, [{ oldText: 'foo bar\r\n', newText: 'baz qux\n' }], 'tolerant'); + assert.equal(result.text, 'baz qux\nnext\r\n'); + assert.deepEqual(result.matches, [{ strategy: 'exact', occurrences: 1 }]); +}); + +test('whitespace-normalized replacement adopts the matched line ending in mixed files', () => { + const original = 'header\r\nfoo bar\nnext\n'; + const result = applyTextEdits(original, [{ + oldText: 'foo bar\n', newText: 'baz\nqux\n', + }], 'tolerant'); + assert.equal(result.text, 'header\r\nbaz\nqux\nnext\n'); +}); + +test('whitespace-normalized replaceAll peels boundary newlines on every mixed-ending match', () => { + const source = 'foo bar\r\nfoo bar\nend'; + const result = applyTextEdits(source, [{ + oldText: 'foo bar\r\n', newText: 'baz qux\n', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'baz qux\r\nbaz qux\nend'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 2 }]); +}); + +test('whitespace-normalized replaceAll uses each matched line ending in mixed files', () => { + const source = 'foo bar\r\nfoo bar\nend'; + const result = applyTextEdits(source, [{ + oldText: 'foo bar', newText: 'baz\nqux', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'baz\r\nqux\r\nbaz\nqux\nend'); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 2 }]); +}); + +test('whitespace-normalized replaceAll finds many same-line matches without rescanning newline tails', () => { + const source = 'foo bar '.repeat(16_000); + const start = performance.now(); + const result = applyTextEdits(source, [{ + oldText: 'foo bar', newText: 'baz\nqux', replaceAll: true, + }], 'tolerant'); + assert.equal(result.text, 'baz\nqux '.repeat(16_000)); + assert.deepEqual(result.matches, [{ strategy: 'whitespace-normalized', occurrences: 16_000 }]); + assert.ok(performance.now() - start < 2_000, 'line endings must not be searched from each match'); +}); + +test('whitespace-normalized matches do not indent new_text twice', () => { + const text = ' total = price *\n quantity;\n'; + const applied = applyTextEdits( + text, + [{ oldText: ' total = price * quantity;', newText: ' total = price * quantity * rate;' }], + 'tolerant', + ); + assert.equal(applied.text, ' total = price * quantity * rate;\n'); + assert.deepEqual(applied.matches, [{ strategy: 'whitespace-normalized', occurrences: 1 }]); +}); + +test('tolerant matching still refuses an ambiguous edit', () => { + const error = rejection(() => + applyTextEdits('x = 1 \ny = 2\nx = 1\n', [{ oldText: 'x = 1', newText: 'x = 3' }], 'tolerant'), + ); + assert.match(error.message, /matched 2 locations at lines 1, 3/); +}); + +test('exact overlap counts UTF-16 offsets while replaceAll consumes whole characters', () => { + const error = rejection(() => applyTextEdits('😀😀😀', [ + { oldText: '😀😀', newText: 'x' }, + ])); + assert.match(error.message, /matched 2 locations/); + const result = applyTextEdits('😀😀😀', [{ oldText: '😀😀', newText: 'x', replaceAll: true }]); + assert.equal(result.text, 'x😀'); + assert.deepEqual(result.matches, [{ strategy: 'exact', occurrences: 1 }]); +}); + +test('long overlapping exact matches are counted without repeatedly rescanning the input', () => { + const text = 'a'.repeat(450_000); + const oldText = 'a'.repeat(220_000); + const started = performance.now(); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'b' }])); + assert.match(error.message, /matched 230001 locations/); + assert.ok(performance.now() - started < 3_000, 'overlapping ambiguity must be counted in linear time'); +}); + +test('highly repeated exact matches report the count with bounded line samples', () => { + const error = rejection(() => applyTextEdits('z'.repeat(200_000), [ + { oldText: 'z', newText: 'y' }, + ])); + assert.match(error.message, /matched 200000 locations at lines 1, 1, 1, 1, 1 and 199995 more/); +}); + +test('replaceAll replaces every location and reports the count', () => { + const applied = applyTextEdits('foo(); bar(); foo();', [ + { oldText: 'foo()', newText: 'baz()', replaceAll: true }, + ]); + assert.equal(applied.text, 'baz(); bar(); baz();'); + assert.deepEqual(applied.matches, [{ strategy: 'exact', occurrences: 2 }]); +}); + +test('replaceAll over line windows never overlaps its own matches', () => { + const applied = applyTextEdits( + 'a\na\na\na\n', + [{ oldText: 'a\na', newText: 'b', replaceAll: true }], + 'tolerant', + ); + assert.equal(applied.text, 'b\nb\n'); + assert.deepEqual(applied.matches, [{ strategy: 'exact', occurrences: 2 }]); +}); + +test('replaceAll bounds memory for a million identical and changed matches', () => { + const source = 'x'.repeat(1024 * 1024); + const noop = applyTextEdits(source, [ + { oldText: 'x', newText: 'x', replaceAll: true }, + { oldText: 'x', newText: 'y', replaceAll: true }, + ]); + assert.equal(noop.text, 'y'.repeat(source.length)); + assert.deepEqual(noop.matches, [ + { strategy: 'exact', occurrences: source.length }, + { strategy: 'exact', occurrences: source.length }, + ]); +}); + +test('replaceAll preserves non-overlapping literal edits when identical hits precede changed hits', () => { + for (const [source, oldText, newText] of [ + ['aaaaa', 'aa', 'a'], + ['foo foo foo', 'foo', 'bar'], + ['😀😀😀', '😀😀', 'x'], + ['x\nx\nx', 'x', ''], + ['aba aba', 'aba', 'aba'], + ]) { + const result = applyTextEdits(source, [{ oldText, newText, replaceAll: true }]); + assert.equal(result.text, source.split(oldText).join(newText)); + assert.deepEqual(result.matches, [{ strategy: 'exact', occurrences: source.split(oldText).length - 1 }]); + } + const source = 'foo\tbar foo bar'; + const tolerant = applyTextEdits(source, [{ + oldText: 'foo bar', newText: 'foo\tbar', replaceAll: true, + }], 'tolerant'); + assert.equal(tolerant.text, 'foo\tbar foo\tbar'); + assert.deepEqual(tolerant.matches, [{ strategy: 'whitespace-normalized', occurrences: 2 }]); +}); + +test('replaceAll handles repeated tolerant and non-overlapping line-window matches', () => { + const lines = 'foo \r\nbar \r\n'.repeat(4000); + const result = applyTextEdits(lines, [ + { oldText: 'foo\nbar', newText: 'baz\nqux', replaceAll: true }, + { oldText: 'baz qux', newText: 'updated', replaceAll: true }, + ], 'tolerant'); + assert.equal(result.text, 'updated\r\n'.repeat(4000)); + assert.deepEqual(result.matches, [ + { strategy: 'line-trimmed', occurrences: 4000 }, + { strategy: 'whitespace-normalized', occurrences: 4000 }, + ]); +}); + +test('replaceAll rejects oversized intermediate output before constructing it', () => { + assert.throws( + () => applyTextEdits('x'.repeat(1_000_000), [ + { oldText: 'x', newText: 'y'.repeat(100_000), replaceAll: true }, + ]), + WorkspaceEditOutputLimitError, + ); +}); + +test('replaceAll still fails when nothing matches', () => { + const error = rejection(() => + applyTextEdits('abc', [{ oldText: 'xyz', newText: '', replaceAll: true }]), + ); + assert.match(error.message, /old_text was not found/); +}); + +test('line diagnostic index follows earlier successful batch edits', () => { + const error = rejection(() => applyTextEdits('alpha\nbeta\n', [ + { oldText: 'not present', newText: 'skip' }, + { oldText: 'alpha', newText: 'first\nsecond' }, + { oldText: 'beta\nmissing', newText: 'nope' }, + ])); + assert.match(error.message, /Edit 3: old_text was not found; its first line appears at line 3/); +}); + +test('newline-dense files do not materialize millions of line objects across failed batch edits', () => { + const source = 'a\n'.repeat(450_000); + const edits = Array.from({ length: 80 }, () => ({ oldText: 'missing\ntext', newText: 'other' })); + const started = performance.now(); + const error = rejection(() => applyTextEdits(source, edits)); + assert.equal(error.failures.length, edits.length); + assert.ok(performance.now() - started < 10_000, 'a bounded file must not re-index every failed diagnostic'); +}); + +test('first-line hints report bounded samples even when the line repeats throughout a file', () => { + const error = rejection(() => applyTextEdits('a\n'.repeat(100_000), [ + { oldText: 'a\nmissing', newText: 'replacement' }, + ])); + assert.match(error.message, /its first line appears at lines 1, 2, 3, 4, 5 and 99995 more/); +}); + +test('one uniquely anchored indentation candidate is allowed on both sides of the verification threshold', () => { + for (const length of [100_000, 100_001]) { + const oldText = ['anchor', ...Array.from({ length: length - 2 }, () => 'x'), 'end'].join('\n'); + const source = ` ${oldText.replaceAll('\n', '\n ')}\n`; + const result = applyTextEdits(source, [{ oldText, newText: 'changed' }], 'tolerant'); + assert.equal(result.text, ' changed\n'); + assert.deepEqual(result.matches, [{ strategy: 'indentation-flexible', occurrences: 1 }]); + } +}); + +test('large indentation verification still bounds repeated candidates', () => { + const oldText = `${'x\n'.repeat(100_000)}x`; + const source = ' x\n'.repeat(100_002); + const error = rejection(() => applyTextEdits(source, [{ oldText, newText: 'changed' }], 'tolerant')); + assert.match(error.message, /too many repetitive line-window candidates/); +}); + +test('repetitive indentation candidates fail closed after a bounded comparison budget', () => { + const error = rejection(() => applyTextEdits(' a\n'.repeat(1_200), [ + { oldText: `${'a\n'.repeat(199)}a`, newText: 'changed' }, + ], 'tolerant')); + assert.match(error.message, /too many repetitive line-window candidates/); +}); + +test('repetitive long line windows return a missing-edit diagnosis without quadratic scans', () => { + const text = 'line\n'.repeat(24_000); + const oldText = `${'line\n'.repeat(6_000)}missing`; + const started = performance.now(); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'replacement' }])); + assert.match(error.message, /did not apply and nothing was written/); + assert.ok(performance.now() - started < 2_000, 'a bounded edit must not compare every long window'); +}); + +test('a hundred failing multi-line edits on a large file are diagnosed quickly', () => { + const text = Array.from({ length: 30_000 }, (_, index) => ` line ${index} = value;`).join('\n'); + const edits = Array.from({ length: 100 }, (_, index) => ({ + oldText: `line ${index} = value;\n other ${index};\n more ${index};`, + newText: 'y', + })); + const started = performance.now(); + rejection(() => applyTextEdits(text, edits)); + assert.ok(performance.now() - started < 5_000); +}); + +test('diagnostics for a hundred failing edits stay within the settlement bound', () => { + const edits = Array.from({ length: 100 }, (_, index) => ({ + oldText: `missing ${'x'.repeat(200)} ${index}`, + newText: 'y', + })); + const error = rejection(() => applyTextEdits('content\n'.repeat(1000), edits)); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.match(error.message, /^100 of 100 workspace edits did not apply/); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: 100 }, (_, index) => index + 1), + ); + assert.doesNotMatch(error.message, /failing edits not shown/); +}); + +test('bounded batch diagnostics preserve sparse failing positions rather than only their count', () => { + const source = `KEEP\nconst shared = '${'x'.repeat(180)}';\n`; + const edits = Array.from({ length: 60 }, (_, index) => index % 2 === 0 + ? { oldText: `const shared missing_${index}`, newText: 'replacement' } + : { oldText: 'KEEP', newText: 'KEEP' }); + const error = rejection(() => applyTextEdits(source, edits)); + assert.equal(error.failures.length, 30); + assert.match(error.message, /^30 of 60 workspace edits did not apply/); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: 30 }, (_, index) => 2 * index + 1), + ); + assert.doesNotMatch(error.message, /failing edits not shown/); +}); + +test('bounded diagnostics keep all positions and well-formed Unicode with mixed reason lengths', () => { + for (const count of [2, 30, 100]) { + const failures = Array.from({ length: count }, (_, index) => ({ + index, + reason: index % 3 === 0 ? 'missing' : `source excerpt ${'😀'.repeat(300)}`, + })); + const error = new WorkspaceEditMatchError(failures, count); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.equal(Buffer.from(error.message).toString('utf8'), error.message); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: count }, (_, index) => index + 1), + ); + } +}); + +test('short batch diagnostic reasons remain complete when the message fits', () => { + const error = rejection(() => applyTextEdits('a\nb\nb\n', [ + { oldText: 'missing', newText: 'x' }, + { oldText: 'b', newText: 'B' }, + ])); + for (const failure of error.failures) { + assert.ok(error.message.includes(`\nEdit ${failure.index + 1}: ${failure.reason}.`)); + } +}); + +const EXCERPT = /the current text at lines? (\d+)(?:-(\d+))? \(~ whitespace differs, ! text differs\) is ("(?:[^"\\]|\\.)*")/; + +function excerptOf(message: string): { first: number; last: number; rows: string[] } | undefined { + const match = EXCERPT.exec(message); + if (!match) return undefined; + const first = Number(match[1]); + return { first, last: Number(match[2] ?? first), rows: (JSON.parse(match[3]) as string).split('\n') }; +} + +test('a missing edit quotes the current text of the region it meant, marking what differs', () => { + const text = 'export function load(user) {\n const id = user.id;\n return fetchUser(id);\n}\n'; + const error = rejection(() => applyTextEdits(text, [{ + oldText: 'export function load(user) {\n const id = user.id;\n return fetchUser(user);\n}', + newText: 'x', + }], 'tolerant')); + assert.match(error.message, /old_text was not found; its first line appears at line 1, but the lines after it differ; the current text/); + assert.deepEqual(excerptOf(error.message), { + first: 1, + last: 4, + rows: [ + '1| export function load(user) {', + '2|~ const id = user.id;', + '3|! return fetchUser(id);', + '4| }', + ], + }); + assert.match(error.message, /\.$/); +}); + +test('the region is found from later lines when the first line itself changed', () => { + const text = 'header();\nfunction save(record) {\n validate(record);\n persist(record);\n}\nfooter();\n'; + const error = rejection(() => applyTextEdits(text, [{ + oldText: 'function save(item) {\n validate(record);\n persist(record);\n}', + newText: 'x', + }])); + assert.deepEqual(excerptOf(error.message), { + first: 2, + last: 5, + rows: ['2|!function save(record) {', '3| validate(record);', '4| persist(record);', '5| }'], + }); +}); + +test('the region with the most aligned lines wins, and ties prefer the earliest', () => { + const block = (name: string) => `function ${name}() {\n prepare();\n run();\n cleanup();\n}\n`; + const text = `${block('first')}${block('second')}`; + const error = rejection(() => applyTextEdits(text, [{ + oldText: 'function second() {\n prepare();\n execute();\n cleanup();\n}', + newText: 'x', + }])); + assert.equal(excerptOf(error.message)?.first, 6); + const tied = rejection(() => applyTextEdits(text, [{ + oldText: 'function third() {\n prepare();\n execute();\n cleanup();\n}', + newText: 'x', + }])); + assert.equal(excerptOf(tied.message)?.first, 1); +}); + +test('a missing one-line edit quotes its closest line', () => { + const error = rejection(() => applyTextEdits('const total = items.length;\nreturn total;\n', [ + { oldText: 'const total = items.size;', newText: 'x' }, + ])); + assert.match(error.message, /the closest line is line 1: "const total = items.length;"/); + assert.deepEqual(excerptOf(error.message), { first: 1, last: 1, rows: ['1|!const total = items.length;'] }); +}); + +test('exact mode quotes the whitespace to copy alongside the whitespace hint', () => { + const error = rejection(() => applyTextEdits('class A {\n a();\n b();\n}\n', [ + { oldText: 'a();\nb();', newText: 'x' }, + ])); + assert.match(error.message, /exists at line 2 with different whitespace \(indentation-flexible\); copy that whitespace exactly; the current text at lines 2-3/); + assert.deepEqual(excerptOf(error.message)?.rows, ['2|~ a();', '3|~ b();']); +}); + +test('a long region is quoted from just before its first difference', () => { + const source = Array.from({ length: 40 }, (_, index) => `step(${index});`); + const needle = source.slice(5, 25); + needle[8] = 'step(changed);'; + const error = rejection(() => applyTextEdits(`${source.join('\n')}\n`, [{ oldText: needle.join('\n'), newText: 'x' }])); + const excerpt = excerptOf(error.message); + assert.equal(excerpt?.first, 13); + assert.equal(excerpt?.last, 20); + assert.equal(excerpt?.rows[1], '14|!step(13);'); + assert.ok(excerpt?.rows.every((row, index) => index === 1 || row.includes('| '))); +}); + +test('excerpts shorten long lines and stay bounded without splitting characters', () => { + const wide = `x${'😀'.repeat(400)}`; + const controls = '\u0001'.repeat(2_000); + const text = Array.from({ length: 10 }, (_, index) => `key_${index} = ${index % 2 === 0 ? wide : controls}`).join('\n'); + const oldText = Array.from({ length: 10 }, (_, index) => `key_${index} = other`).join('\n'); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'x' }])); + const excerpt = excerptOf(error.message); + assert.ok(excerpt != null); + assert.ok(excerpt.rows.length >= 1 && excerpt.rows.length <= 8); + assert.ok(EXCERPT.exec(error.message)![0].length <= 1_600); + for (const row of excerpt.rows) { + assert.ok(row.length <= 175, row.slice(0, 40)); + assert.equal(Buffer.from(row).toString('utf8'), row); + } +}); + +test('an edit that resembles nothing in the file gets no excerpt', () => { + const error = rejection(() => applyTextEdits('alpha\nbeta\n', [ + { oldText: 'completely different text\nwith nothing shared', newText: 'x' }, + ])); + assert.equal(excerptOf(error.message), undefined); + assert.match(error.message, /old_text was not found\.$/); +}); + +test('ambiguous and line-numbered edits keep their messages without an excerpt', () => { + const ambiguous = rejection(() => applyTextEdits('return a;\nreturn a;\n', [{ oldText: 'return a;', newText: 'x' }])); + assert.equal(excerptOf(ambiguous.message), undefined); + const numbered = rejection(() => applyTextEdits('start\nmiddle\nend\n', [{ oldText: '1 | start\n2 | middle', newText: 'x' }])); + assert.equal(excerptOf(numbered.message), undefined); +}); + +test('batch excerpts are granted in edit order only while every position and reason still fits', () => { + const lines = Array.from({ length: 400 }, (_, index) => `value_${index} = compute_${index}(input_${index}, ${'p'.repeat(120)});`); + const edits = Array.from({ length: 12 }, (_, index) => ({ + oldText: `${lines[index * 10]}\n${lines[index * 10 + 1].replace('compute', 'changed')}\n${lines[index * 10 + 2]}`, + newText: 'x', + })); + const error = rejection(() => applyTextEdits(`${lines.join('\n')}\n`, edits)); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: 12 }, (_, index) => index + 1), + ); + const quoted = [...error.message.matchAll(/\nEdit (\d+): [^\n]*the current text at/g)].map((match) => Number(match[1])); + assert.ok(quoted.length >= 1 && quoted.length < 12); + assert.deepEqual(quoted, Array.from({ length: quoted.length }, (_, index) => index + 1)); + for (const failure of error.failures) { + assert.ok(error.message.includes(`\nEdit ${failure.index + 1}: ${failure.reason}`)); + } +}); + +test('a single edit drops its excerpt rather than exceed the bound', () => { + const error = new WorkspaceEditMatchError([ + { index: 0, reason: `old_text was not found${'; x'.repeat(1_200)}`, excerpt: 'the current text at line 1 (~ whitespace differs, ! text differs) is "1|!y"' }, + ], 1); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.doesNotMatch(error.message, /the current text/); +}); + +test('region votes stay bounded on highly repetitive files', () => { + const text = 'item = value;\nother = thing;\n'.repeat(150_000); + const oldText = 'item = value;\nother = thing;\nmissing = line;\nitem = value;'; + const started = performance.now(); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'x' }])); + assert.ok(performance.now() - started < 3_000, 'a repetitive file must not cast unbounded votes'); + assert.match(error.message, /did not apply and nothing was written/); +}); + +test('an excerpt anchored on a first line after blank lines starts where old_text starts', () => { + const error = rejection(() => applyTextEdits('header();\nconst total = items.length;\nreturn total;\n', [ + { oldText: '\nconst total = items.size;', newText: 'x' }, + ])); + assert.deepEqual(excerptOf(error.message), { + first: 1, + last: 2, + rows: ['1|!header();', '2|!const total = items.length;'], + }); +}); + +test('region votes count every inspected anchor offset against the budget', () => { + const oldText = `${'repeat_line();\n'.repeat(5_000)}missing();`; + const text = 'repeat_line();\n'.repeat(20_000); + const started = performance.now(); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'x' }])); + assert.ok(performance.now() - started < 3_000, 'repeated anchor offsets must not escape the vote budget'); + assert.match(error.message, /did not apply and nothing was written/); +}); + +test('quoted text escapes line separators so every batch edit stays on one line', () => { + const separator = String.fromCharCode(0x2028); + const text = `const label = "first${separator}second";\nconst other = "value";\n`; + const error = rejection(() => applyTextEdits(text, [ + { oldText: `const label = "first${separator}third";`, newText: 'x' }, + { oldText: 'const other = "changed";', newText: 'y' }, + ])); + assert.ok(!error.message.includes(separator)); + const lines = error.message.split('\n'); + assert.match(lines[1], /^Edit 1: old_text was not found; the closest line is line 1: .*\\u2028.*the current text at line 1/); + assert.match(lines[2], /^Edit 2: /); + assert.deepEqual(excerptOf(lines[1])?.rows, [`1|!const label = "first${separator}second";`]); +}); + +function errorBodyBytes(message: string): number { + return Buffer.byteLength(JSON.stringify({ error: message, code: 'EDIT_CONFLICT' })); +} + +test('excerpts of multibyte source stay within the 4096-byte error body hosts read', () => { + const text = Array.from({ length: 12 }, (_, index) => `名前_${index} = "${'漢字'.repeat(80)}";`).join('\n'); + const oldText = Array.from({ length: 12 }, (_, index) => `名前_${index} = "changed";`).join('\n'); + const error = rejection(() => applyTextEdits(text, [{ oldText, newText: 'x' }])); + const excerpt = excerptOf(error.message); + assert.ok(excerpt != null && excerpt.rows.length >= 1 && excerpt.rows.length < 8); + assert.ok(Buffer.byteLength(JSON.stringify(EXCERPT.exec(error.message)![0])) - 2 <= 1_600); + assert.ok(errorBodyBytes(error.message) <= 4_096); +}); + +test('batch excerpts are granted by encoded size, so escapes cannot overflow the error body', () => { + const lines = Array.from({ length: 200 }, (_, index) => `path_${index} = "C:\\\\dir\\\\${'"q"'.repeat(48)}";`); + const edits = Array.from({ length: 6 }, (_, index) => ({ + oldText: lines.slice(index * 20, index * 20 + 8).map((line, offset) => (offset === 3 ? 'changed();' : line)).join('\n'), + newText: 'x', + })); + const error = rejection(() => applyTextEdits(`${lines.join('\n')}\n`, edits)); + assert.ok(errorBodyBytes(error.message) <= 4_096, String(errorBodyBytes(error.message))); + assert.match(error.message, /\nEdit 1: [^\n]*the current text at/); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + [1, 2, 3, 4, 5, 6], + ); +}); + +test('shortened batch reasons also respect the encoded error-body size', () => { + const emojis = '😀'.repeat(200); + const text = Array.from({ length: 20 }, (_, index) => `item_${index} emoji_${index} = "${emojis}";`).join('\n'); + const edits = Array.from({ length: 20 }, (_, index) => ({ + oldText: `item_${index} emoji_${index} = "plain";`, + newText: 'x', + })); + const error = rejection(() => applyTextEdits(text, edits)); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.ok(errorBodyBytes(error.message) <= 4_096, String(errorBodyBytes(error.message))); + assert.equal(Buffer.from(error.message).toString('utf8'), error.message); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: 20 }, (_, index) => index + 1), + ); +}); + +test('a single edit with a long multibyte reason stays within both bounds', () => { + const error = new WorkspaceEditMatchError([ + { index: 0, reason: `old_text was not found; the closest line is line 1: "${'😀'.repeat(1_400)}"` }, + ], 1); + assert.ok(error.message.length <= EDIT_DIAGNOSTIC_MAX_CHARS); + assert.ok(errorBodyBytes(error.message) <= 4_096); + assert.equal(Buffer.from(error.message).toString('utf8'), error.message); + assert.match(error.message, /…\.$/); +}); diff --git a/packages/code/src/edits.ts b/packages/code/src/edits.ts new file mode 100644 index 00000000..8205b1d7 --- /dev/null +++ b/packages/code/src/edits.ts @@ -0,0 +1,914 @@ +import { BRIDGE_WORKSPACE_WRITE_MAX_BYTES } from './protocol.js'; + +import type { + WorkspaceEditMatch, + WorkspaceEditMatching, + WorkspaceEditMatchStrategy, + WorkspaceTextEdit, +} from './protocol.js'; + +/** + * The Code API rejects a settlement whose error exceeds 4096 characters, and + * callers prefix their own context, so diagnostics stay well below that. + */ +export const EDIT_DIAGNOSTIC_MAX_CHARS = 3000; +/** + * Code API returns the message inside a JSON error body, and hosts read that + * body through a bounded buffer (4096 bytes in LibreChat). Optional excerpts + * are granted only while the JSON-encoded UTF-8 message stays within this size. + */ +export const EDIT_DIAGNOSTIC_MAX_BODY_BYTES = 3_800; +const MAX_REPORTED_LINES = 5; +const MAX_SNIPPET_CHARS = 120; +/** A missing edit's current-text excerpt: a few lines, each shortened, in one bounded hint. */ +const MAX_EXCERPT_LINES = 8; +const MAX_EXCERPT_LINE_CHARS = 160; +/** Measured like the message bound: UTF-8 bytes once JSON-encoded into the error body. */ +const MAX_EXCERPT_BYTES = 1_600; +/** Alignment votes cast while locating the region a missing edit most likely meant. */ +const MAX_REGION_VOTES = 200_000; +/** Lines too generic to anchor a region on their own (`}`, `);`, blank lines). */ +const ANCHOR_LINE = /[\p{L}\p{N}_]{2}/u; +const LINE_SEPARATORS = /[\u2028\u2029]/g; +/** A highly repetitive indentation candidate must not monopolize the worker. */ +const MAX_LINE_WINDOW_VERIFICATIONS = 100_000; +const MAX_REPLACEMENT_CHUNK_CHARS = 16 * 1024; + +interface MatchedRange { + start: number; + end: number; + /** Replacement for this range, already adapted to its indentation and line endings. */ + replacement: string; +} + +type MatchOutcome = + | { status: 'matched'; strategy: WorkspaceEditMatchStrategy; ranges: MatchedRange[]; occurrences: number; updated?: string } + | { status: 'ambiguous'; strategy: WorkspaceEditMatchStrategy; count: number; starts: number[] } + | { status: 'limit' } + | { status: 'none' }; + +interface CollectedMatches { + count: number; + source: string; + projectedLength: number; + /** Only the first range is needed for a unique edit or an exact-mode hint. */ + ranges: MatchedRange[]; + starts: number[]; + output?: { chunks: string[]; pending: string; cursor: number }; +} + +export interface EditFailure { + /** Zero-based position of the edit in the request. */ + index: number; + reason: string; + /** + * A final `; the current text at ...` hint quoting the region the edit most + * likely meant. It is appended to `reason` only while the whole message stays + * within its bound, and is the first detail dropped when it would not. + */ + excerpt?: string; +} + +export class WorkspaceEditMatchError extends Error { + constructor( + readonly failures: EditFailure[], + readonly editCount: number, + ) { + super(formatEditFailures(failures, editCount)); + this.name = 'WorkspaceEditMatchError'; + } +} + +export class WorkspaceEditOutputLimitError extends Error { + constructor() { + super('Workspace file exceeds write limit'); + this.name = 'WorkspaceEditOutputLimitError'; + } +} + +export interface AppliedEdits { + text: string; + matches: WorkspaceEditMatch[]; +} + +/** + * Applies ordered replacements to `text`. Each edit sees the result of the + * edits before it. Every edit is attempted even after a failure, so a caller + * learns about all unmatched or ambiguous edits from one rejection; nothing is + * returned unless every edit applied. + */ +export function applyTextEdits( + text: string, + edits: readonly WorkspaceTextEdit[], + matching: WorkspaceEditMatching = 'exact', +): AppliedEdits { + let working = text; + let lineIndex: LineIndex | undefined; + const lines = (): LineIndex => (lineIndex ??= new LineIndex(working)); + const matches: WorkspaceEditMatch[] = []; + const failures: EditFailure[] = []; + edits.forEach((edit, index) => { + const outcome = findEditMatch(working, edit, matching, lines); + if (outcome.status === 'matched') { + working = outcome.updated ?? replaceRanges(working, outcome.ranges); + lineIndex = undefined; + matches.push({ strategy: outcome.strategy, occurrences: outcome.occurrences }); + return; + } + if (outcome.status === 'none') { + failures.push({ index, ...describeMissing(working, edit.oldText, matching, lines) }); + return; + } + failures.push({ + index, + reason: + outcome.status === 'ambiguous' + ? describeAmbiguous(working, outcome.strategy, outcome.count, outcome.starts) + : 'old_text has too many repetitive line-window candidates; include more surrounding lines or use an exact match', + }); + }); + if (failures.length > 0) { + throw new WorkspaceEditMatchError(failures, edits.length); + } + return { text: working, matches }; +} + +function findEditMatch( + text: string, + edit: WorkspaceTextEdit, + matching: WorkspaceEditMatching, + lines: () => LineIndex, +): MatchOutcome { + const strategies = matching === 'tolerant' ? TOLERANT_STRATEGIES : EXACT_STRATEGIES; + for (const find of strategies) { + const outcome = find(text, edit, lines); + if (outcome.status !== 'none') return outcome; + } + return { status: 'none' }; +} + +function collectedMatches(text: string, replaceAll?: boolean): CollectedMatches { + return { + count: 0, source: text, projectedLength: text.length, starts: [], ranges: [], + ...(replaceAll === true ? { output: { chunks: [], pending: '', cursor: 0 } } : {}), + }; +} + +function appendReplacement(output: NonNullable, part: string): void { + if (!part) return; + output.pending += part; + if (output.pending.length >= MAX_REPLACEMENT_CHUNK_CHARS) { + output.chunks.push(output.pending); + output.pending = ''; + } +} + +function finishReplacement(matches: CollectedMatches): string { + const output = matches.output!; + if (output.cursor === 0) return matches.source; + appendReplacement(output, matches.source.slice(output.cursor)); + return output.chunks.join('') + output.pending; +} + +function collectMatch( + matches: CollectedMatches, + start: number, + end: number, + replacement: string, + replaceAll?: boolean, +): void { + if (replaceAll === true) { + matches.projectedLength += replacement.length - (end - start); + // Even deleting every remaining source character cannot bring this + // intermediate below the 1 MiB limit. Do not retain more matches. + if (matches.projectedLength - (matches.source.length - end) > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) { + throw new WorkspaceEditOutputLimitError(); + } + // Leave unchanged ranges in the source suffix rather than constructing a + // million identical output pieces for a no-op replaceAll edit. + if (replacement !== matches.source.slice(start, end)) { + const output = matches.output!; + appendReplacement(output, matches.source.slice(output.cursor, start)); + appendReplacement(output, replacement); + output.cursor = end; + } + } + matches.count++; + if (matches.starts.length < MAX_REPORTED_LINES) matches.starts.push(start); + if (matches.count === 1) matches.ranges.push({ start, end, replacement }); +} + +function resolve( + strategy: WorkspaceEditMatchStrategy, + matches: CollectedMatches, + replaceAll: boolean | undefined, +): MatchOutcome { + if (matches.count === 0) return { status: 'none' }; + if (matches.count === 1 || replaceAll === true) { + return { + status: 'matched', strategy, ranges: matches.ranges, occurrences: matches.count, + ...(replaceAll === true ? { updated: finishReplacement(matches) } : {}), + }; + } + return { status: 'ambiguous', strategy, count: matches.count, starts: matches.starts }; +} + +/** + * Overlapping occurrences count toward ambiguity (`aa` in `aaa` is two + * locations), while `replaceAll` replaces non-overlapping occurrences. + */ +function findExact(text: string, edit: WorkspaceTextEdit): MatchOutcome { + if (edit.oldText.length === 0) return { status: 'none' }; + const matches = collectedMatches(text, edit.replaceAll); + const prefix = prefixTable(edit.oldText); + let matched = 0; + for (let index = 0; index < text.length; index++) { + while (matched > 0 && text[index] !== edit.oldText[matched]) matched = prefix[matched - 1]; + if (text[index] === edit.oldText[matched]) matched++; + if (matched !== edit.oldText.length) continue; + collectMatch(matches, index - matched + 1, index + 1, edit.newText, edit.replaceAll); + // Ambiguity counts overlaps, but replaceAll must consume disjoint ranges. + matched = edit.replaceAll === true ? 0 : prefix[matched - 1]; + } + return resolve('exact', matches, edit.replaceAll); +} + +interface Line { + /** Offset of the first character of the line. */ + start: number; + /** Offset just past the line's content, excluding `\r\n` or `\n`. */ + end: number; + /** Offset just past the line terminator, or `undefined` for an unterminated last line. */ + next: number | undefined; + text: string; +} + +/** One bounded index per source revision, shared by matching and diagnostics. */ +class LineIndex { + private readonly newlines: Uint32Array; + readonly length: number; + + constructor(private readonly source: string) { + this.newlines = new Uint32Array(newlineCount(source)); + let index = 0; + for (let offset = source.indexOf('\n'); offset >= 0; offset = source.indexOf('\n', offset + 1)) { + this.newlines[index++] = offset; + } + // A trailing terminator leaves an empty final line, just as before. + this.length = this.newlines.length + 1; + } + + start(index: number): number { + return index === 0 ? 0 : this.newlines[index - 1] + 1; + } + + next(index: number): number | undefined { + return index < this.newlines.length ? this.newlines[index] + 1 : undefined; + } + + end(index: number): number { + const start = this.start(index); + const end = index < this.newlines.length ? this.newlines[index] : this.source.length; + return end > start && this.source[end - 1] === '\r' ? end - 1 : end; + } + + text(index: number): string { + return this.source.slice(this.start(index), this.end(index)); + } + + at(index: number): Line { + return { start: this.start(index), end: this.end(index), next: this.next(index), text: this.text(index) }; + } +} + +/** + * Lines a line-window strategy must match. A trailing line break means "through + * the end of the last line", not "followed by an empty line". + */ +function neededLines(oldText: string): { lines: string[]; throughTerminator: boolean } { + const normalized = oldText.replace(/\r\n/g, '\n'); + const throughTerminator = normalized.endsWith('\n'); + return { + lines: (throughTerminator ? normalized.slice(0, -1) : normalized).split('\n'), + throughTerminator, + }; +} + +/** Fallback when neither the matched line nor its neighbors have a terminator. */ +function fileLineEnding(text: string): '\r\n' | '\n' { + return text.includes('\r\n') ? '\r\n' : '\n'; +} + +function lineEndingAt(text: string, newline: number): '\r\n' | '\n' { + return newline > 0 && text[newline - 1] === '\r' ? '\r\n' : '\n'; +} + +function leadingWhitespace(line: string): string { + return /^[ \t]*/.exec(line)?.[0] ?? ''; +} + +function commonIndent(lines: Iterable): string { + let common: string | undefined; + for (const line of lines) { + if (line.trim().length === 0) continue; + const indent = leadingWhitespace(line); + if (common === undefined) { + common = indent; + continue; + } + let shared = 0; + while (shared < common.length && shared < indent.length && common[shared] === indent[shared]) { + shared++; + } + common = common.slice(0, shared); + } + return common ?? ''; +} + +function withLineEnding(value: string, ending: '\r\n' | '\n'): string { + const normalized = value.replace(/\r\n/g, '\n'); + return ending === '\r\n' ? normalized.replace(/\n/g, '\r\n') : normalized; +} + +function prefixTable(values: ArrayLike): Uint32Array { + const prefix = new Uint32Array(values.length); + for (let index = 1, matched = 0; index < values.length; index++) { + while (matched > 0 && values[index] !== values[matched]) matched = prefix[matched - 1]; + if (values[index] === values[matched]) matched++; + prefix[index] = matched; + } + return prefix; +} + +/** + * Line-window strategies compare whole lines, so a match always spans from the + * start of its first line to the end of its last line's content. The file's + * own line terminators are kept, and the replacement adopts them. + */ +function findLineWindows( + text: string, + edit: WorkspaceTextEdit, + strategy: 'line-trimmed' | 'indentation-flexible', + getLines: () => LineIndex, +): MatchOutcome { + const { lines: needle, throughTerminator } = neededLines(edit.oldText); + if (needle.every((line) => line.trim().length === 0)) return { status: 'none' }; + const lines = getLines(); + const ending = fileLineEnding(text); + const needleIndent = strategy === 'indentation-flexible' ? commonIndent(needle) : ''; + const normalizedNeedle = needle.map((line) => + (strategy === 'indentation-flexible' ? stripIndent(line, needleIndent) : line).trimEnd(), + ); + if (needle.length > lines.length) return { status: 'none' }; + // For line-trimmed matching, the normalized lines are the complete comparison. + // For indentation-flexible matching, whole-line content is a linear-time + // prefilter; only complete candidates need their relative indent verified. + const sought = strategy === 'line-trimmed' + ? normalizedNeedle + : normalizedNeedle.map((line) => line.trimStart()); + const prefix = prefixTable(sought); + const collected = collectedMatches(text, edit.replaceAll); + let matched = 0; + let verifications = 0; + // Allow one complete linear verification even for a large valid block. + const verificationBudget = Math.max(MAX_LINE_WINDOW_VERIFICATIONS, needle.length); + for (let index = 0; index < lines.length; index++) { + const value = strategy === 'line-trimmed' ? lines.text(index).trimEnd() : lines.text(index).trim(); + while (matched > 0 && value !== sought[matched]) matched = prefix[matched - 1]; + if (value === sought[matched]) matched++; + if (matched !== sought.length) continue; + + const first = index - sought.length + 1; + const end = throughTerminator ? lines.next(index) : lines.end(index); + let windowIndent = ''; + let valid = end !== undefined; + if (valid && strategy === 'indentation-flexible') { + if (verifications + needle.length > verificationBudget) return { status: 'limit' }; + verifications += needle.length; + windowIndent = commonIndent((function* () { + for (let offset = 0; offset < needle.length; offset++) yield lines.text(first + offset); + })()); + valid = normalizedNeedle.every((expected, offset) => + stripIndent(lines.text(first + offset), windowIndent).trimEnd() === expected, + ); + } + if (valid) { + const replacement = strategy === 'indentation-flexible' + ? reindent(edit.newText, needleIndent, windowIndent) + : edit.newText; + const lineFeed = lines.next(first) ?? (first > 0 ? lines.next(first - 1) : undefined); + const localEnding = lineFeed === undefined ? ending : lineEndingAt(text, lineFeed - 1); + collectMatch(collected, lines.start(first), end!, withLineEnding(replacement, localEnding), edit.replaceAll); + } + // A replacement cannot consume overlapping lines; ambiguity still counts them. + matched = valid && edit.replaceAll === true ? 0 : prefix[matched - 1]; + } + return resolve(strategy, collected, edit.replaceAll); +} + +function stripIndent(line: string, indent: string): string { + return line.startsWith(indent) ? line.slice(indent.length) : line.trimStart(); +} + +/** + * Moves `value` from the indentation the caller wrote to the file's own. Lines + * the caller indented less than its `old_text` keep their indentation as written. + */ +function reindent(value: string, from: string, to: string): string { + if (from === to) return value; + return value + .replace(/\r\n/g, '\n') + .split('\n') + .map((line) => + line.trim().length > 0 && line.startsWith(from) ? to + line.slice(from.length) : line, + ) + .join('\n'); +} + +function findLineTrimmed(text: string, edit: WorkspaceTextEdit, lines: () => LineIndex): MatchOutcome { + return findLineWindows(text, edit, 'line-trimmed', lines); +} + +function findIndentationFlexible(text: string, edit: WorkspaceTextEdit, lines: () => LineIndex): MatchOutcome { + return findLineWindows(text, edit, 'indentation-flexible', lines); +} + +function newlineCount(text: string): number { + let count = 0; + for (let index = text.indexOf('\n'); index >= 0; index = text.indexOf('\n', index + 1)) count++; + return count; +} + +/** A boundary supplied by oldText must exist beside the candidate tokens. */ +function hasBoundaryWhitespace( + text: string, + position: number, + direction: -1 | 1, + requiredNewlines: number, +): boolean { + for (let index = position; index >= 0 && index < text.length; index += direction) { + const char = text[index]; + if (!/\s/.test(char)) break; + if (char === '\n') requiredNewlines--; + if (requiredNewlines <= 0) return true; + } + return false; +} + +/** A token-only match leaves the source's outer whitespace untouched. */ +function peelBoundaryWhitespace( + text: string, + boundary: string, + side: 'leading' | 'trailing', +): string | undefined { + if (!boundary) return text; + const whitespace = (side === 'leading' ? /^\s*/ : /\s*$/).exec(text)?.[0] ?? ''; + if (whitespace === boundary) { + return side === 'leading' ? text.slice(boundary.length) : text.slice(0, -boundary.length); + } + + const requiredNewlines = newlineCount(boundary); + if (!requiredNewlines) return undefined; + if (newlineCount(whitespace) < requiredNewlines) return undefined; + + if (side === 'leading') { + let end = 0; + for (let count = 0; count < requiredNewlines; count++) { + end = whitespace.indexOf('\n', end) + 1; + } + // Consume the equivalent indentation, stopping before the first extra + // newline. Extra blank lines belong to the replacement, not the wrapper. + while (end < whitespace.length && whitespace[end] !== '\n') end++; + return text.slice(end); + } + + if (newlineCount(whitespace) === requiredNewlines) { + return text.slice(0, text.length - whitespace.length); + } + let start = whitespace.length; + for (let count = 0; count < requiredNewlines; count++) { + start = whitespace.lastIndexOf('\n', start - 1); + } + // Indentation before this newline belongs to the extra blank line. + return text.slice(0, text.length - (whitespace.length - start)); +} + +/** + * Tolerates any run of whitespace, including line breaks, between tokens. The + * match starts and ends on a token, so whitespace the caller wrapped around + * `old_text` is also peeled off `new_text` rather than inserted twice. + */ +function findWhitespaceNormalized(text: string, edit: WorkspaceTextEdit): MatchOutcome { + const oldText = edit.oldText.replace(/\r\n/g, '\n'); + const tokens = oldText.trim().split(/\s+/).filter(Boolean); + if (tokens.length < 2) return { status: 'none' }; + // The matched range contains tokens only; leave the file's boundary whitespace + // outside it. Normalize the caller's line endings *before* peeling equivalent + // wrappers, otherwise CRLF/LF differences insert a second line break. + const leading = /^\s*/.exec(oldText)?.[0] ?? ''; + const trailing = /\s*$/.exec(oldText)?.[0] ?? ''; + const leadingNewlines = newlineCount(leading); + const trailingNewlines = newlineCount(trailing); + const normalizedNewText = edit.newText.replace(/\r\n/g, '\n'); + const withoutLeading = peelBoundaryWhitespace(normalizedNewText, leading, 'leading'); + if (withoutLeading === undefined) return { status: 'none' }; + const newText = peelBoundaryWhitespace(withoutLeading, trailing, 'trailing'); + if (newText === undefined) return { status: 'none' }; + const lfReplacement = newText; + const crlfReplacement = withLineEnding(newText, '\r\n'); + // Match entire whitespace-delimited tokens, never an identifier prefix or + // suffix. A fixed-size regex tokenizes the file; KMP keeps repetitive input + // linear without compiling user-provided text as a regular expression. + const prefix = prefixTable(tokens); + const tokenStarts = new Uint32Array(tokens.length); + const collected = collectedMatches(text, edit.replaceAll); + const words = /\S+/g; + let matched = 0; + let tokenIndex = 0; + // Advance the newline cursor only forwards. replaceAll may encounter many + // matches on one long line, so searching from each match would be quadratic. + let nextNewline = text.indexOf('\n'); + let previousNewline = -1; + for (let word = words.exec(text); word != null; word = words.exec(text)) { + tokenStarts[tokenIndex % tokens.length] = word.index; + while (matched > 0 && word[0] !== tokens[matched]) matched = prefix[matched - 1]; + if (word[0] === tokens[matched]) matched++; + if (matched === tokens.length) { + const start = tokenStarts[(tokenIndex + 1) % tokens.length]; + const end = word.index + word[0].length; + const boundariesMatch = + (leading.length === 0 || hasBoundaryWhitespace(text, start - 1, -1, leadingNewlines)) && + (trailing.length === 0 || hasBoundaryWhitespace(text, end, 1, trailingNewlines)); + if (boundariesMatch) { + while (nextNewline >= 0 && nextNewline < start) { + previousNewline = nextNewline; + nextNewline = text.indexOf('\n', nextNewline + 1); + } + const nearestNewline = nextNewline >= 0 ? nextNewline : previousNewline; + const replacement = nearestNewline >= 0 && lineEndingAt(text, nearestNewline) === '\r\n' + ? crlfReplacement + : lfReplacement; + collectMatch(collected, start, end, replacement, edit.replaceAll); + } + matched = boundariesMatch && edit.replaceAll === true ? 0 : prefix[matched - 1]; + } + tokenIndex++; + } + return resolve('whitespace-normalized', collected, edit.replaceAll); +} + +type Strategy = (text: string, edit: WorkspaceTextEdit, lines: () => LineIndex) => MatchOutcome; + +/** + * Loosest last. Line-window strategies run before whitespace normalization + * because they can carry `new_text` over to the file's indentation; a + * whitespace-normalized match would insert it exactly as written. + */ +const RELAXED_STRATEGIES: readonly Strategy[] = [ + findLineTrimmed, + findIndentationFlexible, + findWhitespaceNormalized, +]; +const EXACT_STRATEGIES: readonly Strategy[] = [findExact]; +const TOLERANT_STRATEGIES: readonly Strategy[] = [findExact, ...RELAXED_STRATEGIES]; + +function replaceRanges(text: string, ranges: readonly MatchedRange[]): string { + const length = ranges.reduce( + (total, range) => total + range.replacement.length - (range.end - range.start), + text.length, + ); + // Every valid UTF-8 string has at least this many encoded bytes. Reject a + // pathological replaceAll before allocating an unbounded intermediate string. + if (length > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) throw new WorkspaceEditOutputLimitError(); + let result = ''; + let cursor = 0; + for (const range of ranges) { + result += text.slice(cursor, range.start) + range.replacement; + cursor = range.end; + } + return result + text.slice(cursor); +} + +function lineNumberAt(text: string, offset: number): number { + let line = 1; + for (let index = text.indexOf('\n'); index >= 0 && index < offset; index = text.indexOf('\n', index + 1)) { + line++; + } + return line; +} + +function formatLineList(text: string, starts: readonly number[], count = starts.length): string { + const shown = starts.slice(0, MAX_REPORTED_LINES).map((start) => lineNumberAt(text, start)); + const more = count - shown.length; + return `line${shown.length === 1 ? '' : 's'} ${shown.join(', ')}${more > 0 ? ` and ${more} more` : ''}`; +} + +function describeAmbiguous( + text: string, + strategy: WorkspaceEditMatchStrategy, + count: number, + starts: readonly number[], +): string { + const how = strategy === 'exact' ? '' : ` (${strategy})`; + return `old_text matched ${count} locations${how} at ${formatLineList(text, starts, count)}; include more surrounding lines so it matches exactly one`; +} + +/** + * JSON quoting that also escapes U+2028/U+2029, which JSON.stringify leaves + * raw: hosts read a batch diagnostic one line per edit, and a regular + * expression `.` stops at those separators. + */ +function quote(value: string): string { + return JSON.stringify(value).replace( + LINE_SEPARATORS, + (separator) => `\\u${separator.charCodeAt(0).toString(16)}`, + ); +} + +function snippet(line: string): string { + const trimmed = line.trim(); + return quote(trimmed.length > MAX_SNIPPET_CHARS ? `${trimmed.slice(0, MAX_SNIPPET_CHARS)}…` : trimmed); +} + +const ELISION_LINE = /^\s*(?:(?:\/\/|#|--|\/\*|\*|))?\s*$/; +const LINE_NUMBER_PREFIX = /^\s*\d+\s*(?:\||:|\t)/; + +function describeMissing( + text: string, + oldText: string, + matching: WorkspaceEditMatching, + lines: () => LineIndex, +): { reason: string; excerpt?: string } { + const needle = neededLines(oldText).lines; + const firstOffset = needle.findIndex((line) => line.trim().length > 0); + const nearest = nearestLine(needle[firstOffset], lines); + const reason = describeMissingReason(text, oldText, needle, matching, lines, nearest); + // The anchor is the first non-blank line, so the region starts that many lines earlier. + const anchor = nearest != null && (!nearest.exact || nearest.count === 1) + ? Math.max(0, nearest.index - firstOffset) + : undefined; + const start = closestRegion(needle, lines()) ?? anchor; + const excerpt = start == null ? undefined : currentTextExcerpt(needle, lines(), start); + return excerpt == null ? { reason } : { reason, excerpt }; +} + +type ExcerptMark = ' ' | '~' | '!'; + +/** + * Finds where a missing multi-line edit most likely belongs: the alignment on + * which the most of its distinctive lines, compared without surrounding + * whitespace, fall into place. One pass over the file with bounded votes. + */ +function closestRegion(needle: readonly string[], lines: LineIndex): number | undefined { + const offsets = new Map(); + let anchors = 0; + needle.forEach((line, offset) => { + const key = line.trim(); + if (!ANCHOR_LINE.test(key)) return; + anchors++; + const known = offsets.get(key); + if (known) known.push(offset); + else offsets.set(key, [offset]); + }); + if (anchors < 2) return undefined; + const votes = new Map(); + let budget = MAX_REGION_VOTES; + let best: number | undefined; + let bestVotes = 0; + for (let index = 0; index < lines.length && budget > 0; index++) { + const hits = offsets.get(lines.text(index).trim()); + if (!hits) continue; + // Offsets ascend, so every later one would start before the file too. + for (const offset of hits) { + if (budget-- <= 0) break; + const start = index - offset; + if (start < 0) break; + const count = (votes.get(start) ?? 0) + 1; + votes.set(start, count); + if (count > bestVotes || (count === bestVotes && best !== undefined && start < best)) { + best = start; + bestVotes = count; + } + } + } + // One shared line is too weak to claim a region; the first-line anchor covers that case. + return bestVotes >= 2 ? best : undefined; +} + +function excerptMark(current: string, expected: string | undefined): ExcerptMark { + if (expected === undefined) return '!'; + if (current === expected) return ' '; + return current.trim() === expected.trim() ? '~' : '!'; +} + +function shortenLine(line: string): string { + if (line.length <= MAX_EXCERPT_LINE_CHARS) return line; + let end = MAX_EXCERPT_LINE_CHARS; + if (/[\uD800-\uDBFF]/.test(line[end - 1]) && /[\uDC00-\uDFFF]/.test(line[end])) end--; + return `${line.slice(0, end)}…`; +} + +/** + * Quotes the current text where a missing edit most likely belongs, so the next + * attempt can copy it instead of re-reading the file. Each line is marked + * against the `old_text` line at the same position: a space when identical, + * `~` when only whitespace differs and `!` when the text differs. A region + * longer than the excerpt is shown from just before its first difference. + */ +function currentTextExcerpt(needle: readonly string[], lines: LineIndex, start: number): string | undefined { + const span = Math.max(1, Math.min(needle.length, lines.length - start)); + const marks = Array.from({ length: span }, (_, offset) => + excerptMark(lines.text(start + offset), needle[offset]), + ); + const shown = Math.min(span, MAX_EXCERPT_LINES); + const firstDifference = Math.max(0, marks.findIndex((mark) => mark !== ' ')); + const first = start + Math.min(Math.max(0, firstDifference - 1), span - shown); + const rows = marks + .slice(first - start, first - start + shown) + .map((mark, offset) => `${first + offset + 1}|${mark}${shortenLine(lines.text(first + offset))}`); + for (; rows.length > 0; rows.pop()) { + const where = rows.length === 1 ? `line ${first + 1}` : `lines ${first + 1}-${first + rows.length}`; + const excerpt = `the current text at ${where} (~ whitespace differs, ! text differs) is ${quote(rows.join('\n'))}`; + if (encodedBytes(excerpt) <= MAX_EXCERPT_BYTES) return excerpt; + } + return undefined; +} + +interface NearestLine { + exact: boolean; + count: number; + /** Line index of the first exact candidate, or of the best token match. */ + index: number; + starts: number[]; + text: string; +} + +function describeMissingReason( + text: string, + oldText: string, + needle: readonly string[], + matching: WorkspaceEditMatching, + lines: () => LineIndex, + nearest: NearestLine | undefined, +): string { + const hints: string[] = []; + const nonBlank = needle.filter((line) => line.trim().length > 0); + if (nonBlank.some((line) => ELISION_LINE.test(line))) { + hints.push('it contains an elision placeholder ("..."); copy the exact lines instead of abbreviating'); + } + if (nonBlank.length > 0 && nonBlank.every((line) => LINE_NUMBER_PREFIX.test(line))) { + hints.push('it appears to include line-number prefixes from read_file output; remove them'); + } + if (matching === 'exact') { + let tolerant: MatchOutcome | undefined; + for (const find of RELAXED_STRATEGIES) { + // Diagnose match existence, not whether the requested replacement can + // preserve its out-of-range wrappers. A no-op keeps those wrappers valid. + tolerant = find(text, { oldText, newText: oldText }, lines); + if (tolerant.status !== 'none') break; + } + if (tolerant?.status === 'matched') { + hints.push( + `the same text exists at ${formatLineList(text, [tolerant.ranges[0].start])} with different whitespace (${tolerant.strategy}); copy that whitespace exactly`, + ); + } else if (text.includes('\r\n') && !oldText.includes('\r\n') && oldText.includes('\n')) { + hints.push('the file uses CRLF line endings'); + } + } + if (nearest != null && hints.length === 0) { + hints.push( + nearest.exact + ? `its first line appears at ${formatLineList(text, nearest.starts, nearest.count)}, but the lines after it differ` + : `the closest line is ${formatLineList(text, nearest.starts)}: ${snippet(nearest.text)}`, + ); + } + return `old_text was not found${hints.length > 0 ? `; ${hints.join('; ')}` : ''}`; +} + +/** Finds where the first line of a failed edit most likely belongs. */ +function nearestLine( + firstLine: string | undefined, + getLines: () => LineIndex, +): NearestLine | undefined { + const target = firstLine?.trim(); + if (!target) return undefined; + const lines = getLines(); + const starts: number[] = []; + let count = 0; + let first = 0; + for (let index = 0; index < lines.length; index++) { + if (lines.text(index).trim() !== target) continue; + if (count++ === 0) first = index; + if (starts.length < MAX_REPORTED_LINES) starts.push(lines.start(index)); + } + if (count > 0) { + return { exact: true, count, index: first, starts, text: lines.text(first) }; + } + const tokens = new Set(target.split(/\W+/).filter((token) => token.length > 1)); + if (tokens.size < 2) return undefined; + let best: number | undefined; + let bestScore = 0; + for (let index = 0; index < lines.length; index++) { + let score = 0; + for (const token of new Set(lines.text(index).split(/\W+/))) { + if (tokens.has(token)) score++; + } + if (score > bestScore) { + best = index; + bestScore = score; + } + } + return best != null && bestScore / tokens.size >= 0.5 + ? { exact: false, count: 1, index: best, starts: [lines.start(best)], text: lines.text(best) } + : undefined; +} + +/** UTF-8 bytes `value` occupies once JSON-encoded as a string in an error body. */ +function encodedBytes(value: string): number { + return Buffer.byteLength(JSON.stringify(value)) - 2; +} + +/** A failure's reason, followed by its excerpt when the excerpt was granted room. */ +function failureText(failure: EditFailure, withExcerpt: boolean): string { + return withExcerpt && failure.excerpt ? `${failure.reason}; ${failure.excerpt}` : failure.reason; +} + +/** + * Grants excerpts in edit order while the message still fits. Excerpts are the + * most expendable detail, so granting one never shortens another reason. + */ +function grantExcerpts( + failures: readonly EditFailure[], + availableChars: number, + availableBytes: number, +): boolean[] { + let chars = availableChars - failures.reduce((total, failure) => total + failure.reason.length, 0); + let bytes = availableBytes - failures.reduce((total, failure) => total + encodedBytes(failure.reason), 0); + return failures.map((failure) => { + if (!failure.excerpt) return false; + const addition = `; ${failure.excerpt}`; + const byteCost = encodedBytes(addition); + if (addition.length > chars || byteCost > bytes) return false; + chars -= addition.length; + bytes -= byteCost; + return true; + }); +} + +/** + * The longest prefix of `reason` within both limits, ending in an ellipsis. It + * never splits a surrogate pair in a shortened source excerpt. + */ +function shortenReason(reason: string, charLimit: number, byteLimit: number): string { + if (reason.length <= charLimit && encodedBytes(reason) <= byteLimit) return reason; + const budget = byteLimit - encodedBytes('…'); + let end = 0; + let bytes = 0; + while (end < reason.length) { + const width = /[\uD800-\uDBFF]/.test(reason[end]) && /[\uDC00-\uDFFF]/.test(reason[end + 1] ?? '') ? 2 : 1; + const cost = encodedBytes(reason.slice(end, end + width)); + if (end + width > charLimit - 1 || bytes + cost > budget) break; + bytes += cost; + end += width; + } + return `${reason.slice(0, end)}…`; +} + +function fitsDiagnosticBounds(message: string): boolean { + return message.length <= EDIT_DIAGNOSTIC_MAX_CHARS && encodedBytes(message) <= EDIT_DIAGNOSTIC_MAX_BODY_BYTES; +} + +function formatEditFailures(failures: readonly EditFailure[], editCount: number): string { + if (editCount === 1) { + const prefix = 'Workspace edit did not apply and nothing was written: '; + const detailed = `${prefix}${failures[0] ? failureText(failures[0], true) : 'no match'}.`; + if (fitsDiagnosticBounds(detailed)) return detailed; + const reason = shortenReason( + failures[0]?.reason ?? 'no match', + EDIT_DIAGNOSTIC_MAX_CHARS - prefix.length - 1, + EDIT_DIAGNOSTIC_MAX_BODY_BYTES - encodedBytes(prefix) - 1, + ); + return `${prefix}${reason}.`; + } + const header = `${failures.length} of ${editCount} workspace edits did not apply, so nothing was written. Every other edit matched.`; + const footer = failures.some((failure) => failure.index > 0) + ? '\nLine numbers account for the earlier edits in this batch.' + : ''; + const prefixes = failures.map((failure) => `\nEdit ${failure.index + 1}: `); + // Labels and punctuation for every failing position take priority over long + // reasons or source excerpts. Never leave callers guessing which edits failed. + const available = EDIT_DIAGNOSTIC_MAX_CHARS - header.length - footer.length - + prefixes.reduce((total, prefix) => total + prefix.length + 1, 0); + const frame = header + footer + prefixes.join('') + '.'.repeat(failures.length); + const availableBytes = EDIT_DIAGNOSTIC_MAX_BODY_BYTES - encodedBytes(frame); + const granted = grantExcerpts(failures, available, availableBytes); + const reasons = failures.map((failure, index) => failureText(failure, granted[index])); + const reasonLength = reasons.reduce((total, reason) => total + reason.length, 0); + const reasonBytes = reasons.reduce((total, reason) => total + encodedBytes(reason), 0); + const charLimit = reasonLength <= available ? Infinity : Math.floor(available / failures.length); + const byteLimit = reasonBytes <= availableBytes ? Infinity : Math.floor(availableBytes / failures.length); + const details = reasons.map((reason, index) => + `${prefixes[index]}${shortenReason(reason, charLimit, byteLimit)}.`, + ); + return header + details.join('') + footer; +} diff --git a/packages/code/src/environment-preparation.test.ts b/packages/code/src/environment-preparation.test.ts new file mode 100644 index 00000000..55186064 --- /dev/null +++ b/packages/code/src/environment-preparation.test.ts @@ -0,0 +1,365 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { + chmod, + mkdtemp, + mkdir, + readFile, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; +import { prepareCodeEnvironment } from './environment-preparation.js'; +import { parseCodeEnvironment } from './environment.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { prepareEnvironmentPreparationDirectory } from './storage.js'; +import type { EnvironmentPreparationOptions } from './environment-preparation.js'; +import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; + +async function fixture(t: test.TestContext) { + const directory = await mkdtemp(join(tmpdir(), 'code-preparation-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'checkout'); + const state = join(directory, 'private'); + await mkdir(root, { mode: 0o700 }); + await prepareEnvironmentPreparationDirectory(state); + await writeFile(join(root, 'package-lock.json'), 'lock-v1'); + const commands: string[] = []; + const options: EnvironmentPreparationOptions = { + root: (await captureWorkspaceRootIdentity(root)).path, + identity: await captureWorkspaceRootIdentity(root), + receiptPath: join(state, 'receipt.json'), + context: 'policy-v1', + setup: { + command: 'install', + timeoutMs: 1000, + reuse: { + inputs: ['package-lock.json'], + checkCommand: 'check', + checkTimeoutMs: 100, + }, + }, + async execute(command) { + commands.push(command); + return { exitCode: 0, timedOut: false }; + }, + }; + return { root, state, commands, options }; +} + +test('unchanged checkout checks readiness without repeating installation, including after restart', async t => { + const { options, commands } = await fixture(t); + assert.equal(await prepareCodeEnvironment(options), 'prepared'); + assert.equal(await prepareCodeEnvironment({ ...options }), 'reused'); + assert.deepEqual(commands, ['install', 'check', 'check']); + assert.equal( + (await readFile(options.receiptPath, 'utf8')).includes('lock-v1'), + false, + ); +}); + +test('changed inputs, recipe and toolchain/policy context invalidate preparation', async t => { + const { options, root, commands } = await fixture(t); + await prepareCodeEnvironment(options); + await writeFile(join(root, 'package-lock.json'), 'lock-v2'); + await prepareCodeEnvironment(options); + await prepareCodeEnvironment({ ...options, context: 'policy-v2' }); + await prepareCodeEnvironment({ + ...options, + context: 'policy-v2', + setup: { ...options.setup, command: 'install-v2' }, + }); + assert.deepEqual(commands, [ + 'install', + 'check', + 'install', + 'check', + 'install', + 'check', + 'install-v2', + 'check', + ]); +}); + +test('missing installed artifacts trigger repair; a failed setup is never stamped', async t => { + const { options, commands } = await fixture(t); + await prepareCodeEnvironment(options); + let installed = false; + await prepareCodeEnvironment({ + ...options, + async execute(command) { + commands.push(command); + if (command === 'install') installed = true; + return { exitCode: installed ? 0 : 1, timedOut: false }; + }, + }); + assert.deepEqual(commands, [ + 'install', + 'check', + 'check', + 'install', + 'check', + ]); + const failed = { + ...options, + receiptPath: join(options.receiptPath, '..', 'failed.json'), + execute: async () => ({ exitCode: 1, timedOut: false }), + }; + await assert.rejects(prepareCodeEnvironment(failed), /setup failed/); + await assert.rejects(readFile(failed.receiptPath), { code: 'ENOENT' }); +}); + +test('timed-out readiness and cancellation do not fall through to another setup', async t => { + const { options, commands } = await fixture(t); + await prepareCodeEnvironment(options); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + execute: async command => { + commands.push(command); + return { exitCode: 0, timedOut: true }; + }, + }), + /did not settle/, + ); + assert.deepEqual(commands, ['install', 'check', 'check']); + const controller = new AbortController(); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + signal: controller.signal, + execute: async () => { + controller.abort(); + return { exitCode: 0, timedOut: false }; + }, + }), + { name: 'AbortError' }, + ); +}); + +test('changing inputs during setup or readiness never publishes success', async t => { + const { options, root } = await fixture(t); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + execute: async () => { + await writeFile(join(root, 'package-lock.json'), 'changed'); + return { exitCode: 0, timedOut: false }; + }, + }), + /inputs changed/, + ); + await assert.rejects(readFile(options.receiptPath), { code: 'ENOENT' }); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + execute: async command => { + if (command === 'check') + await writeFile( + join(root, 'package-lock.json'), + 'changed-again', + ); + return { exitCode: 0, timedOut: false }; + }, + }), + /inputs changed/, + ); +}); + +test('input hashing is bounded and refuses symlink traversal and non-files', async t => { + const { options, root } = await fixture(t); + const outside = join(root, '..', 'outside'); + await mkdir(outside); + await writeFile(join(outside, 'secret'), 'do-not-read'); + await symlink(outside, join(root, 'escape')); + for (const path of ['escape/secret', 'escape']) { + await assert.rejects( + prepareCodeEnvironment({ + ...options, + setup: { + ...options.setup, + reuse: { ...options.setup.reuse!, inputs: [path] }, + }, + }), + ); + } + await writeFile(join(root, 'big'), Buffer.alloc(8 * 1024 * 1024 + 1)); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + setup: { + ...options.setup, + reuse: { ...options.setup.reuse!, inputs: ['big'] }, + }, + }), + /bounded regular-file/, + ); +}); + +test('legacy setup always runs and does not create preparation state', async t => { + const { options, commands } = await fixture(t); + const legacy = { + ...options, + setup: { command: 'install', timeoutMs: 1000 }, + }; + await prepareCodeEnvironment(legacy); + await prepareCodeEnvironment(legacy); + assert.deepEqual(commands, ['install', 'install']); + await assert.rejects(readFile(options.receiptPath), { code: 'ENOENT' }); +}); + +test('a foreign checkout receipt is not installation evidence and exposed state fails closed', async t => { + const { options, commands } = await fixture(t); + await prepareCodeEnvironment(options); + const other = join(options.root, '..', 'other'); + await mkdir(other); + await writeFile(join(other, 'package-lock.json'), 'lock-v1'); + await prepareCodeEnvironment({ + ...options, + root: (await captureWorkspaceRootIdentity(other)).path, + identity: await captureWorkspaceRootIdentity(other), + }); + assert.deepEqual(commands, ['install', 'check', 'install', 'check']); + await chmod(options.receiptPath, 0o644); + await assert.rejects( + prepareCodeEnvironment({ + ...options, + root: (await captureWorkspaceRootIdentity(other)).path, + identity: await captureWorkspaceRootIdentity(other), + }), + /owner-only/, + ); +}); + +test('reuse YAML requires bounded explicit inputs and readiness check', () => { + assert.ok( + parseCodeEnvironment( + 'name: app\nroot: app\nsetup:\n command: npm ci\n reuse:\n inputs: [package.json, package-lock.json]\n checkCommand: test -d node_modules\n', + ).setup?.reuse, + ); + for (const reuse of [ + 'inputs: []\ncheckCommand: ready', + 'inputs: [../outside]\ncheckCommand: ready', + 'inputs: [package.json, package.json]\ncheckCommand: ready', + 'inputs: [package.json]', + 'inputs: [package.json]\ncheckCommand: ready\ncheckTimeoutMs: 0', + 'inputs: [package.json]\ncheckCommand: ready\nunknown: true', + ]) + assert.throws(() => + parseCodeEnvironment( + `name: app\nroot: app\nsetup:\n command: install\n reuse:\n${reuse + .split('\n') + .map(line => ` ${line}`) + .join('\n')}\n`, + ), + ); +}); + +test('real local shell canary: installation reused, deleted output repaired, changed lockfile reinstalled', async t => { + const { options } = await fixture(t); + const run = promisify(execFile); + const ready = join(options.root, 'installed'); + const configured: EnvironmentPreparationOptions = { + ...options, + setup: { + command: 'printf "install\\n" >> installs.log; mkdir -p installed', + timeoutMs: 1000, + reuse: { + inputs: ['package-lock.json'], + checkCommand: 'test -d installed', + checkTimeoutMs: 1000, + }, + }, + async execute(command, timeout) { + try { + await run('/bin/sh', ['-c', command], { + cwd: options.root, + timeout, + }); + return { exitCode: 0, timedOut: false }; + } catch (error) { + return { + exitCode: 1, + timedOut: Boolean((error as { killed?: boolean }).killed), + }; + } + }, + }; + await prepareCodeEnvironment(configured); + assert.equal(await prepareCodeEnvironment(configured), 'reused'); + await rm(ready, { recursive: true }); + await prepareCodeEnvironment(configured); + await writeFile(resolve(options.root, 'package-lock.json'), 'new-lock'); + await prepareCodeEnvironment(configured); + assert.equal( + await readFile(join(options.root, 'installs.log'), 'utf8'), + 'install\ninstall\ninstall\n', + ); +}); + +test( + 'real native SRT preparation reuse keeps receipts inaccessible to checkout commands', + { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 30_000, + }, + async t => { + const { options, state } = await fixture(t); + const sandbox = new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: options.root, + workspaceIdentity: options.identity, + protectedPaths: [state], + }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + const execute = ( + command: string, + timeoutMs: number, + ) => + sandbox.execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command, + timeoutMs, + maxOutputBytes: 1024, + }); + const configured = { + ...options, + execute, + setup: { + command: + 'printf "install\\n" >> installs.log; mkdir -p installed', + timeoutMs: 5000, + reuse: { + inputs: ['package-lock.json'], + checkCommand: 'test -d installed', + checkTimeoutMs: 5000, + }, + }, + }; + assert.equal(await prepareCodeEnvironment(configured), 'prepared'); + assert.equal(await prepareCodeEnvironment(configured), 'reused'); + const receipt = await readFile(options.receiptPath, 'utf8'); + const protectedRead = await execute(`cat '${options.receiptPath}'`, 5000); + // Linux SRT may mask denied reads with an empty file. Exit status alone + // is not evidence that the protected receipt was exposed. + assert.ok(!protectedRead.stdout?.includes(receipt)); + await execute(`printf forged > '${options.receiptPath}'`, 5000); + assert.equal(await readFile(options.receiptPath, 'utf8'), receipt); + assert.equal( + await readFile(join(options.root, 'installs.log'), 'utf8'), + 'install\n', + ); + await rm(join(options.root, 'installed'), { recursive: true }); + assert.equal(await prepareCodeEnvironment(configured), 'prepared'); + assert.equal( + await readFile(join(options.root, 'installs.log'), 'utf8'), + 'install\ninstall\n', + ); + }, +); diff --git a/packages/code/src/environment-preparation.ts b/packages/code/src/environment-preparation.ts new file mode 100644 index 00000000..b17b4d32 --- /dev/null +++ b/packages/code/src/environment-preparation.ts @@ -0,0 +1,242 @@ +import { createHash } from 'node:crypto'; +import { constants } from 'node:fs'; +import { resolve } from 'node:path'; +import { open, withWorkspaceRoot } from './root-access.js'; +import { + loadEnvironmentPreparationKey, + saveEnvironmentPreparationKey, +} from './storage.js'; +import type { CodeEnvironmentDefinition } from './environment.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; +import { + withDependencySnapshot, + restoreDependencySnapshot, + publishDependencySnapshot, +} from './dependency-snapshots.js'; +import type { DependencySnapshotStore } from './dependency-snapshots.js'; +import { assertPreparationSpace } from './snapshot-lifecycle.js'; +import type { EnvironmentStoragePolicy } from './snapshot-lifecycle.js'; + +// Safety bounds on operator-declared hashing, not a dependency-store quota. +const MAX_INPUT_BYTES = 8 * 1024 * 1024; +const MAX_TOTAL_INPUT_BYTES = 32 * 1024 * 1024; + +export interface EnvironmentPreparationOptions { + root: string; + identity: WorkspaceRootIdentity; + setup: NonNullable; + receiptPath: string; + /** Includes the worker's policy and toolchain configuration. */ + context: string; + execute( + command: string, + timeoutMs: number, + ): Promise<{ exitCode: number | null; timedOut: boolean }>; + signal?: AbortSignal; + snapshotStore?: DependencySnapshotStore; + snapshotScope?: string; + beforeMutation?(): Promise; + storage?: EnvironmentStoragePolicy; + /** Frees reproducible storage when the storage floor would defer setup. */ + reclaimSpace?(): Promise; +} + +/** Checkout-local reuse. Never transfers mutable installations between worktrees. */ +export async function prepareCodeEnvironment( + options: EnvironmentPreparationOptions, +): Promise<'prepared' | 'reused' | 'restored'> { + if (options.snapshotStore) { + if (!options.setup.reuse?.snapshot || !options.snapshotScope) + throw new Error( + 'Dependency snapshot requires an explicit preparation scope', + ); + if (options.snapshotStore.identity.dev !== options.identity.dev) + throw new Error( + 'Dependency snapshots and checkouts must use the same clone-capable filesystem', + ); + const portable = await preparationKey(options, true); + return withDependencySnapshot( + options.snapshotStore, + portable!, + () => prepareInLock(options, portable), + options.signal, + true, + ); + } + return prepareInLock(options); +} + +async function prepareInLock( + options: EnvironmentPreparationOptions, + portable?: string, +): Promise<'prepared' | 'reused' | 'restored'> { + options.signal?.throwIfAborted(); + const key = await preparationKey(options); + if ( + key && + (await loadEnvironmentPreparationKey(options.receiptPath)) === key + ) { + const reuse = options.setup.reuse!; + const check = await options.execute( + reuse.checkCommand, + reuse.checkTimeoutMs, + ); + options.signal?.throwIfAborted(); + if (check.timedOut || check.exitCode === null) + throw new Error('Environment readiness check did not settle'); + if (check.exitCode === 0 && (await preparationKey(options)) === key) + return 'reused'; + } + if (options.snapshotStore && portable) { + if (options.storage) + await assertPreparationSpace( + options.root, + options.storage, + undefined, + options.reclaimSpace, + ); + await options.beforeMutation?.(); + if ( + await restoreDependencySnapshot( + options.snapshotStore, + portable, + options.root, + options.identity, + options.signal, + ) + ) { + const reuse = options.setup.reuse!; + const check = await options.execute( + reuse.checkCommand, + reuse.checkTimeoutMs, + ); + options.signal?.throwIfAborted(); + if (check.timedOut || check.exitCode === null) + throw new Error( + 'Restored dependency readiness check did not settle', + ); + if ( + check.exitCode === 0 && + (await preparationKey(options)) === key + ) { + await saveEnvironmentPreparationKey(options.receiptPath, key!); + return 'restored'; + } + } + } + if (options.storage) + await assertPreparationSpace( + options.root, + options.storage, + undefined, + options.reclaimSpace, + ); + const result = await options.execute( + options.setup.command, + options.setup.timeoutMs, + ); + options.signal?.throwIfAborted(); + if (result.timedOut || result.exitCode !== 0) + throw new Error('Environment setup failed'); + if (key) { + // Do not stamp an installation against inputs that changed during setup. + if ((await preparationKey(options)) !== key) + throw new Error( + 'Environment preparation inputs changed during setup', + ); + const reuse = options.setup.reuse!; + const check = await options.execute( + reuse.checkCommand, + reuse.checkTimeoutMs, + ); + options.signal?.throwIfAborted(); + if (check.timedOut || check.exitCode !== 0) + throw new Error('Environment readiness check failed after setup'); + if ((await preparationKey(options)) !== key) + throw new Error( + 'Environment preparation inputs changed during readiness check', + ); + await saveEnvironmentPreparationKey(options.receiptPath, key); + if (options.snapshotStore && portable) + await publishDependencySnapshot( + options.snapshotStore, + portable, + options.root, + options.identity, + options.signal, + ); + } + return 'prepared'; +} + +async function preparationKey( + options: EnvironmentPreparationOptions, + portable = false, +): Promise { + if (!options.setup.reuse) return undefined; + return withWorkspaceRoot(options.root, options.identity, async () => { + const hash = createHash('sha256').update( + JSON.stringify({ + version: 1, + root: portable ? options.snapshotScope : options.identity, + setup: options.setup, + context: options.context, + node: process.version, + abi: process.versions.modules, + platform: process.platform, + arch: process.arch, + }), + ); + let total = 0; + for (const path of options.setup.reuse!.inputs) { + options.signal?.throwIfAborted(); + const handle = await open( + resolve(options.root, path), + constants.O_RDONLY | + constants.O_NOFOLLOW | + constants.O_NONBLOCK, + ); + try { + const before = await handle.stat(); + if ( + !before.isFile() || + before.size > MAX_INPUT_BYTES || + total + before.size > MAX_TOTAL_INPUT_BYTES + ) + throw new Error( + 'Environment preparation inputs exceed the bounded regular-file contract', + ); + const buffer = Buffer.alloc(before.size + 1); + let length = 0; + while (length < buffer.length) { + options.signal?.throwIfAborted(); + const read = await handle.read( + buffer, + length, + buffer.length - length, + length, + ); + if (!read.bytesRead) break; + length += read.bytesRead; + } + const after = await handle.stat(); + if ( + length !== before.size || + after.size !== before.size || + after.mtimeMs !== before.mtimeMs || + after.ctimeMs !== before.ctimeMs + ) + throw new Error( + 'Environment preparation input changed while hashing', + ); + total += length; + hash.update(JSON.stringify([path, length])).update( + buffer.subarray(0, length), + ); + } finally { + await handle.close(); + } + } + return hash.digest('hex'); + }); +} diff --git a/packages/code/src/environment-resources.test.ts b/packages/code/src/environment-resources.test.ts new file mode 100644 index 00000000..32d1d390 --- /dev/null +++ b/packages/code/src/environment-resources.test.ts @@ -0,0 +1,210 @@ +import assert from 'node:assert/strict'; +import { + chmod, + mkdir, + mkdtemp, + readFile, + rename, + rm, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { + parseCodeEnvironment, + loadCodeEnvironment, + assertEnvironmentDefinitionsOutsideRoots, +} from './environment.js'; +import { + parseEnvironmentResources, + loadEnvironmentResource, + assertEnvironmentResourcesStable, +} from './environment-resources.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { NativeProcessWorkspaceCommandSandbox } from './native-process.js'; + +test('shared resources are explicit bounded known cache kinds, not arbitrary environment or filesystem grants', () => { + assert.equal( + parseCodeEnvironment( + 'name: app\nroot: app\nresources:\n - kind: npm-cache\n path: /cache/npm\n access: read-write\n', + ).resources?.[0].kind, + 'npm-cache', + ); + for (const value of [ + [], + [{ kind: 'secrets', path: '/cache', access: 'read-write' }], + [{ kind: 'npm-cache', path: '../cache', access: 'read-write' }], + [{ kind: 'uv-cache', path: '/cache', access: 'all' }], + [ + { + kind: 'uv-cache', + path: '/cache', + access: 'read-only', + env: 'GH_TOKEN', + }, + ], + [1], + [ + { kind: 'uv-cache', path: '/cache', access: 'read-only' }, + { kind: 'uv-cache', path: '/other', access: 'read-only' }, + ], + ]) + assert.throws(() => parseEnvironmentResources(value)); +}); + +test('resource roots must be stable private directories and outside source grants', async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-resources-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const root = join(directory, 'root'); + const cache = join(directory, 'cache'); + await mkdir(root, { mode: 0o700 }); + await mkdir(cache, { mode: 0o700 }); + const resource = { + kind: 'npm-cache' as const, + path: cache, + access: 'read-write' as const, + }; + const loaded = await loadEnvironmentResource(resource); + await assertEnvironmentResourcesStable([loaded]); + await chmod(cache, 0o755); + await assert.rejects(loadEnvironmentResource(resource), /owner-only/); + await chmod(cache, 0o700); + await symlink(cache, join(directory, 'alias')); + await assert.rejects( + loadEnvironmentResource({ + ...resource, + path: join(directory, 'alias'), + }), + /symlink/, + ); + await rename(cache, join(directory, 'old')); + await mkdir(cache, { mode: 0o700 }); + await assert.rejects(assertEnvironmentResourcesStable([loaded]), /changed/); + const file = join(directory, 'app.yaml'); + await writeFile( + file, + `name: app\nroot: ${root}\nresources:\n - kind: npm-cache\n path: ${cache}\n access: read-write\n`, + { mode: 0o600 }, + ); + const environment = await loadCodeEnvironment(file); + const [cacheRoot] = environment.resources!; + await assert.rejects( + assertEnvironmentDefinitionsOutsideRoots( + [ + { + path: cacheRoot.path, + sourceParents: cacheRoot.controlPaths, + fingerprint: '', + definition: { name: 'cache', root: cacheRoot.path }, + }, + ], + [{ id: 'root', root: directory }], + ), + /workspace|root|definition/i, + ); +}); + +test( + 'real native SRT shares only declared stores across independent checkouts and refuses readonly writes', + { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 30_000, + }, + async t => { + const directory = await mkdtemp(join(tmpdir(), 'code-resources-live-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const cache = join(directory, 'cache'); + const browser = join(directory, 'browsers'); + await mkdir(cache, { mode: 0o700 }); + await mkdir(browser, { mode: 0o700 }); + await writeFile(join(browser, 'version'), 'pinned-browser'); + const resources = await Promise.all([ + loadEnvironmentResource({ + kind: 'npm-cache', + path: cache, + access: 'read-write', + }), + loadEnvironmentResource({ + kind: 'playwright-browsers', + path: browser, + access: 'read-only', + }), + ]); + const sandboxes = await Promise.all( + ['a', 'b'].map(async name => { + const root = join(directory, name); + await mkdir(root); + const identity = await captureWorkspaceRootIdentity(root); + await writeFile(join(root, 'private'), 'other-checkout'); + const sandbox = new NativeProcessWorkspaceCommandSandbox({ + workspaceRoot: identity.path, + workspaceIdentity: identity, + homeDirectory: directory, + resources, + }); + await sandbox.prepare(); + return sandbox; + }), + ); + t.after(async () => { + for (const sandbox of sandboxes) await sandbox.close(); + }); + const execute = (index: number, command: string) => + sandboxes[index].execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command, + timeoutMs: 5000, + }); + const results = await Promise.all( + sandboxes.map((_, i) => + execute( + i, + 'printf "%s\\n" "$npm_config_cache"; printf once >> "$npm_config_cache/marker"; cat "$PLAYWRIGHT_BROWSERS_PATH/version"', + ), + ), + ); + for (const result of results) { + assert.equal(result.exitCode, 0, result.stderr); + assert.match(result.stdout, /pinned-browser/); + } + assert.equal(await readFile(join(cache, 'marker'), 'utf8'), 'onceonce'); + assert.notEqual( + ( + await execute( + 0, + 'printf forbidden > "$PLAYWRIGHT_BROWSERS_PATH/version"', + ) + ).exitCode, + 0, + ); + assert.equal( + await readFile(join(browser, 'version'), 'utf8'), + 'pinned-browser', + ); + assert.notEqual( + (await execute(0, `cat '${join(directory, 'b', 'private')}'`)) + .exitCode, + 0, + ); + await rename(cache, join(directory, 'old-cache')); + await mkdir(cache, { mode: 0o700 }); + await assert.rejects( + execute(0, 'printf not-started > side-effect'), + error => { + assert.equal( + (error as { mutationMayHaveCommitted: boolean }) + .mutationMayHaveCommitted, + false, + ); + return /resource changed/.test((error as Error).message); + }, + ); + await assert.rejects(readFile(join(directory, 'a', 'side-effect')), { + code: 'ENOENT', + }); + }, +); diff --git a/packages/code/src/environment-resources.ts b/packages/code/src/environment-resources.ts new file mode 100644 index 00000000..1bb57440 --- /dev/null +++ b/packages/code/src/environment-resources.ts @@ -0,0 +1,164 @@ +import { lstat, open, realpath } from 'node:fs/promises'; +import { constants } from 'node:fs'; +import { isAbsolute, relative, sep } from 'node:path'; +import { + assertPrivateStorageAcl, + assertPrivateStorageAncestors, +} from './private-storage.js'; +import { + captureWorkspaceRootIdentity, + matchesWorkspaceRoot, +} from './root-identity.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; +import { + createEnvironmentMountIsolation, + readEnvironmentMountTable, +} from './environment-mount.js'; + +export type EnvironmentResourceKind = + | 'npm-cache' + | 'uv-cache' + | 'playwright-browsers'; +export interface EnvironmentResource { + kind: EnvironmentResourceKind; + path: string; + access: 'read-only' | 'read-write'; +} +export interface LoadedEnvironmentResource extends EnvironmentResource { + identity: WorkspaceRootIdentity; + controlPaths: string[]; +} +export const RESOURCE_ENVIRONMENT_NAMES: Record< + EnvironmentResourceKind, + string +> = { + 'npm-cache': 'npm_config_cache', + 'uv-cache': 'UV_CACHE_DIR', + 'playwright-browsers': 'PLAYWRIGHT_BROWSERS_PATH', +}; + +export function parseEnvironmentResources( + value: unknown, +): EnvironmentResource[] { + if (!Array.isArray(value) || value.length < 1 || value.length > 3) + throw new Error('Invalid environment resources'); + const kinds = new Set(); + return value.map(raw => { + if (!raw || typeof raw !== 'object' || Array.isArray(raw)) + throw new Error('Invalid environment resource'); + const entry = raw as Record; + if ( + Object.keys(entry).some( + key => !['kind', 'path', 'access'].includes(key), + ) || + typeof entry.kind !== 'string' || + !Object.hasOwn(RESOURCE_ENVIRONMENT_NAMES, entry.kind) || + kinds.has(entry.kind) || + typeof entry.path !== 'string' || + !isAbsolute(entry.path) || + entry.path.length > 4096 || + /[\0\r\n]/.test(entry.path) || + (entry.access !== 'read-only' && entry.access !== 'read-write') + ) + throw new Error('Invalid environment resource'); + kinds.add(entry.kind); + return { + kind: entry.kind as EnvironmentResourceKind, + path: entry.path, + access: entry.access, + }; + }); +} + +/** Operator-owned roots only. Resource contents are still untrusted tool data. */ +export async function loadEnvironmentResource( + resource: EnvironmentResource, +): Promise { + const controlPaths = await assertPrivateStorageAncestors(resource.path); + if ((await lstat(resource.path)).isSymbolicLink()) + throw new Error('Environment resource root cannot be a symlink'); + const path = await realpath(resource.path); + const identity = await captureWorkspaceRootIdentity(path); + const handle = await open( + path, + constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW, + ); + try { + const metadata = await handle.stat({ bigint: true }); + if ( + (metadata.mode & 0o077n) !== 0n || + metadata.dev.toString() !== identity.dev || + metadata.ino.toString() !== identity.ino + ) + throw new Error( + 'Environment resource directory must be stable and owner-only', + ); + await assertPrivateStorageAcl(handle, path, true); + } finally { + await handle.close(); + } + return { ...resource, path, identity, controlPaths }; +} + +export async function assertEnvironmentResourcesStable( + resources: readonly LoadedEnvironmentResource[], +): Promise { + for (const resource of resources) { + if (!(await matchesWorkspaceRoot(resource.path, resource.identity))) + throw new Error( + 'Environment resource root changed after admission', + ); + } +} + +export function environmentResourceVariables( + resources: readonly EnvironmentResource[], +): Record { + return Object.fromEntries( + resources.map(resource => [ + RESOURCE_ENVIRONMENT_NAMES[resource.kind], + resource.path, + ]), + ); +} + +export function assertEnvironmentResourceSeparation( + resources: readonly EnvironmentResource[], + roots: readonly string[], +): void { + const contains = (parent: string, child: string) => { + const path = relative(parent, child); + return ( + path === '' || + (!isAbsolute(path) && path !== '..' && !path.startsWith(`..${sep}`)) + ); + }; + for (const resource of resources) { + if ( + roots.some( + root => + contains(root, resource.path) || + contains(resource.path, root), + ) + ) + throw new Error( + 'Environment resources must not overlap any registered workspace', + ); + } +} + +export async function assertEnvironmentResourceIsolation( + resources: readonly EnvironmentResource[], + roots: readonly string[], + controls: readonly string[], +): Promise { + if (!resources.length) return; + assertEnvironmentResourceSeparation(resources, [...roots, ...controls]); + const table = await readEnvironmentMountTable(); + if (table !== undefined) { + const check = createEnvironmentMountIsolation(table); + const stores = resources.map(resource => resource.path); + check(stores, roots); + check([...roots, ...controls], stores); + } +} diff --git a/packages/code/src/environment.ts b/packages/code/src/environment.ts index c2b2cf1f..c527729f 100644 --- a/packages/code/src/environment.ts +++ b/packages/code/src/environment.ts @@ -3,6 +3,12 @@ import { constants } from 'node:fs'; import { open, realpath, stat } from 'node:fs/promises'; import { dirname, isAbsolute, relative, resolve, sep } from 'node:path'; import { parseDocument } from 'yaml'; +import { parseEnvironmentResources, loadEnvironmentResource } from './environment-resources.js'; +import { parseDependencySnapshot, loadDependencySnapshot } from './dependency-snapshots.js'; +import type { DependencySnapshotConfig, DependencySnapshotStore } from './dependency-snapshots.js'; +import { parseEnvironmentStorage } from './snapshot-lifecycle.js'; +import type { EnvironmentStoragePolicy } from './snapshot-lifecycle.js'; +import type { EnvironmentResource, LoadedEnvironmentResource } from './environment-resources.js'; import { assertPrivateStorageAcl, assertPrivateStorageAncestors, @@ -10,6 +16,7 @@ import { import { BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS, BRIDGE_WORKSPACE_COMMAND_MAX_BYTES, + isSafePortableRelativePath, } from './protocol.js'; import type { LocalWorkspaceConfig } from './workspace.js'; import { WorkspaceToolError } from './workspace.js'; @@ -25,8 +32,15 @@ export interface CodeEnvironmentDefinition { root: string; repo?: string; ref?: string; - setup?: { command: string; timeoutMs: number }; + setup?: { + command: string; + timeoutMs: number; + /** Explicit readiness contract; absent preserves startup setup behavior. */ + reuse?: { inputs: string[]; checkCommand: string; checkTimeoutMs: number; snapshot?: DependencySnapshotConfig }; + }; actions?: { name: string; command: string; timeoutMs: number }[]; + resources?: EnvironmentResource[]; + storage?: EnvironmentStoragePolicy; } export interface LoadedCodeEnvironment { @@ -35,6 +49,8 @@ export interface LoadedCodeEnvironment { rootPaths?: string[]; definition: CodeEnvironmentDefinition; fingerprint: string; + resources?: LoadedEnvironmentResource[]; + snapshotStore?: DependencySnapshotStore; } function record(value: unknown): value is Record { @@ -67,7 +83,7 @@ export function parseCodeEnvironment( !record(value) || Object.keys(value).some( key => - !['name', 'root', 'repo', 'ref', 'setup', 'actions'].includes( + !['name', 'root', 'repo', 'ref', 'setup', 'actions', 'resources', 'storage'].includes( key, ), ) || @@ -89,7 +105,7 @@ export function parseCodeEnvironment( if ( !record(value.setup) || Object.keys(value.setup).some( - key => !['command', 'timeoutMs'].includes(key), + key => !['command', 'timeoutMs', 'reuse'].includes(key), ) || !text(value.setup.command, 16_384) || Buffer.byteLength(value.setup.command) > @@ -109,7 +125,26 @@ export function parseCodeEnvironment( `Environment setup timeout must be between 1 and ${BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS} ms`, ); } - setup = { command: value.setup.command, timeoutMs }; + let reuse: NonNullable['reuse']; + if (value.setup.reuse !== undefined) { + const candidate = value.setup.reuse; + if ( + !record(candidate) || + Object.keys(candidate).some(key => !['inputs', 'checkCommand', 'checkTimeoutMs', 'snapshot'].includes(key)) || + !Array.isArray(candidate.inputs) || + candidate.inputs.length < 1 || candidate.inputs.length > 32 || + candidate.inputs.some(path => typeof path !== 'string' || !isSafePortableRelativePath(path) || path === '.') || + new Set(candidate.inputs).size !== candidate.inputs.length || + !text(candidate.checkCommand, 16_384) || + Buffer.byteLength(candidate.checkCommand) > BRIDGE_WORKSPACE_COMMAND_MAX_BYTES + ) throw new Error('Invalid environment setup reuse'); + const checkTimeoutMs = candidate.checkTimeoutMs ?? 10_000; + if (typeof checkTimeoutMs !== 'number' || !Number.isSafeInteger(checkTimeoutMs) || checkTimeoutMs < 1 || checkTimeoutMs > BRIDGE_WORKSPACE_COMMAND_MAX_TIMEOUT_MS) + throw new Error('Invalid environment readiness check timeout'); + reuse = { inputs: candidate.inputs as string[], checkCommand: candidate.checkCommand, checkTimeoutMs, + ...(candidate.snapshot !== undefined ? { snapshot: parseDependencySnapshot(candidate.snapshot) } : {}) }; + } + setup = { command: value.setup.command, timeoutMs, ...(reuse ? { reuse } : {}) }; } let actions: CodeEnvironmentDefinition['actions']; if (value.actions !== undefined) { @@ -149,6 +184,8 @@ export function parseCodeEnvironment( ...(typeof value.ref === 'string' ? { ref: value.ref } : {}), ...(setup ? { setup } : {}), ...(actions ? { actions } : {}), + ...(value.resources !== undefined ? { resources: parseEnvironmentResources(value.resources) } : {}), + ...(value.storage !== undefined ? { storage: parseEnvironmentStorage(value.storage) } : {}), }; } @@ -200,12 +237,13 @@ export class EnvironmentWorkspaceTools implements WorkspaceToolExecutor { async execute( request: WorkspaceToolRequest, signal?: AbortSignal, + context?: { deadlineAtMs?: number }, ): Promise { if ( request.operation !== 'execute_command' || !request.environmentAction ) { - return this.delegate.execute(request, signal); + return this.delegate.execute(request, signal, context); } const environment = this.environments.get(request.workspaceId); const action = environment?.definition.actions?.find( @@ -234,6 +272,7 @@ export class EnvironmentWorkspaceTools implements WorkspaceToolExecutor { cwd: '.', }, signal, + context, ); } } @@ -304,6 +343,8 @@ export async function loadCodeEnvironment( sourceParents, rootPaths, definition, + ...(definition.resources ? { resources: await Promise.all(definition.resources.map(loadEnvironmentResource)) } : {}), + ...(definition.setup?.reuse?.snapshot ? { snapshotStore: await loadDependencySnapshot(definition.setup.reuse.snapshot) } : {}), fingerprint: createHash('sha256') .update(JSON.stringify(definition)) .digest('hex'), diff --git a/packages/code/src/github.test.ts b/packages/code/src/github.test.ts index b13d70f7..5d401515 100644 --- a/packages/code/src/github.test.ts +++ b/packages/code/src/github.test.ts @@ -239,6 +239,272 @@ test('routes and scopes GitHub App tokens per repository installation', async (t ); }); +test('installation scope shares one token across repositories in an organization and refreshes grants', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-installation-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile( + privateKeyPath, + privateKey.export({ type: 'pkcs8', format: 'pem' }), + { mode: 0o600 }, + ); + let now = new Date('2030-01-01T00:00:00Z'); + const minted: Array<{ installation: string; body?: string }> = []; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + tokenScope: 'installation', + privateKeyPath, + now: () => now, + fetch: (async (input, init) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (/\/repos\/LibreChat-AI\/(agents|LibreChat)\/installation$/.test(url)) { + return Response.json({ id: 111 }); + } + if (url.endsWith('/repos/ClickHouse/Agents/installation')) { + return Response.json({ id: 222 }); + } + const installation = /\/app\/installations\/(\d+)\/access_tokens$/.exec(url)?.[1]; + if (installation) { + minted.push({ + installation, + body: typeof init?.body === 'string' ? init.body : undefined, + }); + return Response.json({ + token: `ghs_${installation}_${minted.length}_abcdefghijklmnopqrstuvwxyz`, + expires_at: '2030-01-01T01:00:00Z', + }, { status: 201 }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + const [agents, librechat] = await Promise.all([ + provider.getCredential(undefined, 'LibreChat-AI/agents'), + provider.getCredential(undefined, 'LibreChat-AI/LibreChat'), + ]); + assert.equal(agents.value, librechat.value); + assert.deepEqual(minted, [{ installation: '111', body: undefined }]); + const clickhouse = await provider.getCredential(undefined, 'ClickHouse/Agents'); + assert.notEqual(clickhouse.value, librechat.value); + assert.deepEqual(minted.map(entry => entry.installation), ['111', '222']); + now = new Date('2030-01-01T00:02:01Z'); + const refreshed = await provider.getCredential(undefined, 'LibreChat-AI/LibreChat'); + assert.notEqual(refreshed.value, librechat.value); + assert.deepEqual(minted.map(entry => entry.installation), ['111', '222', '111']); +}); + +test('installation scope shares a bounded lookup when a waiter cancels', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-shared-lookup-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile(privateKeyPath, privateKey.export({ type: 'pkcs8', format: 'pem' }), { + mode: 0o600, + }); + let releaseLookup!: (response: Response) => void; + const lookupResponse = new Promise(resolve => { releaseLookup = resolve; }); + let lookupStarted!: () => void; + const started = new Promise(resolve => { lookupStarted = resolve; }); + const cancelled = new AbortController(); + let lookups = 0; + let mints = 0; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + tokenScope: 'installation', + privateKeyPath, + now: () => new Date('2030-01-01T00:00:00Z'), + fetch: (async (input, init) => { + const url = String(input); + if (url.endsWith('/repos/acme/project/installation')) { + lookups += 1; + assert.ok(init?.signal); + assert.notEqual(init.signal, cancelled.signal); + lookupStarted(); + return lookupResponse; + } + if (url.endsWith('/app/installations/111/access_tokens')) { + mints += 1; + return Response.json({ + token: 'ghs_shared_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }, { status: 201 }); + } + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + const first = provider.getCredential(cancelled.signal, 'acme/project'); + const others = Array.from({ length: 12 }, () => + provider.getCredential(undefined, 'acme/project')); + await started; + cancelled.abort(new Error('first command cancelled')); + await assert.rejects(first, /first command cancelled/); + releaseLookup(Response.json({ id: 111 })); + const credentials = await Promise.all(others); + assert.equal(lookups, 1); + assert.equal(mints, 1); + assert.equal(new Set(credentials.map(credential => credential.value)).size, 1); +}); + +test('installation scope times out a stalled lookup without a caller signal', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-lookup-timeout-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile(privateKeyPath, privateKey.export({ type: 'pkcs8', format: 'pem' }), { + mode: 0o600, + }); + const timeout = new AbortController(); + t.mock.method(AbortSignal, 'timeout', (ms: number) => { + assert.equal(ms, 30_000); + return timeout.signal; + }); + let lookupStarted!: () => void; + const started = new Promise(resolve => { lookupStarted = resolve; }); + const provider = new GitHubAppCredentialProvider({ + appId: '123', + tokenScope: 'installation', + privateKeyPath, + fetch: (async (input, init) => { + if (String(input).endsWith('/repos/acme/project/installation')) { + assert.equal(init?.signal, timeout.signal); + lookupStarted(); + return new Promise((_resolve, reject) => { + timeout.signal.addEventListener('abort', () => reject(timeout.signal.reason), { once: true }); + }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + const pending = provider.getCredential(undefined, 'acme/project'); + await started; + timeout.abort(new DOMException('Installation lookup timed out', 'TimeoutError')); + await assert.rejects(pending, { name: 'TimeoutError' }); +}); + +test('installation replacement checks every shared waiter’s repository grant', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-replaced-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile(privateKeyPath, privateKey.export({ type: 'pkcs8', format: 'pem' }), { + mode: 0o600, + }); + let now = new Date('2030-01-01T00:00:00Z'); + let replaced = false; + let bLookups = 0; + let oldMints = 0; + let releaseOldMint!: () => void; + const oldMintResponse = new Promise(resolve => { releaseOldMint = resolve; }); + let oldMintStarted!: () => void; + const oldMintPending = new Promise(resolve => { oldMintStarted = resolve; }); + const provider = new GitHubAppCredentialProvider({ + appId: '123', + tokenScope: 'installation', + privateKeyPath, + now: () => now, + fetch: (async input => { + const url = String(input); + if (url.endsWith('/repos/acme/a/installation')) { + return Response.json({ id: replaced ? 222 : 111 }); + } + if (url.endsWith('/repos/acme/b/installation')) { + bLookups += 1; + return replaced ? Response.json({}, { status: 404 }) : Response.json({ id: 111 }); + } + if (url.endsWith('/app/installations/111/access_tokens')) { + oldMints += 1; + if (replaced) { + oldMintStarted(); + await oldMintResponse; + return Response.json({}, { status: 404 }); + } + return Response.json({ + token: 'ghs_old_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T00:06:00Z', + }, { status: 201 }); + } + if (url.endsWith('/app/installations/222/access_tokens')) { + return Response.json({ + token: 'ghs_new_abcdefghijklmnopqrstuvwxyz', + expires_at: '2030-01-01T01:00:00Z', + }, { status: 201 }); + } + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + const old = await provider.getCredential(undefined, 'acme/a'); + assert.equal((await provider.getCredential(undefined, 'acme/b')).value, old.value); + replaced = true; + now = new Date('2030-01-01T00:01:01Z'); // Token needs refresh; both mappings are still live. + const forA = provider.getCredential(undefined, 'acme/a'); + await oldMintPending; + const forB = provider.getCredential(undefined, 'acme/b'); + await new Promise(resolve => setImmediate(resolve)); // Join A's stale installation mint. + releaseOldMint(); + const [a, b] = await Promise.allSettled([forA, forB]); + assert.equal(a.status, 'fulfilled'); + if (a.status === 'fulfilled') assert.equal(a.value.value, 'ghs_new_abcdefghijklmnopqrstuvwxyz'); + assert.equal(b.status, 'rejected'); + if (b.status === 'rejected') assert.match(String(b.reason), /not installed for acme\/b/); + assert.equal(oldMints, 2); // One initial mint and just one shared failed refresh. + assert.equal(bLookups, 2); // B must revalidate instead of receiving A's new token. +}); + +test('installation scope follows a repository transfer after the lookup expires', async (t) => { + const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-transfer-')); + t.after(() => rm(directory, { recursive: true, force: true })); + const privateKeyPath = join(directory, 'app.pem'); + const { privateKey } = generateKeyPairSync('rsa', { modulusLength: 2048 }); + await writeFile(privateKeyPath, privateKey.export({ type: 'pkcs8', format: 'pem' }), { + mode: 0o600, + }); + let now = new Date('2030-01-01T00:00:00Z'); + let lookupCount = 0; + const provider = new GitHubAppCredentialProvider({ + appId: '123', + tokenScope: 'installation', + privateKeyPath, + now: () => now, + fetch: (async (input) => { + const url = String(input); + if (url.endsWith('/app')) return Response.json({ slug: 'lia' }); + if (url.endsWith('/users/lia%5Bbot%5D')) { + return Response.json({ id: 1234, login: 'lia[bot]', type: 'Bot' }); + } + if (url.endsWith('/repos/acme/project/installation')) { + lookupCount += 1; + return Response.json({ id: lookupCount === 1 ? 111 : 222 }); + } + const installation = /\/app\/installations\/(\d+)\/access_tokens$/.exec(url)?.[1]; + if (installation) { + return Response.json({ + token: `ghs_${installation}_abcdefghijklmnopqrstuvwxyz`, + expires_at: '2030-01-01T01:00:00Z', + }); + } + return Response.json({}, { status: 404 }); + }) as typeof fetch, + }); + assert.equal((await provider.getCredential(undefined, 'acme/project')).value, + 'ghs_111_abcdefghijklmnopqrstuvwxyz'); + now = new Date('2030-01-01T00:02:01Z'); + assert.equal((await provider.getCredential(undefined, 'acme/project')).value, + 'ghs_222_abcdefghijklmnopqrstuvwxyz'); + assert.equal(lookupCount, 2); +}); + test('keeps a shared token refresh alive when one waiter is cancelled', async (t) => { const directory = await mkdtemp(join(tmpdir(), 'librechat-code-github-cancel-')); t.after(() => rm(directory, { recursive: true, force: true })); diff --git a/packages/code/src/github.ts b/packages/code/src/github.ts index 19b8e639..8519938f 100644 --- a/packages/code/src/github.ts +++ b/packages/code/src/github.ts @@ -41,6 +41,8 @@ export interface GitHubAppCredentialProviderOptions { appId: string; /** Legacy fixed installation. Omit to resolve the installation per repository. */ installationId?: string; + /** Opt in to all repositories granted to the resolved installation. */ + tokenScope?: 'repository' | 'installation'; privateKeyPath: string; apiUrl?: string; /** Git HTTPS hostname; non-public hosts default to the GHES /api/v3 base. */ @@ -52,6 +54,13 @@ export interface GitHubAppCredentialProviderOptions { const execFileAsync = promisify(execFile); const GITHUB_SHARED_REQUEST_TIMEOUT_MS = 30_000; +const GITHUB_INSTALLATION_CACHE_MS = 2 * 60_000; + +class InstallationChangedError extends Error { + constructor(readonly installationId: string) { + super('GitHub App installation changed while issuing a token'); + } +} async function waitForShared( promise: Promise, @@ -248,8 +257,14 @@ function createAppJwt(appId: string, privateKey: string, now: Date): string { export class GitHubAppCredentialProvider implements GitHubCredentialProvider { private readonly cached = new Map(); + private readonly cachedAt = new Map(); private readonly inFlight = new Map>(); private readonly installationIds = new Map(); + private readonly installationIdCachedAt = new Map(); + private readonly installationIdInFlight = new Map< + string, + Promise<{ id: string; jwt: string }> + >(); private appLogin?: string; private appLoginInFlight?: Promise; private actor?: GitHubCredential['actor']; @@ -423,32 +438,80 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { private async resolveInstallationId( repository: string, - jwt: string, signal?: AbortSignal, - ): Promise { - if (this.options.installationId) return this.options.installationId; + jwt?: string, + ): Promise<{ id: string; jwt?: string }> { + if (this.options.installationId) { + return { id: this.options.installationId, jwt }; + } const cached = this.installationIds.get(repository); - if (cached) return cached; - const { owner, name } = repositoryName(repository); - const response = await this.request( - `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/installation`, - jwt, - signal, - ); - if (!response.ok) { - throw new Error( - response.status === 404 - ? `GitHub App is not installed for ${repository}` - : `GitHub App installation lookup failed with status ${response.status}`, + const now = (this.options.now ?? (() => new Date()))().getTime(); + if ( + cached && + now - (this.installationIdCachedAt.get(repository) ?? 0) < GITHUB_INSTALLATION_CACHE_MS + ) return { id: cached, jwt }; + const existing = this.installationIdInFlight.get(repository); + if (existing) return waitForShared(existing, signal); + const pending = (async () => { + const sharedSignal = AbortSignal.timeout(GITHUB_SHARED_REQUEST_TIMEOUT_MS); + const lookupJwt = jwt ?? await this.appJwt( + (this.options.now ?? (() => new Date()))(), ); + const { owner, name } = repositoryName(repository); + const response = await this.request( + `/repos/${encodeURIComponent(owner)}/${encodeURIComponent(name)}/installation`, + lookupJwt, + sharedSignal, + ); + if (!response.ok) { + throw new Error( + response.status === 404 + ? `GitHub App is not installed for ${repository}` + : `GitHub App installation lookup failed with status ${response.status}`, + ); + } + const body = (await response.json()) as { id?: unknown }; + if (!Number.isSafeInteger(body.id) || Number(body.id) <= 0) { + throw new Error('GitHub App installation response is invalid'); + } + const id = String(body.id); + this.installationIds.set(repository, id); + this.installationIdCachedAt.set( + repository, + (this.options.now ?? (() => new Date()))().getTime(), + ); + return { id, jwt: lookupJwt }; + })(); + this.installationIdInFlight.set(repository, pending); + const clearPending = () => { + if (this.installationIdInFlight.get(repository) === pending) { + this.installationIdInFlight.delete(repository); + } + }; + void pending.then(clearPending, clearPending); + return waitForShared(pending, signal); + } + + private async waitForCredential( + pending: Promise, + signal: AbortSignal | undefined, + repository: string | undefined, + retried: boolean, + ): Promise { + try { + return await waitForShared(pending, signal); + } catch (error) { + if (!(error instanceof InstallationChangedError) || retried || !repository) { + throw error; + } + // A shared mint can serve several repositories. Never pass its replacement + // installation's token to a repository whose grant has not been checked. + if (this.installationIds.get(repository) === error.installationId) { + this.installationIds.delete(repository); + this.installationIdCachedAt.delete(repository); + } + return this.getCredentialAttempt(signal, repository, true); } - const body = (await response.json()) as { id?: unknown }; - if (!Number.isSafeInteger(body.id) || Number(body.id) <= 0) { - throw new Error('GitHub App installation response is invalid'); - } - const installationId = String(body.id); - this.installationIds.set(repository, installationId); - return installationId; } async getCredential( @@ -462,32 +525,46 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { ); } if (repository) repositoryName(repository); + return this.getCredentialAttempt(signal, repository, false); + } + + private async getCredentialAttempt( + signal: AbortSignal | undefined, + repository: string | undefined, + retried: boolean, + ): Promise { + signal?.throwIfAborted(); + const installationScope = this.options.tokenScope === 'installation'; + const installation = installationScope + ? await this.resolveInstallationId(repository ?? '', signal) + : undefined; const now = (this.options.now ?? (() => new Date()))(); - const key = this.options.installationId ?? repository!; + const key = installation?.id ?? this.options.installationId ?? repository!; const cached = this.cached.get(key); if ( cached?.expiresAt != null && - cached.expiresAt.getTime() - now.getTime() > 5 * 60_000 + cached.expiresAt.getTime() - now.getTime() > 5 * 60_000 && + (!installationScope || now.getTime() - (this.cachedAt.get(key) ?? 0) < GITHUB_INSTALLATION_CACHE_MS) ) { return cached; } const existing = this.inFlight.get(key); - if (existing) return waitForShared(existing, signal); + if (existing) return this.waitForCredential(existing, signal, repository, retried); const pending = (async () => { const sharedSignal = AbortSignal.timeout( GITHUB_SHARED_REQUEST_TIMEOUT_MS, ); - const jwt = await this.appJwt(now); - const scopedRepository = repository + const jwt = installation?.jwt ?? await this.appJwt(now); + const scopedRepository = !installationScope && repository ? repositoryName(repository).name : undefined; - const installationId = await this.resolveInstallationId( + const resolvedInstallationId = installation?.id ?? (await this.resolveInstallationId( repository ?? '', - jwt, sharedSignal, - ); + jwt, + )).id; let response = await this.request( - `/app/installations/${installationId}/access_tokens`, + `/app/installations/${resolvedInstallationId}/access_tokens`, jwt, sharedSignal, { @@ -495,18 +572,20 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { headers: { 'Content-Type': 'application/json', }, - ...(this.options.installationId + ...(this.options.installationId || installationScope ? {} : { body: JSON.stringify({ repositories: [scopedRepository] }) }), }, ); if (!this.options.installationId && response.status === 404) { + if (installationScope) throw new InstallationChangedError(resolvedInstallationId); this.installationIds.delete(repository!); - const refreshedInstallationId = await this.resolveInstallationId( + this.installationIdCachedAt.delete(repository!); + const refreshedInstallationId = (await this.resolveInstallationId( repository!, - jwt, sharedSignal, - ); + jwt, + )).id; response = await this.request( `/app/installations/${refreshedInstallationId}/access_tokens`, jwt, @@ -516,7 +595,9 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { headers: { 'Content-Type': 'application/json', }, - body: JSON.stringify({ repositories: [scopedRepository] }), + ...(installationScope + ? {} + : { body: JSON.stringify({ repositories: [scopedRepository] }) }), }, ); } @@ -554,6 +635,10 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { actor, }; this.cached.set(key, credential); + this.cachedAt.set( + key, + (this.options.now ?? (() => new Date()))().getTime(), + ); return credential; })(); this.inFlight.set(key, pending); @@ -561,7 +646,7 @@ export class GitHubAppCredentialProvider implements GitHubCredentialProvider { if (this.inFlight.get(key) === pending) this.inFlight.delete(key); }; void pending.then(clearPending, clearPending); - return waitForShared(pending, signal); + return this.waitForCredential(pending, signal, repository, retried); } } diff --git a/packages/code/src/identity.test.ts b/packages/code/src/identity.test.ts index 4e92b91f..bef3b673 100644 --- a/packages/code/src/identity.test.ts +++ b/packages/code/src/identity.test.ts @@ -1,9 +1,14 @@ import assert from 'node:assert/strict'; +import { randomBytes } from 'node:crypto'; import test from 'node:test'; import { createBridgeIdentity, + signBridgeRecovery, + signBridgeRecoveryStart, signBridgeRequest, + verifyBridgeRecovery, + verifyBridgeRecoveryStart, verifyBridgeRequest, } from './identity.js'; @@ -33,3 +38,41 @@ test('worker identity proves possession for the exact HTTP request', () => { false, ); }); + +test('signed recovery starts bind operation, server, worker, time and nonce', () => { + const identity = createBridgeIdentity(); + const start = { + operation: 'credential.challenge' as const, + serverId: 'https://code.example.test', + workerId: 'vm-1', + timestamp: new Date().toISOString(), + nonce: randomBytes(32).toString('base64url'), + }; + const signature = signBridgeRecoveryStart(identity.privateKey, start); + assert.equal(verifyBridgeRecoveryStart(identity.publicKey, start, signature), true); + for (const modified of [ + { ...start, operation: 'credential.recover' as 'credential.challenge' }, + { ...start, serverId: 'https://other.example.test' }, + { ...start, workerId: 'vm-2' }, + { ...start, timestamp: new Date(Date.now() + 60_000).toISOString() }, + { ...start, nonce: randomBytes(32).toString('base64url') }, + ]) { + assert.equal(verifyBridgeRecoveryStart(identity.publicKey, modified, signature), false); + } + + const challenge = { + operation: 'credential.recover' as const, + serverId: start.serverId, + workerId: start.workerId, + enrollmentGeneration: randomBytes(18).toString('base64url'), + challenge: randomBytes(32).toString('base64url'), + expiresAt: new Date(Date.now() + 60_000).toISOString(), + }; + assert.equal(verifyBridgeRecovery(identity.publicKey, challenge, signature), false); + assert.equal( + verifyBridgeRecoveryStart( + identity.publicKey, start, signBridgeRecovery(identity.privateKey, challenge), + ), + false, + ); +}); diff --git a/packages/code/src/identity.ts b/packages/code/src/identity.ts index ab11c865..3556520c 100644 --- a/packages/code/src/identity.ts +++ b/packages/code/src/identity.ts @@ -19,6 +19,25 @@ export interface BridgeRequestProofInput { body: string; } +/** Signed without an access credential before requesting a server recovery challenge. */ +export interface BridgeRecoveryStartProofInput { + operation: 'credential.challenge'; + serverId: string; + workerId: string; + timestamp: string; + nonce: string; +} + +/** Signed separately from access-credential requests, so neither proof can be reused for the other. */ +export interface BridgeRecoveryProofInput { + operation: 'credential.recover'; + serverId: string; + workerId: string; + enrollmentGeneration: string; + challenge: string; + expiresAt: string; +} + export function createBridgeIdentity(): BridgeIdentity { const { publicKey, privateKey } = generateKeyPairSync('ed25519', { publicKeyEncoding: { type: 'spki', format: 'pem' }, @@ -64,3 +83,78 @@ export function verifyBridgeRequest( return false; } } + +function canonicalBridgeRecoveryStart(input: BridgeRecoveryStartProofInput): string { + return [ + 'librechat-code:bridge-recovery-start:v1', + input.operation, + input.serverId, + input.workerId, + input.timestamp, + input.nonce, + ].join('\n'); +} + +export function signBridgeRecoveryStart( + privateKey: string, + input: BridgeRecoveryStartProofInput, +): string { + return sign(null, Buffer.from(canonicalBridgeRecoveryStart(input)), privateKey).toString( + 'base64url', + ); +} + +export function verifyBridgeRecoveryStart( + publicKey: string, + input: BridgeRecoveryStartProofInput, + signature: string, +): boolean { + try { + return verify( + null, + Buffer.from(canonicalBridgeRecoveryStart(input)), + publicKey, + Buffer.from(signature, 'base64url'), + ); + } catch { + return false; + } +} + +function canonicalBridgeRecovery(input: BridgeRecoveryProofInput): string { + return [ + 'librechat-code:bridge-recovery:v1', + input.operation, + input.serverId, + input.workerId, + input.enrollmentGeneration, + input.challenge, + input.expiresAt, + ].join('\n'); +} + +export function signBridgeRecovery( + privateKey: string, + input: BridgeRecoveryProofInput, +): string { + return sign(null, Buffer.from(canonicalBridgeRecovery(input)), privateKey).toString( + 'base64url', + ); +} + +export function verifyBridgeRecovery( + publicKey: string, + input: BridgeRecoveryProofInput, + signature: string, +): boolean { + try { + return verify( + null, + Buffer.from(canonicalBridgeRecovery(input)), + publicKey, + Buffer.from(signature, 'base64url'), + ); + } catch { + return false; + } +} diff --git a/packages/code/src/lane-git.test.ts b/packages/code/src/lane-git.test.ts new file mode 100644 index 00000000..27b400b5 --- /dev/null +++ b/packages/code/src/lane-git.test.ts @@ -0,0 +1,830 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, realpath, rename, rm, symlink, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { promisify } from 'node:util'; + +import { LaneGitWorkspaceTools, ownsGitMetadata, ownsLinkedWorktreeMetadata, readLaneGit, snapshotLaneGit } from './lane-git.js'; +import { isValidBridgeWorkspaceToolCapabilities, isWorkspaceLaneGit, isWorkspaceToolResult } from './protocol.js'; +import { BridgeWorker } from './worker.js'; + +import type { TestContext } from 'node:test'; +import type { WorkspaceExecuteCommandRequest, WorkspaceToolRequest } from './protocol.js'; + +const exec = promisify(execFile); +const incarnationId = 'incarnation-00000001'; +const identity = ['-c', 'user.name=Test', '-c', 'user.email=test@example.com']; + +async function scratch(t: TestContext): Promise { + const root = await realpath(await mkdtemp(join(tmpdir(), 'lane-git-'))); + t.after(() => rm(root, { recursive: true, force: true })); + return root; +} + +async function repo(root: string, name: string, commit = true): Promise { + const directory = join(root, name); + await mkdir(directory, { recursive: true }); + await exec('git', ['init', '--initial-branch=main', directory]); + if (commit) await exec('git', ['-C', directory, ...identity, 'commit', '--allow-empty', '-m', 'initial']); + return directory; +} + +async function sha(directory: string): Promise { + return (await exec('git', ['-C', directory, 'rev-parse', 'HEAD'])).stdout.trim(); +} + +test('isolated worktree lane reports its own branch and head', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const lane = join(root, 'lane'); + await exec('git', ['-C', source, 'worktree', 'add', '-b', 'librechat/conversation-abcd1234-0123', lane]); + assert.deepEqual(await readLaneGit(lane), { branch: 'librechat/conversation-abcd1234-0123', head: await sha(lane) }); + assert.equal((await readLaneGit(source))?.branch, 'main'); +}); + +test('source checkout reports whatever is checked out', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', 'feature/pr-header']); + assert.deepEqual(await readLaneGit(source), { branch: 'feature/pr-header', head: await sha(source) }); +}); + +test('a branch renamed by the agent shows up on the next read', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + assert.equal((await readLaneGit(source))?.branch, 'main'); + await exec('git', ['-C', source, 'branch', '-m', 'renamed-by-agent']); + assert.equal((await readLaneGit(source))?.branch, 'renamed-by-agent'); +}); + +test('detached HEAD reports a null branch and the commit', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '--detach']); + assert.deepEqual(await readLaneGit(source), { branch: null, head: await sha(source) }); +}); + +test('a repository without commits reports the branch and a null head', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source', false); + assert.deepEqual(await readLaneGit(source), { branch: 'main', head: null }); +}); + +test('a branch name beyond the bound reports null and never forwards it', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const long = ['a'.repeat(100), 'b'.repeat(100), 'c'.repeat(100)].join('/'); + await exec('git', ['-C', source, 'checkout', '-b', long]); + assert.deepEqual(await readLaneGit(source), { branch: null, head: await sha(source) }); +}); + +test('a directory that is not a repository reports nothing', async (t) => { + const root = await scratch(t); + assert.equal(await readLaneGit(root), undefined); +}); + +test('lane Git state never carries paths, remotes or credentials', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'remote', 'add', 'origin', 'https://user:secret@github.com/example/app.git']); + const state = await readLaneGit(source); + assert.deepEqual(Object.keys(state ?? {}).sort(), ['branch', 'head']); + const wire = JSON.stringify(state); + assert.equal(wire.includes(root), false); + assert.equal(wire.includes('secret'), false); + assert.equal(wire.includes('github.com'), false); +}); + +test('wire validator accepts only bounded branch and head', () => { + const head = 'a'.repeat(40); + assert.equal(isWorkspaceLaneGit({ branch: 'main', head }), true); + assert.equal(isWorkspaceLaneGit({ branch: null, head: null }), true); + assert.equal(isWorkspaceLaneGit({ branch: 'main', head: 'b'.repeat(64) }), true); + for (const bad of [ + { branch: 'x'.repeat(257), head }, + { branch: 'bad\nname', head }, + { branch: '', head }, + { branch: 'main', head: 'A'.repeat(40) }, + { branch: 'main', head: 'a'.repeat(41) }, + { branch: 'main' }, + { branch: 'main', head, path: '/srv/repo' }, + { branch: undefined, head }, + null, + [], + ]) { + assert.equal(isWorkspaceLaneGit(bad), false, JSON.stringify(bad)); + } +}); + +const commandRequest: WorkspaceExecuteCommandRequest = { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'primary', + command: 'git checkout -b agent-branch', +}; +const commandResult = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, +}; + +function delegate(onRun?: () => Promise) { + return { + capabilities: { + protocolVersion: 1 as const, + operations: ['read_file' as const, 'execute_command' as const], + workspaces: [{ id: 'primary' }], + }, + async execute(request: WorkspaceToolRequest) { + await onRun?.(); + return request.operation === 'execute_command' + ? commandResult + : { protocolVersion: 1 as const, operation: 'read_file' as const, workspaceId: 'primary', path: 'a', content: '', startLine: 1, endLine: 1, truncated: false }; + }, + }; +} + +test('a finished command refreshes the lane state, including a branch the command created', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(async () => { + await exec('git', ['-C', source, 'checkout', '-b', 'agent-branch']); + }), + resolveRoot: async () => source, + }); + const result = await tools.execute(commandRequest); + assert.deepEqual('laneGit' in result && result.laneGit, { branch: 'agent-branch', head: await sha(source) }); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); +}); + +test('file operations do not read Git', async () => { + let resolved = 0; + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + resolveRoot: async () => { + resolved++; + return undefined; + }, + }); + const result = await tools.execute({ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'a' }); + assert.equal('laneGit' in result, false); + assert.equal(resolved, 0); +}); + +test('an unreadable lane leaves the command result untouched', async (t) => { + const root = await scratch(t); + for (const resolveRoot of [async () => root, async () => undefined, async () => Promise.reject(new Error('gone'))]) { + const result = await new LaneGitWorkspaceTools({ delegate: delegate(), resolveRoot }).execute(commandRequest); + assert.deepEqual(result, commandResult); + } +}); + +test('the feature is advertised only when commands are', () => { + const withCommands = new LaneGitWorkspaceTools({ delegate: delegate(), resolveRoot: async () => undefined }); + assert.deepEqual(withCommands.capabilities.commandResultFeatures, ['lane_git']); + assert.equal(isValidBridgeWorkspaceToolCapabilities(withCommands.capabilities), true); + const readOnly = new LaneGitWorkspaceTools({ + delegate: { ...delegate(), capabilities: { protocolVersion: 1, operations: ['read_file'], workspaces: [{ id: 'primary' }] } }, + resolveRoot: async () => undefined, + }); + assert.equal(readOnly.capabilities.commandResultFeatures, undefined); + assert.equal( + isValidBridgeWorkspaceToolCapabilities({ ...withCommands.capabilities, commandResultFeatures: ['other'] }), + false, + ); +}); + +test('the probe is skipped until lane_git is negotiated, then runs', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + let negotiated = false; + let resolved = 0; + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + isEnabled: () => negotiated, + resolveRoot: async () => { + resolved++; + return source; + }, + }); + const before = await tools.execute(commandRequest); + assert.equal('laneGit' in before, false); + assert.equal(resolved, 0); + negotiated = true; + const after = await tools.execute(commandRequest); + assert.equal(resolved, 1); + assert.deepEqual('laneGit' in after && after.laneGit, { branch: 'main', head: await sha(source) }); +}); + +test('the probe is skipped when the command left too little of its settlement budget', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const timed = { ...commandRequest, timeoutMs: 1_000 }; + // Budget is timeoutMs plus a 5 s grace (6 s here), minus a 1 s settlement reserve; the clock jumps while the command runs. + for (const [elapsedMs, probed] of [ + [100, true], + [4_000, true], + [4_800, false], + [7_000, false], + ] as const) { + let clock = 0; + let resolved = 0; + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(async () => { + clock += elapsedMs; + }), + now: () => clock, + resolveRoot: async () => { + resolved++; + return source; + }, + }); + const result = await tools.execute(timed); + assert.equal(resolved, probed ? 1 : 0, `elapsed ${elapsedMs}`); + assert.equal('laneGit' in result, probed, `elapsed ${elapsedMs}`); + } +}); + +test('the probe budget follows the worker deadline, not a rebuilt timeout', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + // A full 31 s budget by timeout alone, but the worker has only 1.5 s left before it aborts. + for (const [remainingMs, probed] of [ + [10_000, true], + [1_500, true], + [1_200, false], + [500, false], + ] as const) { + let resolved = 0; + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + now: () => 1_000, + resolveRoot: async () => { + resolved++; + return source; + }, + }); + await tools.execute(commandRequest, undefined, { deadlineAtMs: 1_000 + remainingMs }); + assert.equal(resolved, probed ? 1 : 0, `remaining ${remainingMs}`); + } +}); + +test('a slow probe is cut short and never fails the command', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + probeTimeoutMs: 1, + resolveRoot: async (_request, signal) => { + await new Promise((resolve) => setTimeout(resolve, 50)); + signal?.throwIfAborted(); + return source; + }, + }); + assert.deepEqual(await tools.execute(commandRequest), commandResult); +}); + +test('a HEAD that names a blob is reported as a null head, not as a commit', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const id = (await exec('bash', ['-c', `echo hello | git -C '${source}' hash-object -w --stdin`])).stdout.trim(); + await writeFile(join(source, '.git', 'HEAD'), `${id}\n`); + assert.deepEqual(await readLaneGit(source), { branch: null, head: null }); +}); + +test('branch names with Unicode control or format characters are not reported', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', 'feature/\u202eabc']); + assert.deepEqual(await readLaneGit(source), { branch: null, head: await sha(source) }); + for (const name of ['a\u0085b', 'a\u202eb', 'a\u200bb', 'a\u2028b']) { + assert.equal(isWorkspaceLaneGit({ branch: name, head: null }), false, JSON.stringify(name)); + } + assert.equal(isWorkspaceLaneGit({ branch: 'feature/caf\u00e9-\u65e5\u672c', head: null }), true); +}); + +test('a resolver that ignores its signal cannot hold the command result past the probe limit', async () => { + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + probeTimeoutMs: 50, + resolveRoot: () => new Promise(() => {}), + }); + const started = Date.now(); + const result = await Promise.race([ + tools.execute(commandRequest), + new Promise<'hung'>((resolve) => setTimeout(() => resolve('hung'), 2_000)), + ]); + assert.notEqual(result, 'hung'); + assert.deepEqual(result, commandResult); + assert.ok(Date.now() - started < 1_000); +}); + +test('the worker reports lane_git active only when the registration that stuck advertised it', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + for (const secondFails of [false, true]) { + let registrations = 0; + const tools = new LaneGitWorkspaceTools({ delegate: delegate(), resolveRoot: async () => source }); + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: tools.capabilities }, + workspaceTools: tools, + workspaceMutationQuarantine: quarantine(), + fetchImpl: async () => { + registrations++; + if (secondFails && registrations === 2) return new Response('{}', { status: 500 }); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'execute_command'], + supportedWorkspaceCommandResultFeatures: ['lane_git'], + }); + }, + }); + await worker.register(); + assert.equal(worker.commandResultFeatureActive('lane_git'), !secondFails, `secondFails=${secondFails}`); + } +}); + +test('a branch name with leading or trailing Unicode whitespace is reported as it is', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', '\u00a0main\u00a0']); + assert.deepEqual(await readLaneGit(source), { branch: '\u00a0main\u00a0', head: await sha(source) }); +}); + +test('a branch name that is not valid UTF-8 is reported as null, not as a lossy copy', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await writeFile(join(source, '.git', 'HEAD'), Buffer.concat([Buffer.from('ref: refs/heads/bad'), Buffer.from([0xff, 0x0a])])); + const state = await readLaneGit(source); + assert.equal(state?.branch, null); + assert.equal(JSON.stringify(state).includes('\ufffd'), false); +}); + +test('a valid non-ASCII branch name survives the byte level read', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', 'feature/caf\u00e9-\u65e5\u672c']); + assert.equal((await readLaneGit(source))?.branch, 'feature/caf\u00e9-\u65e5\u672c'); +}); + +test('an own Git directory is accepted, and a commit made there keeps it accepted', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + assert.equal(await ownsGitMetadata(source), true); + await exec('git', ['-C', source, ...identity, 'commit', '--allow-empty', '-m', 'second']); + await exec('git', ['-C', source, 'pack-refs', '--all']); + assert.equal(await ownsGitMetadata(source), true); +}); + +test('a .git rewritten to point at another repository is not probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const victim = await repo(root, 'victim'); + await exec('git', ['-C', victim, 'checkout', '-b', 'victim-secret-branch']); + await rm(join(source, '.git'), { recursive: true }); + await writeFile(join(source, '.git'), `gitdir: ${join(victim, '.git')}\n`); + assert.equal(await ownsGitMetadata(source), false); + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + resolveRoot: async () => ((await ownsGitMetadata(source)) ? source : undefined), + }); + const result = await tools.execute(commandRequest); + assert.equal('laneGit' in result, false); + assert.equal(JSON.stringify(result).includes('victim-secret-branch'), false); +}); + +test('Git storage redirected by a symlink, commondir or a swapped directory is not probed', async (t) => { + const root = await scratch(t); + const victim = await repo(root, 'victim'); + const cases: Array<[string, (source: string) => Promise]> = [ + ['.git symlink', async (s) => { await rm(join(s, '.git'), { recursive: true }); await symlink(join(victim, '.git'), join(s, '.git')); }], + ['commondir', async (s) => { await writeFile(join(s, '.git', 'commondir'), `${join(victim, '.git')}\n`); }], + ['HEAD symlink', async (s) => { await rm(join(s, '.git', 'HEAD')); await symlink(join(victim, '.git', 'HEAD'), join(s, '.git', 'HEAD')); }], + ['refs symlink', async (s) => { await rm(join(s, '.git', 'refs'), { recursive: true }); await symlink(join(victim, '.git', 'refs'), join(s, '.git', 'refs')); }], + ['packed-refs symlink', async (s) => { await symlink(join(victim, '.git', 'HEAD'), join(s, '.git', 'packed-refs')); }], + ['no .git', async (s) => { await rename(join(s, '.git'), join(root, `moved-${Math.random()}`)); }], + ]; + for (const [name, tamper] of cases) { + const source = await repo(root, `source-${name.replace(/\W/g, '')}`); + await tamper(source); + assert.equal(await ownsGitMetadata(source), false, name); + } +}); + +test('a nested ref symlinked to another checkout is not probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const firstCommit = await sha(source); + await exec('git', ['-C', source, ...identity, 'commit', '--allow-empty', '-m', 'second']); + // A clone shares history, so the commit its branch sits on also exists in the source's objects. + const other = join(root, 'other'); + await exec('git', ['clone', '-q', source, other]); + await exec('git', ['-C', other, 'checkout', '-q', '-b', 'other-wip', firstCommit]); + await symlink(join(other, '.git', 'refs', 'heads', 'other-wip'), join(source, '.git', 'refs', 'heads', 'leak')); + await writeFile(join(source, '.git', 'HEAD'), 'ref: refs/heads/leak\n'); + assert.equal(await ownsGitMetadata(source), false); + const tools = new LaneGitWorkspaceTools({ + delegate: delegate(), + resolveRoot: async () => ((await ownsGitMetadata(source)) ? source : undefined), + }); + const result = await tools.execute(commandRequest); + assert.equal('laneGit' in result, false); +}); + +test('a nested branch ref that is a plain file inside .git is still probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', 'feature/deep/name']); + assert.equal(await ownsGitMetadata(source), true); + assert.equal((await readLaneGit(source))?.branch, 'feature/deep/name'); +}); + +test('a branch that shares its short name with a tag is reported by its real name', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'checkout', '-b', 'foo']); + await exec('git', ['-C', source, 'tag', 'foo']); + assert.equal((await readLaneGit(source))?.branch, 'foo'); +}); + +test('a HEAD that points outside refs/heads reports a null branch', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, 'update-ref', 'refs/remotes/origin/main', 'HEAD']); + await exec('git', ['-C', source, 'symbolic-ref', 'HEAD', 'refs/remotes/origin/main']); + assert.equal((await readLaneGit(source))?.branch, null); +}); + +test('a linked lane whose HEAD names a ref symlinked at another checkout is not probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const firstCommit = await sha(source); + await exec('git', ['-C', source, ...identity, 'commit', '--allow-empty', '-m', 'second']); + const other = join(root, 'other'); + await exec('git', ['clone', '-q', source, other]); + await exec('git', ['-C', other, 'checkout', '-q', '-b', 'other-wip', firstCommit]); + const lane = join(root, 'lane'); + await exec('git', ['-C', source, 'worktree', 'add', '-b', 'lane-branch', lane]); + const common = join(source, '.git'); + assert.equal(await ownsLinkedWorktreeMetadata(common, 'lane'), true); + await symlink(join(other, '.git', 'refs', 'heads', 'other-wip'), join(common, 'refs', 'heads', 'leak')); + await writeFile(join(common, 'worktrees', 'lane', 'HEAD'), 'ref: refs/heads/leak\n'); + assert.equal(await ownsLinkedWorktreeMetadata(common, 'lane'), false); +}); + +test('a linked lane with a symlinked metadata HEAD is not probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const lane = join(root, 'lane'); + await exec('git', ['-C', source, 'worktree', 'add', '-b', 'lane-branch', lane]); + const head = join(source, '.git', 'worktrees', 'lane', 'HEAD'); + await rm(head); + await symlink(join(source, '.git', 'HEAD'), head); + assert.equal(await ownsLinkedWorktreeMetadata(join(source, '.git'), 'lane'), false); +}); + +test('the refs walk stops as soon as the probe signal aborts', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + for (let i = 0; i < 30; i++) await mkdir(join(source, '.git', 'refs', 'heads', `dir${i}`), { recursive: true }); + let checks = 0; + const aborting = { + aborted: false, + throwIfAborted() { + if (++checks >= 3) throw new Error('aborted'); + }, + } as unknown as AbortSignal; + assert.equal(await ownsGitMetadata(source, aborting), false); + assert.equal(checks, 3); + const linked = join(root, 'lane'); + await exec('git', ['-C', source, 'worktree', 'add', '-b', 'lane-branch', linked]); + checks = 0; + assert.equal(await ownsLinkedWorktreeMetadata(join(source, '.git'), 'lane', aborting), false); + assert.equal(checks, 3); +}); + +test('the head is read from the branch ref the first read named, and only a stable branch is reported', async () => { + const head = 'a'.repeat(40); + const sequence = (values: Array) => { + let index = 0; + return async () => values[index++]; + }; + const refsRead: string[] = []; + const commit = (answer: string | null | undefined) => async (ref: string) => { + refsRead.push(ref); + return answer; + }; + // Stable branch: head comes from that branch's own ref, never from HEAD. + assert.deepEqual(await snapshotLaneGit(sequence(['refs/heads/a', 'refs/heads/a']), commit(head)), { branch: 'a', head }); + assert.deepEqual(refsRead, ['refs/heads/a']); + // A checkout away during the read is rejected. + assert.equal(await snapshotLaneGit(sequence(['refs/heads/a', 'refs/heads/b']), commit(head)), undefined); + assert.equal(await snapshotLaneGit(sequence(['refs/heads/a', null]), commit(head)), undefined); + assert.equal(await snapshotLaneGit(sequence([null, 'refs/heads/a']), commit(head)), undefined); + // Detached: HEAD is read twice and must agree, so a move between the reads is rejected. + refsRead.length = 0; + let calls = 0; + const moving = async (ref: string) => { + refsRead.push(ref); + return calls++ === 0 ? head : 'b'.repeat(40); + }; + assert.equal(await snapshotLaneGit(sequence([null, null]), moving), undefined); + assert.deepEqual(refsRead, ['HEAD', 'HEAD']); + assert.deepEqual(await snapshotLaneGit(sequence([null, null]), commit(head)), { branch: null, head }); + // HEAD on a non-branch ref reports no branch but still a head. + assert.deepEqual(await snapshotLaneGit(sequence(['refs/remotes/o/m', 'refs/remotes/o/m']), commit(head)), { branch: null, head }); + // Unreadable values omit the field. + assert.equal(await snapshotLaneGit(sequence(['refs/heads/a', 'refs/heads/a']), commit(undefined)), undefined); +}); + +test('a replace ref cannot turn a blob into a reported commit', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const commit = await sha(source); + const blob = (await exec('bash', ['-c', `echo hello | git -C '${source}' hash-object -w --stdin`])).stdout.trim(); + await mkdir(join(source, '.git', 'refs', 'replace'), { recursive: true }); + await writeFile(join(source, '.git', 'refs', 'replace', blob), `${commit}\n`); + await writeFile(join(source, '.git', 'refs', 'heads', 'main'), `${blob}\n`); + assert.deepEqual(await readLaneGit(source), { branch: 'main', head: null }); +}); + +test('redirected object storage is not probed, for source and linked checks alike', async (t) => { + const root = await scratch(t); + const victim = await repo(root, 'victim'); + const cases: Array<[string, (source: string) => Promise]> = [ + ['alternates', async (s) => { + await mkdir(join(s, '.git', 'objects', 'info'), { recursive: true }); + await writeFile(join(s, '.git', 'objects', 'info', 'alternates'), `${join(victim, '.git', 'objects')}\n`); + }], + ['http-alternates', async (s) => { + await mkdir(join(s, '.git', 'objects', 'info'), { recursive: true }); + await writeFile(join(s, '.git', 'objects', 'info', 'http-alternates'), 'https://example.invalid/\n'); + }], + ['objects symlink', async (s) => { + await rename(join(s, '.git', 'objects'), join(root, `moved-${Math.random()}`)); + await symlink(join(victim, '.git', 'objects'), join(s, '.git', 'objects')); + }], + ['fanout symlink', async (s) => { + await symlink(join(victim, '.git', 'objects', 'info'), join(s, '.git', 'objects', 'ab')); + }], + ['pack entry symlink', async (s) => { + await mkdir(join(s, '.git', 'objects', 'pack'), { recursive: true }); + await symlink(join(victim, '.git', 'HEAD'), join(s, '.git', 'objects', 'pack', 'pack-x.pack')); + }], + ['reftable redirect', async (s) => { + await mkdir(join(s, '.git', 'reftable'), { recursive: true }); + }], + ]; + for (const [name, tamper] of cases) { + const source = await repo(root, `src-${name.replace(/\W/g, '')}`); + await tamper(source); + assert.equal(await ownsGitMetadata(source), false, `source ${name}`); + } + const common = join(root, 'common'); + await exec('git', ['init', '-q', '--initial-branch=main', common]); + await exec('git', ['-C', common, ...identity, 'commit', '--allow-empty', '-m', 'i']); + await exec('git', ['-C', common, 'worktree', 'add', '-b', 'lane', join(root, 'lane')]); + assert.equal(await ownsLinkedWorktreeMetadata(join(common, '.git'), 'lane'), true); + await mkdir(join(common, '.git', 'objects', 'info'), { recursive: true }); + await writeFile(join(common, '.git', 'objects', 'info', 'alternates'), `${join(victim, '.git', 'objects')}\n`); + assert.equal(await ownsLinkedWorktreeMetadata(join(common, '.git'), 'lane'), false); +}); + +test('a repository with packed objects and refs is still probed', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + await exec('git', ['-C', source, ...identity, 'commit', '--allow-empty', '-m', 'second']); + await exec('git', ['-C', source, 'gc', '-q']); + assert.equal(await ownsGitMetadata(source), true); + assert.equal((await readLaneGit(source))?.head, await sha(source)); +}); + +test('a promisor remote cannot make the probe fetch or run a command on the host', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const victim = await repo(root, 'victim'); + await exec('git', ['-C', victim, ...identity, 'commit', '--allow-empty', '-m', 'secret']); + const secret = await sha(victim); + const marker = join(root, 'ran-on-host'); + const config = [ + '[remote "origin"]', + `\turl = ${victim}`, + '\tpromisor = true', + '\tpartialclonefilter = blob:none', + `\tuploadpack = touch ${marker}; git-upload-pack`, + '[extensions]', + '\tpartialclone = origin', + '[core]', + '\trepositoryformatversion = 1', + '[protocol]', + '\tallow = always', + '[protocol "file"]', + '\tallow = always', + '', + ].join('\n'); + await writeFile(join(source, '.git', 'config'), config, { flag: 'a' }); + await writeFile(join(source, '.git', 'refs', 'heads', 'main'), `${secret}\n`); + const state = await readLaneGit(source); + assert.notEqual(state?.head, secret); + await assert.rejects(readFile(marker), { code: 'ENOENT' }); +}); + +test('a linked worktree .git file is not claimed by the source checkout check', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const lane = join(root, 'lane'); + await exec('git', ['-C', source, 'worktree', 'add', '-b', 'lane-branch', lane]); + assert.equal(await ownsGitMetadata(lane), false); + assert.equal((await readLaneGit(lane))?.branch, 'lane-branch'); +}); + +test('a laneGit supplied by the delegate is never forwarded, only the probe value is', async (t) => { + const root = await scratch(t); + const source = await repo(root, 'source'); + const forged = { branch: 'forged', head: 'f'.repeat(40) }; + const forging = { + ...delegate(), + async execute() { + return { ...commandResult, laneGit: forged }; + }, + }; + const unreadable = await new LaneGitWorkspaceTools({ delegate: forging, resolveRoot: async () => undefined }).execute(commandRequest); + assert.equal('laneGit' in unreadable, false); + const probed = await new LaneGitWorkspaceTools({ delegate: forging, resolveRoot: async () => source }).execute(commandRequest); + assert.deepEqual('laneGit' in probed && probed.laneGit, { branch: 'main', head: await sha(source) }); +}); + +test('the worker passes its real assignment deadline to the executor', async () => { + const seen: Array = []; + const base = delegate(); + const spying = { + ...base, + async execute(request: WorkspaceToolRequest, signal?: AbortSignal, context?: { deadlineAtMs?: number }) { + seen.push(context?.deadlineAtMs); + return base.execute(request); + }, + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: spying.capabilities }, + workspaceTools: spying, + workspaceMutationQuarantine: quarantine(), + fetchImpl: async () => + Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'execute_command'], + }), + }); + await worker.register(); + const before = Date.now(); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-1', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + remainingMs: 2_000, + executionKind: 'workspace_tool', + workspaceId: 'primary', + request: commandRequest, + }); + assert.equal(seen.length, 1); + const deadline = seen[0] as number; + assert.ok(deadline >= before + 1_500 && deadline <= Date.now() + 2_000, `deadline ${deadline - before}`); +}); + +// Compatibility: negotiation with old and new Code API servers. + +function quarantine() { + return { async assertAvailable() {}, async arm() {}, async clear() {}, async quarantine() {} }; +} + +async function run(t: TestContext, supported: boolean) { + const root = await scratch(t); + const source = await repo(root, 'source'); + const tools = new LaneGitWorkspaceTools({ delegate: delegate(), resolveRoot: async () => source }); + const registrations: Array<{ commandResultFeatures?: string[] }> = []; + const settlements: Array<{ result?: Record }> = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: tools.capabilities }, + workspaceTools: tools, + workspaceMutationQuarantine: quarantine(), + fetchImpl: async (input, init) => { + const url = String(input); + const body = init?.body ? JSON.parse(String(init.body)) : {}; + if (url.endsWith('/bridge/workers/register')) { + registrations.push(body.capabilities.workspaceTools ?? {}); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'execute_command'], + ...(supported ? { supportedWorkspaceCommandResultFeatures: ['lane_git'] } : {}), + }); + } + if (url.endsWith('/settle')) settlements.push(body); + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + await worker.register(); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-1', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + workspaceId: 'primary', + request: commandRequest, + }); + return { registrations, settlements }; +} + +test('a Code API without the feature still pairs, registers without it, and never receives laneGit', async (t) => { + const { registrations, settlements } = await run(t, false); + assert.ok(registrations.length > 0); + for (const registration of registrations) assert.equal('commandResultFeatures' in registration, false); + assert.equal(settlements.length, 1); + assert.equal(settlements[0]?.result?.exitCode, 0); + assert.equal('laneGit' in (settlements[0]?.result ?? {}), false); +}); + +test('a Code API that acknowledges the feature registers it and receives laneGit', async (t) => { + const { registrations, settlements } = await run(t, true); + assert.deepEqual(registrations.at(-1)?.commandResultFeatures, ['lane_git']); + assert.deepEqual(Object.keys((settlements[0]?.result?.laneGit as object) ?? {}).sort(), ['branch', 'head']); + assert.equal((settlements[0]?.result?.laneGit as { branch: string }).branch, 'main'); +}); + +test('a worker without the field still pairs and reports status', async () => { + const executor = { + capabilities: { protocolVersion: 1 as const, operations: ['read_file' as const], workspaces: [{ id: 'primary' }] }, + async execute(): Promise { + throw new Error('not executed'); + }, + }; + const registrations: unknown[] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: false, sandboxProfile: 'anthropic-srt', runtimes: [], workspaceTools: executor.capabilities }, + workspaceTools: executor, + fetchImpl: async (_input, init) => { + registrations.push(JSON.parse(String(init?.body)).capabilities.workspaceTools); + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file'], + }); + }, + }); + const registration = await worker.register(); + assert.equal(registration.workerId, 'vm-1'); + assert.ok(registrations.length > 0); + for (const advertised of registrations) { + assert.equal(JSON.stringify(advertised).includes('commandResultFeatures'), false); + assert.equal(JSON.stringify(advertised).includes('laneGit'), false); + } +}); diff --git a/packages/code/src/lane-git.ts b/packages/code/src/lane-git.ts new file mode 100644 index 00000000..f7aa3122 --- /dev/null +++ b/packages/code/src/lane-git.ts @@ -0,0 +1,297 @@ +import { lstat, opendir, realpath } from 'node:fs/promises'; +import { join } from 'node:path'; + +import { BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, boundedBranch, boundedHead } from './protocol.js'; +import { gitBytes } from './worktree-retirement.js'; + +import type { WorkspaceLaneGit, WorkspaceToolRequest, WorkspaceToolResult } from './protocol.js'; +import type { WorkspaceToolExecutor } from './workspace.js'; + +/** Longest the advisory probe may run, however much budget is left. */ +const LANE_GIT_PROBE_MAX_MS = 1_000; +/** Settlement budget the probe never touches, for validation and the settle request. */ +const LANE_GIT_SETTLEMENT_RESERVE_MS = 1_000; +/** Below this, a probe is not worth starting. */ +const LANE_GIT_PROBE_MIN_MS = 250; +/** Code API gives a command its own timeout plus this much before it expires the assignment. */ +const COMMAND_EXECUTION_GRACE_MS = 5_000; +const LANE_GIT_TIMEOUT_MS = 5_000; +const LANE_GIT_OUTPUT_LIMIT = 4096; + +/** + * A read-only probe must never reach the network or run a transport command, whatever the checkout's + * own config says. A command can rewrite `.git/config` to mark a remote as a promisor and point a ref + * at a missing object, which makes a plain object lookup fetch lazily and run `ext::`, `ssh` or + * `uploadpack` programs on the host. An empty `GIT_ALLOW_PROTOCOL` disallows every transport and + * overrides repository configuration; `GIT_NO_LAZY_FETCH` covers Git 2.44 and later directly. + */ +const LANE_GIT_PROBE_ENV = { GIT_ALLOW_PROTOCOL: '', GIT_NO_LAZY_FETCH: '1' }; + +/** Exit codes that mean "no value" rather than "could not read": detached HEAD, unborn branch. */ +const DETACHED_EXIT = 1; +const UNBORN_EXIT = 128; + +async function read( + root: string, + args: string[], + emptyExit: number, + signal?: AbortSignal, + timeoutMs: number = LANE_GIT_TIMEOUT_MS, +): Promise { + try { + const bytes = await gitBytes(root, args, signal, timeoutMs, LANE_GIT_OUTPUT_LIMIT, LANE_GIT_PROBE_ENV); + // Git permits ref names that are not UTF-8. Decoding them leniently would report a different, + // valid-looking name, so anything that is not strictly valid UTF-8 is reported as null. + let text: string; + try { + text = new TextDecoder('utf-8', { fatal: true }).decode(bytes); + } catch { + return null; + } + // Drop only Git's own trailing line terminator: a ref name may legally begin or end with + // other whitespace, and trimming it would report a different branch. + return text.replace(/\r?\n$/, ''); + } catch (error) { + signal?.throwIfAborted(); + return (error as { code?: unknown }).code === emptyExit ? null : undefined; + } +} + +/** + * The branch and head commit checked out in a lane root, or undefined when Git + * could not be read (timeout, missing binary, aborted). Detached HEAD and a + * repository without commits are ordinary states and report null. Values that + * fail the bounds are reported as null, never forwarded. Nothing else from the + * repository is read or returned. + */ +export async function readLaneGit( + root: string, + signal?: AbortSignal, + timeoutMs: number = LANE_GIT_TIMEOUT_MS, +): Promise { + return await snapshotLaneGit( + () => read(root, ['symbolic-ref', '--quiet', 'HEAD'], DETACHED_EXIT, signal, timeoutMs), + // `--no-replace-objects`: a `refs/replace` entry must not turn a blob into a reported commit. + (ref) => read(root, ['--no-replace-objects', 'rev-parse', '--verify', `${ref}^{commit}`], UNBORN_EXIT, signal, timeoutMs), + ); +} + +/** + * Two Git processes cannot see one atomic snapshot, so a checkout between them could pair one + * branch with another branch's commit, a tuple that was never checked out. The head is therefore + * read from the branch ref the first read named, not from `HEAD`, and the branch is read again + * afterwards: a checkout away during the read omits the field. A detached HEAD has no ref to pin, + * so it is read twice and must agree. A change that happens and reverts within one probe is not + * detectable without a lock, and is not worth one for an advisory field. + */ +export async function snapshotLaneGit( + readBranch: () => Promise, + readCommit: (ref: string) => Promise, +): Promise { + const before = await readBranch(); + if (before === undefined) return undefined; + const head = await readCommit(before ?? 'HEAD'); + const after = await readBranch(); + if (head === undefined || after === undefined || before !== after) return undefined; + if (before === null && (await readCommit('HEAD')) !== head) return undefined; + // `--short` shortens ambiguously (`heads/foo` when a tag `foo` exists). Take the full ref and strip + // exactly `refs/heads/`; a HEAD pointing anywhere else is not a branch. + const name = before?.startsWith('refs/heads/') ? before.slice('refs/heads/'.length) : null; + return { branch: boundedBranch(name), head: boundedHead(head) }; +} + +async function absent(path: string): Promise { + try { + await lstat(path); + return false; + } catch (error) { + return (error as NodeJS.ErrnoException).code === 'ENOENT'; + } +} + +async function plain(path: string, kind: 'file' | 'directory'): Promise { + try { + const status = await lstat(path); + return !status.isSymbolicLink() && (kind === 'file' ? status.isFile() : status.isDirectory()); + } catch { + return false; + } +} + +/** Entries a checkout may have before the probe gives up on proving they are all plain. */ +const LANE_GIT_REFS_MAX_ENTRIES = 20_000; +const LANE_GIT_OBJECTS_MAX_ENTRIES = 100_000; + +/** Whether everything beneath a directory is a plain file or directory: no link can redirect a read. */ +async function treeIsLinkFree( + refs: string, + signal?: AbortSignal, + maxEntries = LANE_GIT_REFS_MAX_ENTRIES, +): Promise { + const pending = [refs]; + let seen = 0; + while (pending.length > 0) { + // Stop between directory operations: the probe timeout returns the command result, but only an + // abort observed here keeps an abandoned walk from running on in the background. + signal?.throwIfAborted(); + const current = pending.pop()!; + const directory = await opendir(current); + for await (const entry of directory) { + signal?.throwIfAborted(); + if (++seen > maxEntries) return false; + if (entry.isSymbolicLink()) return false; + // `Dirent.parentPath` needs Node 20.12; the package supports 20.11. + if (entry.isDirectory()) pending.push(join(current, entry.name)); + else if (!entry.isFile()) return false; + } + } + return true; +} + +/** + * The shared part of both ownership checks: the refs and object storage of a Git directory hold + * no link, no `alternates` redirect and no `reftable` backend, so the host-side reads cannot be + * steered at another repository's refs or objects. + */ +async function storageIsOwn(gitDir: string, signal?: AbortSignal): Promise { + const objects = join(gitDir, 'objects'); + return ( + (await plain(join(gitDir, 'refs'), 'directory')) && + (await treeIsLinkFree(join(gitDir, 'refs'), signal)) && + (await plain(objects, 'directory')) && + (await absent(join(objects, 'info', 'alternates'))) && + (await absent(join(objects, 'info', 'http-alternates'))) && + (await treeIsLinkFree(objects, signal, LANE_GIT_OBJECTS_MAX_ENTRIES)) && + (await absent(join(gitDir, 'reftable'))) && + ((await absent(join(gitDir, 'packed-refs'))) || (await plain(join(gitDir, 'packed-refs'), 'file'))) + ); +} + +/** + * Whether `/.git` is the checkout's own Git directory, so the host-side probe cannot be + * steered at another repository. A command can rewrite its own `.git` after it ran: replace it + * with a `gitdir:` file, add a `commondir` redirect, or link `HEAD`, `packed-refs`, `refs` or any + * ref beneath it to another checkout, and the unsandboxed probe would then report that checkout's + * branch and head. Every entry under `refs/` is therefore checked, up to a bound. + * The check runs after the command has finished, so it does not cover a process the command left + * running in the background. + * Anything else, including a `.git` file (a linked worktree or submodule as a source root), is + * not probed: the field is simply omitted. Linked worktree lanes are verified separately. + */ +export async function ownsGitMetadata(root: string, signal?: AbortSignal): Promise { + const dotGit = join(root, '.git'); + try { + if (!(await plain(dotGit, 'directory')) || (await realpath(dotGit)) !== join(await realpath(root), '.git')) { + return false; + } + return ( + (await plain(join(dotGit, 'HEAD'), 'file')) && + (await absent(join(dotGit, 'commondir'))) && + (await storageIsOwn(dotGit, signal)) + ); + } catch { + return false; + } +} + +/** + * The linked worktree counterpart of `ownsGitMetadata`. `verifyLinkedWorktree` proves the lane's + * `.git` pointer chain, but the probe also reads the lane's own `HEAD` and the refs it names from + * the shared common directory, and a command can write to both. Without this, a ref symlinked at + * another checkout and named by the lane's `HEAD` would be followed by the host-side probe. + */ +export async function ownsLinkedWorktreeMetadata(commonGitDir: string, name: string, signal?: AbortSignal): Promise { + const metadata = join(commonGitDir, 'worktrees', name); + try { + if (!(await plain(join(metadata, 'HEAD'), 'file'))) return false; + if (!(await absent(join(metadata, 'refs'))) && !(await treeIsLinkFree(join(metadata, 'refs'), signal))) return false; + return await storageIsOwn(commonGitDir, signal); + } catch { + return false; + } +} + +export interface LaneGitWorkspaceToolsOptions { + delegate: WorkspaceToolExecutor; + /** The lane root a request ran in (isolated worktree, linked worktree or source checkout), or undefined. */ + resolveRoot: (request: WorkspaceToolRequest, signal?: AbortSignal) => Promise; + /** Whether Code API negotiated `lane_git`. Git is not probed while this is false. Defaults to true. */ + isEnabled?: () => boolean; + /** Longest the probe may run. Defaults to 1 s; it is also cut to fit the command's remaining budget. */ + probeTimeoutMs?: number; + /** Clock, for tests. */ + now?: () => number; +} + +/** + * Attaches `laneGit` to every finished `execute_command` result, so a branch the + * agent created or switched is visible as soon as the command returns. Git is + * read once per command, never on a timer. The field is stripped by the worker + * unless Code API acknowledged the `lane_git` feature at registration. + */ +export class LaneGitWorkspaceTools implements WorkspaceToolExecutor { + readonly mutationFailuresAreAtomic?: true; + readonly capabilities: WorkspaceToolExecutor['capabilities']; + + constructor(private readonly options: LaneGitWorkspaceToolsOptions) { + this.mutationFailuresAreAtomic = options.delegate.mutationFailuresAreAtomic; + const base = options.delegate.capabilities; + this.capabilities = base.operations.includes('execute_command') + ? { ...base, commandResultFeatures: ['lane_git'] } + : base; + } + + async execute( + request: WorkspaceToolRequest, + signal?: AbortSignal, + context?: { deadlineAtMs?: number }, + ): Promise { + const now = this.options.now ?? Date.now; + const startedAt = now(); + const delegated = await this.options.delegate.execute(request, signal, context); + if (request.operation !== 'execute_command' || delegated.operation !== 'execute_command') return delegated; + // Only the value this wrapper reads is trusted. A sandbox executor must not be able to supply + // its own branch and head, whether or not the probe below produces anything. + const { laneGit: _untrusted, ...result } = delegated; + if (this.options.isEnabled?.() === false) return result; + // The probe is advisory, so it only runs in budget the command left over, and never into the + // part reserved for settlement. The worker's real deadline accounts for time already spent on + // lease transport and credential refresh; without it, assume the full timeout plus the grace. + const deadlineAtMs = + context?.deadlineAtMs ?? + startedAt + (request.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS) + COMMAND_EXECUTION_GRACE_MS; + const probeMs = Math.min( + this.options.probeTimeoutMs ?? LANE_GIT_PROBE_MAX_MS, + deadlineAtMs - now() - LANE_GIT_SETTLEMENT_RESERVE_MS, + ); + if (probeMs < Math.min(LANE_GIT_PROBE_MIN_MS, this.options.probeTimeoutMs ?? LANE_GIT_PROBE_MIN_MS)) return result; + // The resolver may do filesystem work that cannot observe a signal, so the race, not the + // signal, is what bounds the wait. The signal still stops Git when the limit hits first. + const controller = new AbortController(); + const probeSignal = AbortSignal.any([...(signal ? [signal] : []), controller.signal]); + let timer: NodeJS.Timeout | undefined; + try { + const probe = (async (): Promise => { + try { + const root = await this.options.resolveRoot(request, probeSignal); + return root == null ? undefined : await readLaneGit(root, probeSignal, probeMs); + } catch { + return undefined; + } + })(); + const limit = new Promise((resolve) => { + timer = setTimeout(() => { + controller.abort(); + resolve(undefined); + }, probeMs); + }); + const laneGit = await Promise.race([probe, limit]); + return laneGit ? { ...result, laneGit } : result; + } catch { + // Git state is advisory; a failed read must never fail or delay the command's own result. + return result; + } finally { + clearTimeout(timer); + } + } +} diff --git a/packages/code/src/linked-worktree-git-guard.test.ts b/packages/code/src/linked-worktree-git-guard.test.ts new file mode 100644 index 00000000..96386418 --- /dev/null +++ b/packages/code/src/linked-worktree-git-guard.test.ts @@ -0,0 +1,289 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { promisify } from 'node:util'; + +import { writeLinkedWorktreeGitGuard } from './linked-worktree-git-guard.js'; + +const execFileAsync = promisify(execFile); + +test('lane Git guard prevents destructive maintenance after global options without breaking ordinary Git', async t => { + if (process.platform === 'win32') return t.skip('the lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-git-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + await Promise.all([mkdir(bin), mkdir(repo)]); + await writeLinkedWorktreeGitGuard(bin); + const wrapper = join(bin, 'git'); + assert.equal((await stat(wrapper)).mode & 0o222, 0, 'the script is not writable'); + assert.match(await readFile(wrapper, 'utf8'), /^#!\/bin\/bash/); + await execFileAsync('/bin/bash', ['-n', wrapper]); + assert.match(await readFile(wrapper, 'utf8'), /run storage maintenance from the checkout/); + + const run = (args: string[]) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + assert.match((await run(['--version'])).stdout, /git version/); + assert.equal((await run(['-C', repo, '-c', 'core.quotepath=false', 'status', '--short'])).stdout, ''); + for (const option of ['--literal-pathspecs', '--glob-pathspecs', '--noglob-pathspecs', '--icase-pathspecs']) { + assert.equal((await run([option, '-C', repo, 'status', '--short'])).stdout, '', option); + } + await writeFile(join(repo, 'literal[abc].txt'), 'literal\n'); + await run(['--literal-pathspecs', 'add', '--', 'literal[abc].txt']); + assert.match((await run(['status', '--short'])).stdout, /A literal\[abc\]\.txt/); + + // Git has created this object, but no ref points to it yet. Another lane can be + // in exactly this window between writing objects and updating a ref. + await writeFile(join(repo, 'unpublished.txt'), 'from a second lane\n'); + const object = (await run(['hash-object', '-w', 'unpublished.txt'])).stdout.trim(); + const blocked = [ + ['prune', '--expire', 'now'], + ['-C', repo, 'prune', '--expire=now'], + ['-C' + repo, '-c', 'gc.auto=0', '--git-dir=' + join(repo, '.git'), 'prune', '--expire=now'], + ['--no-pager', '-cgc.auto=0', 'gc', '--force'], + ['repack', '-ad'], + ['prune-packed'], + ['maintenance', 'run', '--task=gc'], + ['multi-pack-index', 'expire'], + ['lfs', 'prune'], + ['--literal-pathspecs', 'prune', '--expire', 'now'], + ]; + for (const args of blocked) { + await assert.rejects(run(args), (error: unknown) => { + const result = error as { code?: number; stderr?: string }; + return result.code === 1 && /run storage maintenance from the checkout/.test(result.stderr ?? ''); + }, `must reject git ${args.join(' ')}`); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } + await assert.rejects(run(['--unrecognized-option', 'prune']), /unsupported global option/); + assert.match((await run(['-C', repo, 'status', '--short'])).stdout, /\?\? unpublished\.txt\n/); + + // The guard is deliberately not a sandbox boundary: resetting PATH or using + // an absolute Git binary can still prune shared objects. + await execFileAsync('git', ['prune', '--expire', 'now'], { cwd: repo }); + await assert.rejects(run(['cat-file', '-e', object])); +}); + +test('lane Git guard rejects aliases from checkout config, -c, include.path and --config-env', async t => { + if (process.platform === 'win32') return t.skip('the lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-alias-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + const otherRepo = join(parent, 'other'); + await Promise.all([mkdir(bin), mkdir(repo), mkdir(otherRepo)]); + await writeLinkedWorktreeGitGuard(bin); + const run = (args: string[], extraEnv: Record = {}) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, ...extraEnv, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + await run(['config', 'alias.cleanup', 'prune --expire now']); + await run(['config', 'alias.inspect', 'status --short']); + await run(['config', 'alias.indirect', 'cleanup']); + await run(['-C', otherRepo, 'init', '-q', '-b', 'main']); + await run(['-C', otherRepo, 'config', 'alias.othercleanup', 'prune --expire now']); + await writeFile(join(parent, 'aliases.cfg'), '[alias]\n frominclude = prune --expire now\n'); + await writeFile(join(repo, 'unpublished.txt'), 'a different lane wrote this object\n'); + const object = (await run(['hash-object', '-w', 'unpublished.txt'])).stdout.trim(); + const blocked: Array<[string[], Record?]> = [ + [['cleanup']], + [['-C', repo, 'cleanup']], + [['-C', otherRepo, 'othercleanup']], + [['--git-dir=' + join(otherRepo, '.git'), 'othercleanup']], + [['inspect']], + [['indirect']], + [['-c', 'alias.fromoption=prune --expire now', 'fromoption']], + [['-c', 'alias.fromshell=!git prune --expire now', 'fromshell']], + [['--git-dir=' + join(repo, '.git'), '-c', 'alias.fromgitdir=prune --expire now', 'fromgitdir']], + [['-c', `include.path=${join(parent, 'aliases.cfg')}`, 'frominclude']], + [['--config-env=alias.fromenv=LANE_TEST_GIT_ALIAS', 'fromenv'], { LANE_TEST_GIT_ALIAS: 'prune --expire now' }], + ]; + for (const [args, environment] of blocked) { + await assert.rejects(run(args, environment), (error: unknown) => { + const result = error as { code?: number; stderr?: string }; + return result.code === 1 && /Git aliases are unavailable in linked worktree lanes/.test(result.stderr ?? ''); + }, `must reject Git alias in ${JSON.stringify(args)}`); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } + assert.match((await run(['status', '--short'])).stdout, /unpublished\.txt/); +}); + +test('a lane rejects a checkout-defined maintenance alias before pruning another lane’s unpublished object', async t => { + if (process.platform === 'win32') return t.skip('the lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-shared-alias-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const checkout = join(parent, 'checkout'); + const bin = join(parent, 'bin'); + await Promise.all([mkdir(checkout), mkdir(bin)]); + await execFileAsync('git', ['init', '-q', '-b', 'main'], { cwd: checkout }); + await execFileAsync('git', [ + '-c', 'user.name=test', '-c', 'user.email=test@example.test', + '-c', 'commit.gpgsign=false', 'commit', '--allow-empty', '-q', '-m', 'seed', + ], { cwd: checkout }); + await mkdir(join(checkout, '.worktrees')); + const lane = join(checkout, '.worktrees', 'task'); + await execFileAsync('git', ['worktree', 'add', '-q', '-b', 'task', lane], { cwd: checkout }); + await execFileAsync('git', ['config', 'alias.cleanup', 'prune --expire now'], { cwd: checkout }); + await writeLinkedWorktreeGitGuard(bin); + await writeFile(join(lane, 'pending.txt'), 'another lane has not updated its ref\n'); + const run = (args: string[]) => execFileAsync('git', args, { + cwd: lane, + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + const object = (await run(['hash-object', '-w', 'pending.txt'])).stdout.trim(); + await assert.rejects(run(['cleanup']), (error: unknown) => { + const result = error as { code?: number; stderr?: string }; + return result.code === 1 && /Git aliases are unavailable in linked worktree lanes/.test(result.stderr ?? ''); + }); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); +}); + +test('a lane never lets a misspelled maintenance command autocorrect to prune', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-autocorrect-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + await Promise.all([mkdir(bin), mkdir(repo)]); + await writeLinkedWorktreeGitGuard(bin); + const run = (args: string[]) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + await run(['config', 'help.autocorrect', 'immediate']); + await writeFile(join(repo, 'unpublished'), 'a sibling has not made this blob reachable'); + const object = (await run(['hash-object', '-w', 'unpublished'])).stdout.trim(); + for (const args of [ + ['prun', '--expire', 'now'], + ['-c', 'help.autocorrect=immediate', 'prun', '--expire=now'], + ]) { + await assert.rejects(run(args), /not a git command|not a Git command|unknown Git command/i); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } +}); + +test('Git options cannot re-enable automatic maintenance for lane commands', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-maintenance-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + const remote = join(parent, 'remote.git'); + await Promise.all([mkdir(bin), mkdir(repo)]); + await writeLinkedWorktreeGitGuard(bin); + const run = (args: string[], additionalEnv: NodeJS.ProcessEnv = {}) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, ...additionalEnv, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + await execFileAsync('git', ['init', '-q', '--bare', remote]); + for (const [key, supplied, enforced] of [ + ['maintenance.auto', 'true', 'false'], + ['gc.auto', '1', '0'], + ['help.autocorrect', 'immediate', '0'], + ]) { + const { stdout } = await run(['-c', `${key}=${supplied}`, 'config', '--get', key]); + assert.equal(stdout.trim(), enforced, `${key} must be enforced after caller-supplied -c`); + } + const { stdout } = await run(['config', '--get', 'maintenance.auto'], { + GIT_CONFIG_COUNT: '1', GIT_CONFIG_KEY_0: 'maintenance.auto', GIT_CONFIG_VALUE_0: 'true', + }); + assert.equal(stdout.trim(), 'false', 'inherited configuration must not re-enable maintenance'); + const configuredGc = await run(['--config-env=gc.auto=LANE_TEST_GC_AUTO', 'config', '--get', 'gc.auto'], { + LANE_TEST_GC_AUTO: '1', + }); + assert.equal(configuredGc.stdout.trim(), '0', 'config-env cannot override the enforced setting'); + await writeFile(join(repo, 'unpublished'), 'not referenced yet'); + const object = (await run(['hash-object', '-w', 'unpublished'])).stdout.trim(); + for (const args of [ + ['fetch', '--auto-maintenance', remote], + ['fetch', '--auto-maintenance=true', remote], + ['-c', 'maintenance.auto=true', 'fetch', '--auto-gc', remote], + ['fetch', '--auto-gc=true', remote], + ['-c', 'gc.auto=1', 'fetch', '--auto-maintenance', remote], + ['pull', '--auto-maintenance', remote], + ]) { + await assert.rejects(run(args), /run storage maintenance from the checkout/); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } + await run(['-c', 'maintenance.auto=true', '-c', 'gc.auto=1', 'fetch', '--no-auto-maintenance', remote, 'refs/heads/*:refs/remotes/origin/*']); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); +}); + +test('for-each-repo cannot dispatch maintenance behind the lane Git guard', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-for-each-repo-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + await Promise.all([mkdir(bin), mkdir(repo)]); + await writeLinkedWorktreeGitGuard(bin); + const run = (args: string[]) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + await writeFile(join(repo, 'pending'), 'a sibling has not published this object yet\n'); + const object = (await run(['hash-object', '-w', 'pending'])).stdout.trim(); + const dispatcher = ['-c', `maintenance.repo=${repo}`, 'for-each-repo', '--config=maintenance.repo']; + for (const args of [ + [...dispatcher, 'prune', '--expire', 'now'], + [...dispatcher, 'status', '--short'], + ]) { + await assert.rejects(run(args), (error: unknown) => { + const result = error as { code?: number; stderr?: string }; + return result.code === 1 && /run storage maintenance from the checkout/.test(result.stderr ?? ''); + }, 'Git dispatchers cannot be inspected safely inside a lane'); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } + assert.match((await run(['status', '--short'])).stdout, /pending/); +}); + +test('Git LFS fetch and pull cannot request pruning in a lane', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const parent = await mkdtemp(join(tmpdir(), 'linked-worktree-lfs-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + const repo = join(parent, 'repo'); + await Promise.all([mkdir(bin), mkdir(repo)]); + await writeLinkedWorktreeGitGuard(bin); + const run = (args: string[]) => execFileAsync('git', args, { + cwd: repo, + env: { ...process.env, PATH: `${bin}:${process.env.PATH ?? '/usr/bin:/bin'}` }, + }); + await run(['init', '-q', '-b', 'main']); + await writeFile(join(repo, 'unpublished'), 'an LFS sibling has no ref yet'); + const object = (await run(['hash-object', '-w', 'unpublished'])).stdout.trim(); + for (const args of [ + ['lfs', 'fetch', '--prune'], + ['lfs', 'fetch', '-p'], + ['lfs', 'fetch', '-rp'], + ['lfs', 'fetch', '--prune', '--recent'], + ['lfs', 'pull', '--prune'], + ['-C', repo, 'lfs', 'fetch', '--prune'], + ]) { + await assert.rejects(run(args), /run storage maintenance from the checkout/); + assert.equal((await run(['cat-file', '-t', object])).stdout.trim(), 'blob'); + } +}); + +test('lane guard rejects registration changes but permits read-only worktree listing', async t => { + const parent = await mkdtemp(join(tmpdir(), 'lane-registration-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const bin = join(parent, 'bin'); + await mkdir(bin); await writeLinkedWorktreeGitGuard(bin); + await execFileAsync('/usr/bin/git', ['init', '-q', '-b', 'main'], { cwd: parent }); + const run = (args: string[]) => execFileAsync(join(bin, 'git'), args, { cwd: parent }); + assert.match((await run(['worktree', 'list', '--porcelain'])).stdout, /worktree/); + for (const operation of ['add', 'remove', 'move', 'repair', 'prune', 'lock', 'unlock']) { + await assert.rejects(run(['-C', parent, 'worktree', operation, '.worktrees/other']), /require checkout admission/); + } +}); diff --git a/packages/code/src/linked-worktree-git-guard.ts b/packages/code/src/linked-worktree-git-guard.ts new file mode 100644 index 00000000..cee4a7f0 --- /dev/null +++ b/packages/code/src/linked-worktree-git-guard.ts @@ -0,0 +1,129 @@ +import { constants as fsConstants } from 'node:fs'; +import { access, realpath, stat, writeFile } from 'node:fs/promises'; +import { join, sep } from 'node:path'; + +import { WorkspaceToolError } from './workspace.js'; + +// Match the system-only executable search used for native programmatic tools. +const GIT_BIN_DIRS = [ + '/opt/homebrew/bin', + '/usr/local/bin', + '/usr/bin', + '/bin', + '/home/linuxbrew/.linuxbrew/bin', +]; +const TRUSTED_GIT_DIRS = [ + ...GIT_BIN_DIRS, + '/opt/homebrew/Cellar', + '/usr/local/Cellar', + '/home/linuxbrew/.linuxbrew/Cellar', + '/usr/lib/git-core', +]; + +async function systemGitExecutable(): Promise { + for (const directory of GIT_BIN_DIRS) { + try { + const executable = await realpath(join(directory, 'git')); + if (!TRUSTED_GIT_DIRS.some(path => executable.startsWith(`${path}${sep}`))) continue; + if (!(await stat(executable)).isFile()) continue; + await access(executable, fsConstants.X_OK); + return executable; + } catch { + // A missing system installation is not a reason to run Git from the workspace or PATH. + } + } + throw new WorkspaceToolError('Trusted Git executable is unavailable for linked worktree lanes', 'COMMAND_UNAVAILABLE'); +} + +/** This guards accidental `git` calls, not absolute executable paths or caller shell aliases. */ +export async function writeLinkedWorktreeGitGuard(directory: string): Promise { + try { + await access('/bin/bash', fsConstants.X_OK); + } catch { + throw new WorkspaceToolError('Linked worktree Git guard requires /bin/bash', 'COMMAND_UNAVAILABLE'); + } + const git = await systemGitExecutable(); + const quotedGit = `'${git.replaceAll("'", "'\\''")}'`; + const script = [ + // Bash arrays preserve global option arguments when checking the same + // effective Git configuration without eval or interpreting alias contents. + '#!/bin/bash', + 'check() {', + ' local -a globals=() forwarded=()', + ' local aliasStatus=0', + ' while [ "$#" -gt 0 ]; do', + ' case "$1" in', + ' -C|-c|--git-dir|--work-tree|--namespace|--config-env)', + ' if [ "$#" -lt 2 ]; then echo "git: missing global option argument" >&2; return 2; fi', + ' globals+=("$1" "$2"); forwarded+=("$1" "$2"); shift 2 ;;', + ' -C?*|-c?*|--git-dir=*|--work-tree=*|--namespace=*|--config-env=*)', + ' globals+=("$1"); forwarded+=("$1"); shift ;;', + ' --bare)', + ' globals+=("$1"); forwarded+=("$1"); shift ;;', + ' -p|-P|--paginate|--no-pager|--no-replace-objects|--no-optional-locks|--exec-path=*)', + ' forwarded+=("$1"); shift ;;', + ' --literal-pathspecs|--glob-pathspecs|--noglob-pathspecs|--icase-pathspecs)', + ' forwarded+=("$1"); shift ;;', + ' --) forwarded+=("$1"); shift; break ;;', + ' -v|--version|-h|--help|--exec-path|--html-path|--man-path|--info-path)', + ' return 0 ;;', + ' -*) echo "git: unsupported global option in linked worktree lane" >&2; return 2 ;;', + ' *) break ;;', + ' esac', + ' done', + ' case "${1:-}" in', + ' worktree)', + ' if [ "${2:-}" != list ]; then', + ' echo "git: worktree registration changes require checkout admission or the lifecycle API" >&2', + ' return 1', + ' fi ;;', + + ' prune|gc|repack|prune-packed|maintenance|multi-pack-index|for-each-repo)', + ' echo "git: run storage maintenance from the checkout, not a linked worktree lane" >&2', + ' return 1 ;;', + ' fetch|pull)', + ' for option in "${@:2}"; do', + ' case "$option" in', + ' --auto-maintenance|--auto-maintenance=*|--auto-gc|--auto-gc=*)', + ' echo "git: run storage maintenance from the checkout, not a linked worktree lane" >&2', + ' return 1 ;;', + ' esac', + ' done ;;', + ' lfs)', + ' if [ "${2:-}" = prune ]; then', + ' echo "git: run storage maintenance from the checkout, not a linked worktree lane" >&2', + ' return 1', + ' fi', + ' if [ "${2:-}" = fetch ] || [ "${2:-}" = pull ]; then', + ' for option in "${@:3}"; do', + ' case "$option" in', + ' --prune|--prune=*|-p|-p?*|-[!-]*p*)', + ' echo "git: run storage maintenance from the checkout, not a linked worktree lane" >&2', + ' return 1 ;;', + ' esac', + ' done', + ' fi ;;', + ' esac', + ' if [ "$#" -eq 0 ]; then return 0; fi', + // Git itself loads local config, -C, -c, includes and --config-env, so an + // alias from any of those sources is rejected even when it hides a prune. + // Reject all aliases, including harmless ones: Git shell aliases and nested + // aliases cannot be proven safe by inspecting the first token. + ` ${quotedGit} "\${globals[@]}" config --get "alias.\${1}" >/dev/null 2>&1 || aliasStatus=$?`, + ' case "$aliasStatus" in', + ' 0) echo "git: Git aliases are unavailable in linked worktree lanes; run the underlying command or use the checkout" >&2; return 1 ;;', + ' 1) ;;', + ' *) echo "git: cannot verify Git aliases in linked worktree lane" >&2; return 2 ;;', + ' esac', + // Command-line -c values override both Git config files and GIT_CONFIG_COUNT. + // Add these after every caller global option: fetch can start maintenance + // internally without invoking the PATH guard a second time. + ` exec ${quotedGit} "\${forwarded[@]}" -c maintenance.auto=false -c gc.auto=0 -c help.autocorrect=0 "$@"`, + '}', + 'check "$@" || exit "$?"', + `exec ${quotedGit} "$@"`, + '', + ].join('\n'); + await writeFile(join(directory, 'git'), script, { flag: 'wx', mode: 0o500 }); + await access(join(directory, 'git'), fsConstants.X_OK); +} diff --git a/packages/code/src/linked-worktrees-live.test.ts b/packages/code/src/linked-worktrees-live.test.ts new file mode 100644 index 00000000..c9052a35 --- /dev/null +++ b/packages/code/src/linked-worktrees-live.test.ts @@ -0,0 +1,108 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, realpath, rm, stat, writeFile } from 'node:fs/promises'; +import { homedir, tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { verifyLinkedWorktree } from './linked-worktrees.js'; +import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; +import { resolveNativeSrtCommandPolicy } from './native-policy.js'; + +const execFileAsync = promisify(execFile); +const IDENTITY = ['-c', 'user.name=lane', '-c', 'user.email=lane@example.com', '-c', 'commit.gpgsign=false']; + +async function git(cwd: string, ...args: string[]): Promise { + return (await execFileAsync('git', [...IDENTITY, ...args], { cwd })).stdout.trim(); +} + +async function snapshot(paths: string[]): Promise> { + const entries = await Promise.all( + paths.map(async (path) => [path, await readFile(path, 'utf8').catch(() => null)] as const), + ); + return Object.fromEntries(entries); +} + +/** + * The worker's home is read-denied, so a checkout beneath it exercises the + * sandbox's tmpfs re-binding; one beneath the system temporary directory does not. + */ +for (const [location, parent] of [ + ['beneath the worker home', homedir()], + ['outside the worker home', tmpdir()], +] as const) { + test(`real SRT lets a linked worktree lane commit while checkout and sibling Git metadata stay intact (${location})`, { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 60_000, + }, async (t) => { + const base = await realpath(await mkdtemp(join(parent, 'lane-live-'))); + t.after(() => rm(base, { recursive: true, force: true })); + const root = join(base, 'repo'); + await mkdir(root); + await git(root, 'init', '-q', '-b', 'main'); + await writeFile(join(root, 'tracked.txt'), 'checkout\n'); + await git(root, 'add', 'tracked.txt'); + await git(root, 'commit', '-qm', 'init'); + await mkdir(join(root, '.worktrees')); + await git(root, 'worktree', 'add', '-q', '-b', 'task-a', '.worktrees/task-a'); + await git(root, 'worktree', 'add', '-q', '-b', 'task-b', '.worktrees/task-b'); + + const commonGitDir = join(root, '.git'); + const protectedFiles = [ + join(commonGitDir, 'HEAD'), + join(commonGitDir, 'index'), + join(commonGitDir, 'config'), + join(commonGitDir, 'MERGE_HEAD'), + join(commonGitDir, 'packed-refs'), + join(commonGitDir, 'hooks', 'pre-commit'), + join(commonGitDir, 'worktrees', 'task-b', 'HEAD'), + join(root, 'tracked.txt'), + ]; + const before = await snapshot(protectedFiles); + const lane = await verifyLinkedWorktree(root, 'task-a'); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: lane.root, + workspaceIdentity: lane.identity, + linkedWorktree: { + checkoutRoot: lane.checkoutRoot, + commonGitDir: lane.commonGitDir, + writableGitPaths: lane.writableGitPaths, + }, + commandPolicy: resolveNativeSrtCommandPolicy('trusted-vm'), + environment: { PATH: process.env.PATH, LANG: 'C.UTF-8' }, + }); + t.after(() => sandbox.close()); + const run = (command: string) => + sandbox.execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'lane', + command, + timeoutMs: 30_000, + maxOutputBytes: 16_384, + }); + const gitInLane = `git ${IDENTITY.join(' ')}`; + + const committed = await run( + `printf lane > lane.txt && ${gitInLane} add lane.txt && ${gitInLane} commit -qm lane && ${gitInLane} branch lane-extra`, + ); + assert.equal(committed.exitCode, 0, committed.stderr); + + for (const path of protectedFiles) { + await run(`printf tampered > '${path}'`); + } + // Packing must not move refs into storage that vanishes with the sandbox. + await run(`${gitInLane} pack-refs --all`); + await sandbox.close(); + + for (const branch of ['main', 'task-a', 'task-b', 'lane-extra']) { + assert.ok(await git(root, 'rev-parse', '--verify', '-q', `refs/heads/${branch}`), branch); + } + + assert.equal(await git(root, 'log', '-1', '--format=%s', 'task-a'), 'lane'); + assert.deepEqual(await snapshot(protectedFiles), before); + assert.equal(await git(root, 'status', '--porcelain', '--untracked-files=no'), ''); + await assert.rejects(stat(join(commonGitDir, 'MERGE_HEAD')), { code: 'ENOENT' }); + }); +} diff --git a/packages/code/src/linked-worktrees.test.ts b/packages/code/src/linked-worktrees.test.ts new file mode 100644 index 00000000..30c5b7e6 --- /dev/null +++ b/packages/code/src/linked-worktrees.test.ts @@ -0,0 +1,347 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { + mkdir, + mkdtemp, + readFile, + realpath, + rm, + stat, + symlink, + writeFile, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { + LinkedWorktreeWorkspaceTools, + linkedWorktreeWorkspaceId, + verifyLinkedWorktree, +} from './linked-worktrees.js'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; + +import type { NativeWorkspaceCommandPool } from './native-pool.js'; +import type { NativeProcessSandboxOptions } from './native-process.js'; +import type { WorkspaceToolRequest } from './protocol.js'; + +const execFileAsync = promisify(execFile); + +async function git(cwd: string, ...args: string[]): Promise { + await execFileAsync( + 'git', + ['-c', 'user.name=test', '-c', 'user.email=test@example.com', '-c', 'commit.gpgsign=false', ...args], + { cwd }, + ); +} + +async function checkout(): Promise<{ parent: string; root: string }> { + const parent = await realpath(await mkdtemp(join(tmpdir(), 'linked-worktree-'))); + const root = join(parent, 'repo'); + await mkdir(root); + await git(root, 'init', '-q', '-b', 'main'); + await writeFile(join(root, 'README.md'), 'root\n'); + await git(root, 'add', '.'); + await git(root, 'commit', '-q', '-m', 'init'); + await mkdir(join(root, '.worktrees')); + await git(root, 'worktree', 'add', '-q', '-b', 'task-a', '.worktrees/task-a'); + return { parent, root }; +} + +async function rejects(promise: Promise, code = 'INVALID_REQUEST'): Promise { + await assert.rejects(promise, (error: unknown) => error instanceof WorkspaceToolError && error.code === code); +} + +test('verifies a linked worktree of the checkout and lists the only Git paths it may write', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + await git(root, 'worktree', 'add', '-q', '-b', 'task-b', '.worktrees/task-b'); + + const lane = await verifyLinkedWorktree(root, 'task-a'); + + assert.equal(lane.root, join(root, '.worktrees', 'task-a')); + assert.equal(lane.checkoutRoot, root); + assert.equal(lane.commonGitDir, join(root, '.git')); + assert.deepEqual(lane.writableGitPaths, [ + join(root, '.git', 'objects'), + join(root, '.git', 'refs'), + join(root, '.git', 'logs', 'refs'), + join(root, '.git', 'lfs'), + join(root, '.git', 'worktrees', 'task-a'), + ]); +}); + + +test('rejects directories that are not linked worktrees of this checkout', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + + await mkdir(join(root, '.worktrees', 'plain')); + await rejects(verifyLinkedWorktree(root, 'plain')); + + await mkdir(join(root, '.worktrees', 'borrowed')); + await writeFile( + join(root, '.worktrees', 'borrowed', '.git'), + `gitdir: ${join(root, '.git', 'worktrees', 'task-a')}\n`, + ); + await rejects(verifyLinkedWorktree(root, 'borrowed')); + + await symlink(join(root, '.worktrees', 'task-a'), join(root, '.worktrees', 'alias')); + await rejects(verifyLinkedWorktree(root, 'alias')); + + const other = join(parent, 'other'); + await mkdir(other); + await git(other, 'init', '-q', '-b', 'main'); + await writeFile(join(other, 'file'), 'x\n'); + await git(other, 'add', '.'); + await git(other, 'commit', '-q', '-m', 'other'); + await git(other, 'worktree', 'add', '-q', '-b', 'foreign', join(root, '.worktrees', 'foreign')); + await rejects(verifyLinkedWorktree(root, 'foreign')); + + for (const name of ['..', '.git', 'a/b', 'task.lock']) { + await rejects(verifyLinkedWorktree(root, name)); + } + await rejects(verifyLinkedWorktree(root, 'missing')); +}); + +test('rejects a checkout whose .worktrees directory is a symlink', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + const elsewhere = join(parent, 'elsewhere'); + await git(root, 'worktree', 'move', '.worktrees/task-a', elsewhere); + await rm(join(root, '.worktrees'), { recursive: true }); + await symlink(parent, join(root, '.worktrees')); + await rejects(verifyLinkedWorktree(root, 'elsewhere')); +}); + +test('rejects shared Git storage symlinks into sibling worktree metadata', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + await git(root, 'worktree', 'add', '-q', '-b', 'task-b', '.worktrees/task-b'); + const commonGitDir = join(root, '.git'); + const siblingMetadata = join(commonGitDir, 'worktrees', 'task-b'); + await rm(join(commonGitDir, 'objects'), { recursive: true }); + await symlink(siblingMetadata, join(commonGitDir, 'objects')); + + await rejects(verifyLinkedWorktree(root, 'task-a')); +}); + +function recordingPool(): { + pool: NativeWorkspaceCommandPool; + calls: Array<{ action: string; id: string; options?: NativeProcessSandboxOptions }>; +} { + const calls: Array<{ action: string; id: string; options?: NativeProcessSandboxOptions }> = []; + const pool = { + async registerRoot(id: string, options: NativeProcessSandboxOptions) { + calls.push({ action: 'register', id, options }); + }, + async unregisterRoot(id: string) { + calls.push({ action: 'unregister', id }); + }, + async execute(request: WorkspaceToolRequest) { + calls.push({ action: 'execute', id: request.workspaceId }); + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + async executeProgrammatic(id: string) { + calls.push({ action: 'programmatic', id }); + return {}; + }, + } as unknown as NativeWorkspaceCommandPool; + return { pool, calls }; +} + +async function laneTools( + root: string, + pool?: NativeWorkspaceCommandPool, + onRelease?: (root: string) => void, +) { + const delegate = await LocalWorkspaceTools.create({ + repositoryInstructions: false, + workspaces: [{ id: 'repo', root, writable: true }], + }); + return new LinkedWorktreeWorkspaceTools({ + commandPool: pool, + delegate, + onRelease, + sources: new Map([ + [ + 'repo', + { + root, + command: { workspaceRoot: root } as NativeProcessSandboxOptions, + repositoryInstructions: false, + writable: true, + }, + ], + ]), + }); +} + +test('lane file tools are confined to the worktree and report the public workspace', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + const tools = await laneTools(root); + + assert.deepEqual(tools.capabilities.workspaces[0]?.workspaceScopes, ['git_linked_worktree']); + const written = await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'repo', + worktree: 'task-a', + path: 'lane.txt', + content: 'from the lane\n', + }); + assert.equal(written.workspaceId, 'repo'); + assert.equal(await readFile(join(root, '.worktrees', 'task-a', 'lane.txt'), 'utf8'), 'from the lane\n'); + await assert.rejects(stat(join(root, 'lane.txt'))); + + const read = await tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'repo', + path: 'README.md', + }); + assert.equal(read.operation === 'read_file' && read.content.trimEnd(), 'root'); + + await rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'repo', + worktree: 'task-a', + workspaceInstanceId: 'a'.repeat(64), + path: 'README.md', + }), + ); +}); + +test('lane writes create missing parents inside the worktree and report missing files', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + const tools = await laneTools(root); + const lane = join(root, '.worktrees', 'task-a'); + + await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'repo', + worktree: 'task-a', + path: 'packages/new-package/package.json', + content: '{}\n', + }); + assert.equal(await readFile(join(lane, 'packages', 'new-package', 'package.json'), 'utf8'), '{}\n'); + await assert.rejects(stat(join(root, 'packages'))); + + await rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'repo', + worktree: 'task-a', + path: '.checks/tests.log', + }), + 'NOT_FOUND', + ); + + await symlink(root, join(lane, 'checkout-link')); + await rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'repo', + worktree: 'task-a', + path: 'checkout-link/escaped/notes.txt', + content: 'blocked', + }), + 'INVALID_PATH', + ); + await rejects( + tools.execute({ + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'repo', + worktree: 'task-a', + path: 'checkout-link/missing.txt', + }), + 'INVALID_PATH', + ); + await assert.rejects(stat(join(root, 'escaped'))); +}); + +test('lane commands register a confined root once, whatever siblings come and go', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + const { pool, calls } = recordingPool(); + const tools = await laneTools(root, pool); + const command = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'repo', + worktree: 'task-a', + command: 'git status', + }; + const id = linkedWorktreeWorkspaceId('repo', 'task-a'); + + const result = await tools.execute(command); + await tools.execute(command); + assert.equal(result.workspaceId, 'repo'); + assert.deepEqual( + calls.map((call) => call.action), + ['register', 'execute', 'execute'], + ); + const registered = calls[0]!.options!; + assert.equal(calls[0]!.id, id); + assert.equal(registered.workspaceRoot, join(root, '.worktrees', 'task-a')); + assert.equal(registered.linkedWorktree?.commonGitDir, join(root, '.git')); + assert.equal(registered.linkedWorktree?.checkoutRoot, root); + + assert.ok(!registered.linkedWorktree?.writableGitPaths.includes(join(root, '.git'))); + + await rejects(tools.execute({ + ...command, + environmentAction: { name: 'unresolved', fingerprint: 'a'.repeat(64) }, + })); + assert.equal(calls.length, 3, 'an unresolved action must never reach the command pool'); + + await git(root, 'worktree', 'add', '-q', '-b', 'task-b', '.worktrees/task-b'); + await tools.execute(command); + assert.deepEqual( + calls.slice(3).map((call) => call.action), + ['execute'], + ); +}); + +test('a removed lane releases its command root and credential route', async (t) => { + const { parent, root } = await checkout(); + t.after(() => rm(parent, { recursive: true, force: true })); + const { pool, calls } = recordingPool(); + const released: string[] = []; + const tools = await laneTools(root, pool, (lane) => released.push(lane)); + const command = { + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'repo', + worktree: 'task-a', + command: 'git status', + }; + await tools.execute(command); + + await git(root, 'worktree', 'remove', '.worktrees/task-a'); + await rejects(tools.execute(command)); + + assert.deepEqual( + calls.map((call) => call.action), + ['register', 'execute', 'unregister'], + ); + assert.deepEqual(released, [join(root, '.worktrees', 'task-a')]); +}); diff --git a/packages/code/src/linked-worktrees.ts b/packages/code/src/linked-worktrees.ts new file mode 100644 index 00000000..225abebd --- /dev/null +++ b/packages/code/src/linked-worktrees.ts @@ -0,0 +1,534 @@ +import { createHash } from 'node:crypto'; +import { lstat, open, realpath } from 'node:fs/promises'; +import { isAbsolute, join, resolve } from 'node:path'; + +import { isValidLinkedWorktreeName } from './protocol.js'; +import { + captureWorkspaceRootIdentity, + matchesWorkspaceRoot, +} from './root-identity.js'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; + +import type { NativeWorkspaceCommandPool } from './native-pool.js'; +import type { NativeProcessSandboxOptions } from './native-process.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; +import type { + BridgeWorkspaceProgrammaticRequest, + WorkspaceExecuteCommandRequest, + WorkspaceToolRequest, + WorkspaceToolResult, +} from './protocol.js'; +import type { WorkspaceToolExecutor } from './workspace.js'; + +/** Linked worktrees are only admitted from this directory beneath a checkout. */ +export const LINKED_WORKTREE_DIRECTORY = '.worktrees'; +/** + * Shared Git storage a lane may write beneath the common Git directory. Every + * other path there (config, hooks, the checkout's own HEAD, index and + * operation state, sibling metadata) stays read-only. + */ +const LINKED_WORKTREE_SHARED_GIT_PATHS = ['objects', 'refs', join('logs', 'refs'), 'lfs']; +/** Lane registrations kept per worker; the least recently used idle lanes are released first. */ +const LINKED_WORKTREE_LANE_LIMIT = 32; +/** Git pointer files are a single line; anything larger is not one. */ +const GIT_POINTER_MAX_BYTES = 4096; +/** Last-use timestamps kept per worker; the oldest are forgotten first. */ +const LINKED_WORKTREE_ACTIVITY_LIMIT = 4096; + +/** What this worker knows about lane use, for deciding when a worktree may be retired. */ +export interface LinkedWorktreeActivity { + /** When a request for the lane last started or finished in this process. */ + lastUsed(workspaceId: string, worktree: string): number | undefined; + /** Changes whenever a checkout request (outside any lane) starts or finishes. */ + checkoutActivity(workspaceId: string): number; + /** + * Run `task` only while neither the lane nor its checkout has a request in + * flight. Requests for the lane or its checkout that arrive meanwhile wait + * for it to finish (or for their own cancellation); a lane request then + * verifies the lane again. + */ + whileIdle( + workspaceId: string, + worktree: string, + task: () => Promise, + ): Promise<{ ran: true; value: T } | { ran: false }>; +} + +export interface LinkedWorktreeSource { + root: string; + identity?: WorkspaceRootIdentity; + command?: NativeProcessSandboxOptions; + repositoryInstructions: boolean; + writable: boolean; +} + +export interface VerifiedLinkedWorktree { + root: string; + identity: WorkspaceRootIdentity; + checkoutRoot: string; + commonGitDir: string; + /** Shared object and ref storage plus the lane's own metadata; nothing else in the common Git directory. */ + writableGitPaths: string[]; +} + +export interface LinkedWorktreeWorkspaceToolsOptions { + commandPool?: NativeWorkspaceCommandPool; + delegate: WorkspaceToolExecutor; + /** Called with each verified lane root, e.g. to route credentials for its repository. */ + onResolve?: (workspaceId: string, root: string) => void; + /** Called when a lane root is released, e.g. to drop its credential route. */ + onRelease?: (root: string) => void; + programmaticDelegate?: { + executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise; + }; + sources: ReadonlyMap; +} + +export function linkedWorktreeWorkspaceId(workspaceId: string, worktree: string): string { + return `lane-${createHash('sha256').update(`${workspaceId}\0${worktree}`).digest('hex')}`; +} + +function rejected(message: string): WorkspaceToolError { + return new WorkspaceToolError(message, 'INVALID_REQUEST'); +} + +async function realDirectory(path: string): Promise { + try { + const status = await lstat(path); + return status.isDirectory() && !status.isSymbolicLink(); + } catch { + return false; + } +} + +/** Optional Git paths may be absent, but cannot redirect a write grant into sibling metadata. */ +async function safeSharedGitStorage(commonGitDir: string): Promise { + for (const path of ['objects', 'refs', 'logs', join('logs', 'refs'), 'lfs']) { + try { + const status = await lstat(join(commonGitDir, path)); + if (!status.isDirectory() || status.isSymbolicLink()) return false; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'ENOENT' || + path === 'objects' || path === 'refs') return false; + } + } + return true; +} + +async function readPointer(path: string): Promise { + let handle; + try { + const status = await lstat(path); + if (!status.isFile() || status.isSymbolicLink() || status.size > GIT_POINTER_MAX_BYTES) { + return undefined; + } + handle = await open(path, 'r'); + const buffer = Buffer.alloc(GIT_POINTER_MAX_BYTES + 1); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); + if (bytesRead > GIT_POINTER_MAX_BYTES) return undefined; + return buffer.subarray(0, bytesRead).toString('utf8'); + } catch { + return undefined; + } finally { + await handle?.close().catch(() => undefined); + } +} + +async function canonicalOrUndefined(path: string): Promise { + try { + return await realpath(path); + } catch { + return undefined; + } +} + +function singleLine(value: string | undefined): string | undefined { + const line = value?.replace(/\r?\n$/, ''); + return line == null || line.length === 0 || /[\r\n\0]/.test(line) ? undefined : line; +} + +/** + * Verify, without running Git, that `/.worktrees/` is a linked + * worktree of that checkout: a real directory whose `.git` file points at + * `/.git/worktrees/`, whose metadata points back at it, and + * whose common directory is the checkout's own `.git`. Neither the lane path + * nor a writable shared Git storage directory may be a symlink, so a forged + * worktree cannot borrow a lane or redirect its Git write grants. + */ +export async function verifyLinkedWorktree( + checkoutRoot: string, + name: string, + checkoutIdentity?: WorkspaceRootIdentity, +): Promise { + if (!isValidLinkedWorktreeName(name)) { + throw rejected('Invalid linked worktree name'); + } + const checkout = await canonicalOrUndefined(checkoutRoot); + if (checkout == null || !(await realDirectory(checkout))) { + throw new WorkspaceToolError('Selected project is unavailable', 'REGISTRATION_INVALID'); + } + if (checkoutIdentity != null && !(await matchesWorkspaceRoot(checkout, checkoutIdentity))) { + throw new WorkspaceToolError('Selected project changed before lane admission', 'REGISTRATION_INVALID'); + } + const commonGitDir = join(checkout, '.git'); + const worktreesDirectory = join(checkout, LINKED_WORKTREE_DIRECTORY); + const root = join(worktreesDirectory, name); + const metadata = join(commonGitDir, 'worktrees', name); + if ( + !(await realDirectory(commonGitDir)) || + !(await realDirectory(worktreesDirectory)) || + !(await realDirectory(root)) || + (await canonicalOrUndefined(root)) !== root || + !(await realDirectory(metadata)) || + (await canonicalOrUndefined(metadata)) !== metadata || + !(await safeSharedGitStorage(commonGitDir)) + ) { + throw rejected(`No linked worktree named ${name} in ${LINKED_WORKTREE_DIRECTORY}`); + } + const dotGit = join(root, '.git'); + const pointer = singleLine(await readPointer(dotGit))?.match(/^gitdir: (.+)$/)?.[1]; + const gitDir = pointer == null ? undefined : isAbsolute(pointer) ? pointer : resolve(root, pointer); + const commonPointer = singleLine(await readPointer(join(metadata, 'commondir'))); + const backPointer = singleLine(await readPointer(join(metadata, 'gitdir'))); + if ( + gitDir == null || + (await canonicalOrUndefined(gitDir)) !== metadata || + commonPointer == null || + (await canonicalOrUndefined(resolve(metadata, commonPointer))) !== commonGitDir || + backPointer == null || + (await canonicalOrUndefined(resolve(metadata, backPointer))) !== dotGit + ) { + throw rejected(`${LINKED_WORKTREE_DIRECTORY}/${name} is not a linked worktree of this project`); + } + let identity: WorkspaceRootIdentity; + try { + identity = await captureWorkspaceRootIdentity(root); + } catch { + throw rejected(`${LINKED_WORKTREE_DIRECTORY}/${name} is unavailable`); + } + const writableGitPaths = [ + ...LINKED_WORKTREE_SHARED_GIT_PATHS.map((path) => join(commonGitDir, path)), + metadata, + ]; + return { root, identity, checkoutRoot: checkout, commonGitDir, writableGitPaths }; +} + +function publicResult(result: WorkspaceToolResult, workspaceId: string): WorkspaceToolResult { + return { ...result, workspaceId }; +} + +/** + * Route requests that name a linked worktree into their own isolated executor. + * Code API schedules each `.worktrees/` as its own lane beneath the + * checkout, so file tools and commands here run confined to that worktree while + * sibling lanes run concurrently. Requests without a worktree pass through. + */ +export class LinkedWorktreeWorkspaceTools implements WorkspaceToolExecutor, LinkedWorktreeActivity { + readonly mutationFailuresAreAtomic?: true; + readonly capabilities: WorkspaceToolExecutor['capabilities']; + private readonly executors = new Map< + string, + { fingerprint: string; value: Promise } + >(); + private readonly commandRoots = new Map(); + /** Verified lane roots by internal ID, least recently used first. */ + private readonly lanes = new Map(); + /** Requests in flight by lane internal ID. */ + private readonly laneRequests = new Map(); + /** Requests in flight by checkout workspace ID, outside any lane. */ + private readonly checkoutRequests = new Map(); + /** Last request start or finish by lane internal ID, oldest first. */ + private readonly used = new Map(); + /** Checkout request starts and finishes by workspace ID. */ + private readonly checkoutEvents = new Map(); + /** Lanes being retired by internal ID; requests for the lane or its checkout wait for them to settle. */ + private readonly retiring = new Map }>(); + + constructor(private readonly options: LinkedWorktreeWorkspaceToolsOptions) { + this.mutationFailuresAreAtomic = options.delegate.mutationFailuresAreAtomic; + this.capabilities = { + ...options.delegate.capabilities, + workspaces: options.delegate.capabilities.workspaces.map((workspace) => ({ + ...workspace, + ...(options.sources.has(workspace.id) + ? { workspaceScopes: ['git_linked_worktree' as const] } + : {}), + })), + }; + } + + private async resolveLane( + workspaceId: string, + worktree: string, + workspaceInstanceId: string | undefined, + ): Promise<{ lane: VerifiedLinkedWorktree; source: LinkedWorktreeSource; internalId: string }> { + if (workspaceInstanceId != null) { + throw rejected('Linked worktree lanes are not available inside conversation worktrees'); + } + const source = this.options.sources.get(workspaceId); + if (!source) { + throw rejected('Workspace does not allow linked worktree lanes'); + } + const internalId = linkedWorktreeWorkspaceId(workspaceId, worktree); + let lane: VerifiedLinkedWorktree; + try { + lane = await verifyLinkedWorktree(source.root, worktree, source.identity); + } catch (error) { + await this.release(internalId); + throw error; + } + this.lanes.delete(internalId); + this.lanes.set(internalId, lane.root); + this.options.onResolve?.(workspaceId, lane.root); + await this.releaseIdleLanes(internalId); + return { lane, source, internalId }; + } + + /** Forget a lane's executors and routes; a lane still running a command is kept. */ + private async release(internalId: string): Promise { + const root = this.lanes.get(internalId); + if (root == null) return; + if (this.commandRoots.has(internalId)) { + try { + await this.options.commandPool?.unregisterRoot(internalId); + } catch { + return; + } + this.commandRoots.delete(internalId); + } + this.executors.delete(internalId); + this.lanes.delete(internalId); + this.options.onRelease?.(root); + } + + private async releaseIdleLanes(current: string): Promise { + for (const internalId of [...this.lanes.keys()]) { + if (this.lanes.size <= LINKED_WORKTREE_LANE_LIMIT) return; + if (internalId !== current) await this.release(internalId); + } + } + + private async fileExecutor( + internalId: string, + lane: VerifiedLinkedWorktree, + source: LinkedWorktreeSource, + ): Promise { + const fingerprint = `${lane.identity.path}\0${lane.identity.dev}\0${lane.identity.ino}`; + let cached = this.executors.get(internalId); + if (cached == null || cached.fingerprint !== fingerprint) { + cached = { + fingerprint, + value: LocalWorkspaceTools.create({ + repositoryInstructions: source.repositoryInstructions, + workspaces: [ + { id: internalId, identity: lane.identity, root: lane.root, writable: source.writable }, + ], + }), + }; + this.executors.set(internalId, cached); + } + return await cached.value; + } + + /** Register the lane's command root, replacing it when its identity or writable Git paths changed. */ + private async registerCommandRoot( + internalId: string, + lane: VerifiedLinkedWorktree, + source: LinkedWorktreeSource, + ): Promise { + const pool = this.options.commandPool; + if (!source.command || !pool) { + throw new WorkspaceToolError('Linked worktree commands are unavailable', 'COMMAND_DISABLED'); + } + const fingerprint = JSON.stringify([ + lane.identity.path, + lane.identity.dev, + lane.identity.ino, + lane.writableGitPaths, + ]); + const registered = this.commandRoots.get(internalId); + if (registered !== fingerprint) { + if (registered != null) await pool.unregisterRoot(internalId); + await pool.registerRoot(internalId, { + ...source.command, + workspaceIdentity: lane.identity, + workspaceRoot: lane.root, + linkedWorktree: { + checkoutRoot: lane.checkoutRoot, + commonGitDir: lane.commonGitDir, + writableGitPaths: lane.writableGitPaths, + }, + }); + this.commandRoots.set(internalId, fingerprint); + } + return pool; + } + + lastUsed(workspaceId: string, worktree: string): number | undefined { + return this.used.get(linkedWorktreeWorkspaceId(workspaceId, worktree)); + } + + checkoutActivity(workspaceId: string): number { + return this.checkoutEvents.get(workspaceId) ?? 0; + } + + async whileIdle( + workspaceId: string, + worktree: string, + task: () => Promise, + ): Promise<{ ran: true; value: T } | { ran: false }> { + const internalId = linkedWorktreeWorkspaceId(workspaceId, worktree); + if ( + this.laneRequests.has(internalId) || + this.checkoutRequests.has(workspaceId) || + this.retiring.has(internalId) + ) { + return { ran: false }; + } + let settle!: () => void; + const done = new Promise((resolve) => { settle = resolve; }); + this.retiring.set(internalId, { workspaceId, done }); + try { + return { ran: true, value: await task() }; + } finally { + await this.release(internalId); + this.retiring.delete(internalId); + settle(); + } + } + + private touch(internalId: string): void { + this.used.delete(internalId); + this.used.set(internalId, Date.now()); + if (this.used.size <= LINKED_WORKTREE_ACTIVITY_LIMIT) return; + const oldest = this.used.keys().next().value; + if (oldest != null) this.used.delete(oldest); + } + + /** Count a request from its first synchronous step, so `whileIdle` never races its admission. */ + private async tracked( + requests: Map, + key: string, + task: () => Promise, + lane: boolean, + ): Promise { + const record = (): void => { + if (lane) this.touch(key); + else this.checkoutEvents.set(key, (this.checkoutEvents.get(key) ?? 0) + 1); + }; + requests.set(key, (requests.get(key) ?? 0) + 1); + record(); + try { + return await task(); + } finally { + const remaining = (requests.get(key) ?? 1) - 1; + if (remaining > 0) requests.set(key, remaining); + else requests.delete(key); + record(); + } + } + + /** Wait out retirements a request could collide with, unless the request is cancelled first. */ + private async afterRetirement( + collides: (internalId: string, workspaceId: string) => boolean, + signal: AbortSignal | undefined, + ): Promise { + const pending = [...this.retiring] + .filter(([internalId, entry]) => collides(internalId, entry.workspaceId)) + .map(([, entry]) => entry.done); + if (pending.length === 0) return; + signal?.throwIfAborted(); + await new Promise((resolve, reject) => { + const onAbort = (): void => reject(signal?.reason); + signal?.addEventListener('abort', onAbort, { once: true }); + void Promise.all(pending).then(() => { + signal?.removeEventListener('abort', onAbort); + resolve(); + }); + }); + } + + private async inCheckout(workspaceId: string, task: () => Promise, signal?: AbortSignal): Promise { + return await this.tracked(this.checkoutRequests, workspaceId, async () => { + await this.afterRetirement((_internalId, laneWorkspaceId) => laneWorkspaceId === workspaceId, signal); + return await task(); + }, false); + } + + private async inLane( + workspaceId: string, + worktree: string, + task: () => Promise, + signal?: AbortSignal, + ): Promise { + const internalId = linkedWorktreeWorkspaceId(workspaceId, worktree); + return await this.tracked(this.laneRequests, internalId, async () => { + await this.afterRetirement((retiringId) => retiringId === internalId, signal); + return await task(); + }, true); + } + + async execute(request: WorkspaceToolRequest, signal?: AbortSignal): Promise { + if (request.worktree == null) { + return await this.inCheckout( + request.workspaceId, + () => this.options.delegate.execute(request, signal), + signal, + ); + } + if (request.operation === 'execute_command' && request.environmentAction) { + throw rejected('Environment action was not resolved by this worker'); + } + const { worktree, ...baseRequest } = request; + return await this.inLane(request.workspaceId, worktree, async () => { + const { lane, source, internalId } = await this.resolveLane( + request.workspaceId, + worktree, + request.workspaceInstanceId, + ); + const laneRequest = { ...baseRequest, workspaceId: internalId } as WorkspaceToolRequest; + if (request.operation === 'execute_command') { + const pool = await this.registerCommandRoot(internalId, lane, source); + return publicResult( + await pool.execute(laneRequest as WorkspaceExecuteCommandRequest, signal), + request.workspaceId, + ); + } + const executor = await this.fileExecutor(internalId, lane, source); + return publicResult(await executor.execute(laneRequest, signal), request.workspaceId); + }, signal); + } + + async executeProgrammatic( + workspaceId: string, + request: BridgeWorkspaceProgrammaticRequest, + signal?: AbortSignal, + ): Promise { + const worktree = request.body.workspace_worktree; + if (worktree == null) { + const delegate = this.options.programmaticDelegate; + if (!delegate) { + throw new WorkspaceToolError('Workspace programmatic execution is unavailable', 'COMMAND_DISABLED'); + } + return await this.inCheckout( + workspaceId, + () => delegate.executeProgrammatic(workspaceId, request, signal), + signal, + ); + } + return await this.inLane(workspaceId, worktree, async () => { + const { lane, source, internalId } = await this.resolveLane( + workspaceId, + worktree, + request.body.workspace_instance_id, + ); + const pool = await this.registerCommandRoot(internalId, lane, source); + const { workspace_worktree: _worktree, ...body } = request.body; + return await pool.executeProgrammatic(internalId, { ...request, body }, signal); + }, signal); + } +} diff --git a/packages/code/src/native-process.test.ts b/packages/code/src/native-process.test.ts index 5f7eedbb..8abf263c 100644 --- a/packages/code/src/native-process.test.ts +++ b/packages/code/src/native-process.test.ts @@ -150,6 +150,22 @@ test('executor bootstrap excludes bridge credentials and Node injection variable await sandbox.close(); }); +test('executor forwards the linked worktree policy to the sandbox process', async () => { + const fake = fixture(); + const linkedWorktree = { + checkoutRoot: '/checkout', + commonGitDir: '/checkout/.git', + writableGitPaths: ['/checkout/.git/objects', '/checkout/.git/refs'], + }; + const sandbox = new NativeProcessWorkspaceCommandSandbox( + { workspaceRoot: '/checkout/.worktrees/task-a', linkedWorktree }, + fake.fork, + ); + await sandbox.prepare(); + assert.deepEqual(fake.messages[0].options.linkedWorktree, linkedWorktree); + await sandbox.close(); +}); + test('executor forwards the resolved command policy without worker credentials', async () => { const fake = fixture(); const sandbox = new NativeProcessWorkspaceCommandSandbox( diff --git a/packages/code/src/native-process.ts b/packages/code/src/native-process.ts index ba411042..588f4c33 100644 --- a/packages/code/src/native-process.ts +++ b/packages/code/src/native-process.ts @@ -343,6 +343,8 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan homeDirectory, shellPath, programmaticFileUpstream, + linkedWorktree, + resources, } = this.options; await this.rpc( 'prepare', @@ -356,6 +358,8 @@ export class NativeProcessWorkspaceCommandSandbox implements WorkspaceCommandSan homeDirectory, shellPath, programmaticFileUpstream, + linkedWorktree, + resources, variables: this.options.maskedEnvironment?.variables, }, }, diff --git a/packages/code/src/native-sandbox.test.ts b/packages/code/src/native-sandbox.test.ts index eab72966..90215156 100644 --- a/packages/code/src/native-sandbox.test.ts +++ b/packages/code/src/native-sandbox.test.ts @@ -33,6 +33,7 @@ import { } from './native-sandbox.js'; import { restoreScratchTraversal } from './native-scratch.js'; import { WorkspaceToolError } from './workspace.js'; +import { isWorkspaceToolResult } from './protocol.js'; const request = { protocolVersion: 1 as const, @@ -1355,14 +1356,151 @@ test('executes in the canonical workspace and bounds aggregate output', async t operation: 'execute_command', workspaceId: 'primary', exitCode: 0, - stdout: '1234567890', - stderr: 'ab', + stdout: '123890', + stderr: 'abchij', truncated: true, timedOut: false, }, ); }); +test('retains real command summaries on both streams under the legacy combined budget', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + t.after(() => sandbox.close()); + const commandRequest = { + ...request, + maxOutputBytes: 256, + command: + "printf 'OUT\\n'; printf '%20000d' 0; printf '\\n42 tests passed\\n'; printf 'ERR\\n' >&2; printf '%20000d' 0 >&2; printf '\\nlate stderr summary\\n' >&2", + }; + const result = await sandbox.execute(commandRequest); + assert.equal(result.exitCode, 0); + assert.equal(result.timedOut, false); + assert.equal(result.truncated, true); + assert.ok(result.stdout.startsWith('OUT\n')); + assert.ok(result.stderr.startsWith('ERR\n')); + assert.ok(result.stdout.endsWith('\n42 tests passed\n')); + assert.ok(result.stderr.endsWith('\nlate stderr summary\n')); + assert.ok(result.stdout.includes('bytes omitted')); + assert.ok(result.stderr.includes('bytes omitted')); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); +}); + +test('partly filled command buffers retain final summaries regardless of stream order', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + t.after(() => sandbox.close()); + const summary = 'late stderr summary\n'; + const quiet = `printf '%109d' 0; printf 'late stderr summary\\n'`; + const noisy = `printf '%20000d' 0`; + for (const [command, stream] of [ + [`{ ${quiet}; } >&2; ${noisy}`, 'stderr'], + [`${noisy}; { ${quiet}; } >&2`, 'stderr'], + [`{ ${quiet}; }; ${noisy} >&2`, 'stdout'], + [`${noisy} >&2; { ${quiet}; }`, 'stdout'], + ] as const) { + const commandRequest = { ...request, command, maxOutputBytes: 256 }; + const result = await sandbox.execute(commandRequest); + assert.ok(result[stream].endsWith(summary)); + assert.equal(result.truncated, true); + assert.equal(result.exitCode, 0); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); + } +}); + +test('quiet malformed command output is retained beside a noisy stream', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + t.after(() => sandbox.close()); + for (const [command, stream] of [ + ["printf '%20000d' 0; printf '\\377\\377' >&2", 'stderr'], + ["printf '\\377\\377'; printf '%20000d' 0 >&2", 'stdout'], + ] as const) { + const commandRequest = { ...request, command, maxOutputBytes: 32 }; + const result = await sandbox.execute(commandRequest); + assert.equal(result[stream], '\ufffd\ufffd'); + assert.equal(result.truncated, true); + assert.equal(result.exitCode, 0); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); + } +}); + +test('sandbox annotations survive full stdout and remain bounded when stderr is noisy', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + fake.manager.annotateStderrWithSandboxFailures = (_commandId, stderr) => + stderr + '\n\ndenied write\n'; + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + for (const stderrCommand of ['', "printf '%20000d' 0 >&2;"]) { + const commandRequest = { + ...request, + maxOutputBytes: 512, + command: `printf '%20000d' 0; ${stderrCommand} true`, + }; + const result = await sandbox.execute(commandRequest); + assert.ok( + result.stderr.endsWith( + '\ndenied write\n' + ) + ); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); + assert.equal(result.truncated, true); + } + fake.manager.annotateStderrWithSandboxFailures = () => { + throw new Error('annotation unavailable'); + }; + const result = await sandbox.execute({ + ...request, + maxOutputBytes: 128, + command: "printf '%20000d' 0; printf 'child failure' >&2", + }); + assert.equal(result.stderr, 'child failure'); +}); + +test('timeout settlement keeps late diagnostics, signal, and truncation without widening the result', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + manager: fakeManager().manager, + }); + t.after(() => sandbox.close()); + const commandRequest = { + ...request, + maxOutputBytes: 128, + timeoutMs: 100, + command: + "printf 'START\\n'; printf '%20000d' 0; printf '\\nlast progress\\n'; printf 'last failure' >&2; sleep 30", + }; + const result = await sandbox.execute(commandRequest); + assert.ok(result.stdout.startsWith('START\n')); + assert.ok(result.stdout.endsWith('\nlast progress\n')); + assert.equal(result.stderr, 'last failure'); + assert.equal(result.timedOut, true); + assert.equal(result.truncated, true); + assert.equal(result.exitCode, null); + assert.equal(result.signal, 'SIGKILL'); + assert.equal(isWorkspaceToolResult(commandRequest, result), true); +}); + test('rejects an escaping or unavailable command working directory', async t => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-native-')); t.after(() => rm(root, { recursive: true, force: true })); @@ -1598,3 +1736,278 @@ test('cleans allocated command state exactly once on every execution exit', asyn } } }); + +test('a linked worktree lane may write only shared Git storage and its own metadata', async t => { + const checkoutRoot = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-lane-'))); + t.after(() => rm(checkoutRoot, { recursive: true, force: true })); + const commonGitDir = join(checkoutRoot, '.git'); + const lane = join(checkoutRoot, '.worktrees', 'task-a'); + const writableGitPaths = [ + join(commonGitDir, 'objects'), + join(commonGitDir, 'refs'), + join(commonGitDir, 'worktrees', 'task-a'), + ]; + await Promise.all( + [lane, ...writableGitPaths].map(path => mkdir(path, { recursive: true })), + ); + const prepare = async (paths: string[], platform: NodeJS.Platform = 'linux') => { + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: lane, + linkedWorktree: { checkoutRoot, commonGitDir, writableGitPaths: paths }, + environment: { PATH: '/usr/bin' }, + manager: fake.manager, + platform, + }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + return fake.config!; + }; + + const config = await prepare(writableGitPaths); + assert.deepEqual(config.filesystem.allowWrite.slice(0, 4), [lane, ...writableGitPaths]); + assert.ok(!config.filesystem.allowWrite.includes(commonGitDir)); + assert.ok(config.filesystem.allowRead?.includes(commonGitDir)); + // Deeper read denies make SRT re-bind each writable Git directory after the + // read-only bind of the common directory (Linux tmpfs re-binding order). + for (const path of writableGitPaths) { + assert.ok(config.filesystem.denyRead.includes(path), path); + } + assert.ok(!config.filesystem.denyRead.includes(join(commonGitDir, 'lfs'))); + const darwin = await prepare(writableGitPaths, 'darwin'); + for (const path of writableGitPaths) { + assert.ok(!darwin.filesystem.denyRead.includes(path), path); + } + const gitGuard = config.filesystem.allowRead?.find(path => path.includes('librechat-code-git-')); + assert.ok(gitGuard, 'lane Git guard must be readable'); + assert.ok(!config.filesystem.allowWrite.includes(gitGuard)); + assert.ok(config.filesystem.denyWrite.includes(gitGuard)); + assert.equal((await stat(join(gitGuard, 'git'))).mode & 0o222, 0); + + const probed = fakeManager(); + const prober = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: lane, + linkedWorktree: { checkoutRoot, commonGitDir, writableGitPaths }, + environment: { PATH: '/usr/bin' }, + manager: probed.manager, + }); + t.after(() => prober.close()); + const dataDirectory = await prober.createExecutionDirectory(); + await prober.executeProgrammatic(request, dataDirectory, undefined, { probe: true }); + assert.ok(probed.customConfigSeenDuringWrap?.filesystem?.allowRead?.includes(commonGitDir)); + assert.ok(!probed.customConfigSeenDuringWrap?.filesystem?.allowWrite?.includes(commonGitDir)); + const probeGuard = probed.config?.filesystem.allowRead?.find(path => path.includes('librechat-code-git-')); + assert.ok(probeGuard); + assert.ok(probed.customConfigSeenDuringWrap?.filesystem?.allowRead?.includes(probeGuard)); + assert.ok(probed.customConfigSeenDuringWrap?.filesystem?.denyWrite?.includes(probeGuard)); + + await assert.rejects( + prepare([commonGitDir]), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'REGISTRATION_INVALID', + ); + + const siblingMetadata = join(commonGitDir, 'worktrees', 'task-b'); + await mkdir(siblingMetadata, { recursive: true }); + await rm(join(commonGitDir, 'objects'), { recursive: true }); + await symlink(siblingMetadata, join(commonGitDir, 'objects')); + await assert.rejects( + prepare(writableGitPaths), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'REGISTRATION_INVALID', + ); +}); + +test('lane commands put the read-only Git guard ahead of the ordinary PATH', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const checkoutRoot = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-guard-'))); + t.after(() => rm(checkoutRoot, { recursive: true, force: true })); + const commonGitDir = join(checkoutRoot, '.git'); + const lane = join(checkoutRoot, '.worktrees', 'task-a'); + const writableGitPaths = [join(commonGitDir, 'objects'), join(commonGitDir, 'worktrees', 'task-a')]; + await Promise.all([lane, ...writableGitPaths].map(path => mkdir(path, { recursive: true }))); + const fake = fakeManager(); + let commandPath: string | undefined; + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: lane, + linkedWorktree: { checkoutRoot, commonGitDir, writableGitPaths }, + environment: { PATH: '/usr/bin:/bin' }, + manager: fake.manager, + spawnCommand(command, args, options) { + commandPath = options.env?.PATH; + return spawn(command, args, options); + }, + }); + t.after(() => sandbox.close()); + const result = await sandbox.execute({ ...request, command: 'git --version' }); + assert.equal(result.exitCode, 0); + assert.match(result.stdout, /git version/); + const guard = fake.config?.filesystem.allowRead?.find(path => path.includes('librechat-code-git-')); + assert.ok(guard); + assert.ok(commandPath?.startsWith(`${guard}:`)); + await sandbox.close(); + await assert.rejects(stat(guard), { code: 'ENOENT' }); +}); + +test('linked worktree Git guard is removed when sandbox initialization fails', async t => { + if (process.platform === 'win32') return t.skip('lane Git guard requires POSIX'); + const checkoutRoot = await mkdtemp(join(tmpdir(), 'librechat-code-failed-guard-')); + t.after(() => rm(checkoutRoot, { recursive: true, force: true })); + const commonGitDir = join(checkoutRoot, '.git'); + const lane = join(checkoutRoot, '.worktrees', 'task-a'); + const objects = join(commonGitDir, 'objects'); + await Promise.all([lane, objects].map(path => mkdir(path, { recursive: true }))); + const fake = fakeManager({ initializeError: new Error('init failed') }); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: lane, + linkedWorktree: { checkoutRoot, commonGitDir, writableGitPaths: [objects] }, + manager: fake.manager, + }); + await assert.rejects(sandbox.prepare(), /init failed/); + const guard = fake.config?.filesystem.allowRead?.find(path => path.includes('librechat-code-git-')); + assert.ok(guard); + await assert.rejects(stat(guard), { code: 'ENOENT' }); + await sandbox.close(); +}); + +test('linked worktree Git guard refuses Windows rather than admitting an unguarded lane', async t => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-win-guard-')); + t.after(() => rm(root, { recursive: true, force: true })); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + linkedWorktree: { + checkoutRoot: root, + commonGitDir: join(root, '.git'), + writableGitPaths: [join(root, '.git', 'objects')], + }, + platform: 'win32', + manager: fakeManager().manager, + }); + await assert.rejects(sandbox.prepare(), (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'COMMAND_UNAVAILABLE' && + /POSIX host/.test(error.message), + ); + await sandbox.close(); +}); + +test('non-lane roots deny their own executable Git metadata at initialization', async t => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-gitmeta-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const gitDir = join(root, '.git'); + await mkdir(join(gitDir, 'hooks'), { recursive: true }); + await writeFile(join(gitDir, 'config'), '[core]\n'); + await writeFile(join(gitDir, 'config.worktree'), '[core]\n'); + // A submodule Git directory (identified by its HEAD file) and a nested one. + // This submodule has no config.worktree, so it is not denied one. + await mkdir(join(gitDir, 'modules', 'sub', 'modules', 'inner'), { recursive: true }); + await writeFile(join(gitDir, 'modules', 'sub', 'HEAD'), 'ref: refs/heads/main\n'); + await writeFile(join(gitDir, 'modules', 'sub', 'config'), '[core]\n'); + await writeFile( + join(gitDir, 'modules', 'sub', 'modules', 'inner', 'HEAD'), + 'ref: refs/heads/main\n', + ); + // A linked worktree's per-worktree metadata: an existing commondir and + // config.worktree are both denied. + await mkdir(join(gitDir, 'worktrees', 'wt'), { recursive: true }); + await writeFile(join(gitDir, 'worktrees', 'wt', 'commondir'), '../..\n'); + await writeFile(join(gitDir, 'worktrees', 'wt', 'config.worktree'), '[core]\n'); + + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + environment: { PATH: '/usr/bin', LANG: 'C.UTF-8' }, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + + const denyWrite = fake.config?.filesystem.denyWrite ?? []; + for (const relativePath of [ + '.git/hooks', + '.git/config', + '.git/config.worktree', + '.git/modules/sub/hooks', + '.git/modules/sub/config', + '.git/modules/sub/modules/inner/hooks', + '.git/modules/sub/modules/inner/config', + '.git/worktrees/wt/config.worktree', + '.git/worktrees/wt/commondir', + ]) { + assert.ok(denyWrite.includes(join(root, relativePath)), relativePath); + } + // Deny-if-exists paths are skipped when absent: masking them with an empty + // bind would make Git read a broken redirect or config from every command. + assert.ok(!denyWrite.includes(join(gitDir, 'commondir'))); + assert.ok(!denyWrite.includes(join(gitDir, 'modules', 'sub', 'config.worktree'))); +}); + +test('ordinary root commands deny Git metadata created after initialization', async t => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-gitmeta-late-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + environment: { PATH: '/usr/bin', LANG: 'C.UTF-8' }, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + const session = fake.config!.filesystem; + const gitDir = join(root, '.git'); + assert.ok(!session.denyWrite.some(path => path.startsWith(gitDir))); + + // The host initializes the repository and adds a submodule and a linked + // worktree while the worker keeps running. + await mkdir(join(gitDir, 'hooks'), { recursive: true }); + await writeFile(join(gitDir, 'config'), '[core]\n'); + await mkdir(join(gitDir, 'modules', 'late'), { recursive: true }); + await writeFile(join(gitDir, 'modules', 'late', 'HEAD'), 'ref: refs/heads/main\n'); + await mkdir(join(gitDir, 'worktrees', 'late'), { recursive: true }); + await writeFile(join(gitDir, 'worktrees', 'late', 'commondir'), '../..\n'); + + const result = await sandbox.execute(request); + assert.equal(result.exitCode, 0); + const filesystem = fake.customConfigSeenDuringWrap?.filesystem; + for (const relativePath of [ + '.git/hooks', + '.git/config', + '.git/modules/late/hooks', + '.git/modules/late/config', + '.git/worktrees/late/commondir', + ]) { + assert.ok(filesystem?.denyWrite.includes(join(root, relativePath)), relativePath); + } + // Only denyWrite differs from the session filesystem policy. + assert.deepEqual({ ...filesystem, denyWrite: session.denyWrite }, session); +}); + +test('a root command fails closed when its Git metadata cannot be inspected', async t => { + if (process.getuid?.() === 0) { + t.skip('directory permissions do not restrict root'); + return; + } + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-gitmeta-denied-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const modules = join(root, '.git', 'modules'); + await mkdir(modules, { recursive: true }); + await writeFile(join(root, '.git', 'config'), '[core]\n'); + const fake = fakeManager(); + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + environment: { PATH: '/usr/bin', LANG: 'C.UTF-8' }, + manager: fake.manager, + }); + t.after(() => sandbox.close()); + await sandbox.prepare(); + + await chmod(modules, 0o000); + try { + await assert.rejects( + sandbox.execute(request), + (error: unknown) => + error instanceof WorkspaceToolError && error.code === 'COMMAND_UNAVAILABLE', + ); + } finally { + await chmod(modules, 0o700); + } +}); diff --git a/packages/code/src/native-sandbox.ts b/packages/code/src/native-sandbox.ts index 5bf61c50..2701fe44 100644 --- a/packages/code/src/native-sandbox.ts +++ b/packages/code/src/native-sandbox.ts @@ -11,9 +11,11 @@ import { sep, } from 'node:path'; import { constants as fsConstants } from 'node:fs'; -import { access, mkdtemp, open, realpath, rm, stat } from 'node:fs/promises'; +import type { Dirent, Stats } from 'node:fs'; +import { access, mkdtemp, open, readdir, realpath, rm, stat } from 'node:fs/promises'; import type { FileHandle } from 'node:fs/promises'; import { matchesWorkspaceRoot } from './root-identity.js'; +import { assertEnvironmentResourcesStable, environmentResourceVariables } from './environment-resources.js'; import type { WorkspaceRootIdentity } from './root-identity.js'; import { withWorkspaceRoot, WorkspaceRootAccessError, spawnWithinWorkspace, realpath as rootedRealpath, stat as rootedStat } from './root-access.js'; @@ -31,7 +33,9 @@ import { removePrivateStorageAcl, } from './private-storage.js'; import { WorkspaceToolError } from './workspace.js'; +import { OutputBuffer, renderCommandOutput } from './output.js'; import { restoreScratchTraversal } from './native-scratch.js'; +import { writeLinkedWorktreeGitGuard } from './linked-worktree-git-guard.js'; import type { ChildProcessWithoutNullStreams, @@ -106,10 +110,15 @@ const TRUSTED_GIT_ENVIRONMENT = { GIT_CONFIG_KEY_3: 'filter.lfs.required', GIT_CONFIG_VALUE_3: 'true', } as const; -const { - GIT_CONFIG_COUNT: TRUSTED_GIT_CONFIG_COUNT, - ...TRUSTED_GIT_CONFIG_ENTRIES -} = TRUSTED_GIT_ENVIRONMENT; +/** Sibling lanes share object storage, so a lane never starts automatic gc or maintenance. */ +const LINKED_WORKTREE_GIT_ENVIRONMENT = { + ...TRUSTED_GIT_ENVIRONMENT, + GIT_CONFIG_COUNT: '6', + GIT_CONFIG_KEY_4: 'gc.auto', + GIT_CONFIG_VALUE_4: '0', + GIT_CONFIG_KEY_5: 'maintenance.auto', + GIT_CONFIG_VALUE_5: 'false', +} as const; const NATIVE_SANDBOX_SCRATCH_PREFIX = 'librechat-code-srt-'; // SRT grants these shared compatibility paths by default. A worker-specific @@ -160,8 +169,19 @@ type SpawnCommand = ( ) => ChildProcessWithoutNullStreams; export interface NativeSrtWorkspaceCommandSandboxOptions { + /** Explicit operator-approved stores, never inferred from process environment. */ + resources?: import('./environment-resources.js').LoadedEnvironmentResource[]; workspaceIdentity?: WorkspaceRootIdentity; workspaceRoot: string; + /** Present when `workspaceRoot` is a verified linked worktree lane of a checkout. */ + linkedWorktree?: { + /** The checkout that owns the worktree; trusted as a Git safe directory. */ + checkoutRoot: string; + /** `/.git`: readable, but writable only at `writableGitPaths`. */ + commonGitDir: string; + /** Shared objects and refs plus the lane's own metadata beneath `commonGitDir`. */ + writableGitPaths: string[]; + }; commandPolicy?: NativeSrtCommandPolicy; /** Trusted worker files that must never become workspace-readable or writable. */ protectedPaths?: string[]; @@ -219,6 +239,130 @@ async function canonicalPath(path: string): Promise { } } +/** + * Paths inside a repository's own Git directory that Git treats as executable + * configuration: hooks run on the next Git invocation, and `config` / + * an existing `config.worktree` can define filter, diff, or fsmonitor commands + * that later run unsandboxed. An existing `commondir` is denied too, since it + * redirects the whole common Git directory to attacker-controlled config. + * + * A sandboxed command may write anywhere in the workspace root, so these are + * denied explicitly for the registered root's `.git`. SRT's Linux mandatory + * denies derive equivalents from the worker process's current directory, which + * is not the registered root, so they never cover it; macOS instead applies + * global `.git/hooks` and `.git/config` Seatbelt patterns. Denying the root's + * own metadata directly keeps the guarantee identical on both platforms and + * independent of the worker's cwd. Lane workspaces keep the whole common Git + * directory read-only already, so this augments only the non-lane root. + * + * The set is recomputed before every ordinary root command, so a repository, + * submodule, or linked worktree created after initialization is covered from + * the next command on. Only submodule and linked-worktree Git directories + * that exist at that moment are enumerated: masking a non-existent + * `.git/modules` would block a later `git submodule add` from creating it + * during the same command. `.git/info` + * is intentionally omitted — its attributes reference filter/diff drivers by + * name, but the commands those names resolve to live in the denied config, so + * `info` alone cannot introduce a new executable. + */ +async function collectGitMetadataDenies(root: string): Promise { + const gitDir = join(root, '.git'); + if (!(await statIfPresent(gitDir))?.isDirectory()) return []; + const denies = [join(gitDir, 'hooks'), join(gitDir, 'config')]; + await pushExistingDenies(denies, gitDir); + for (const submoduleGitDir of await collectSubmoduleGitDirs( + join(gitDir, 'modules'), + )) { + denies.push(join(submoduleGitDir, 'hooks'), join(submoduleGitDir, 'config')); + await pushExistingDenies(denies, submoduleGitDir); + } + const worktreesDir = join(gitDir, 'worktrees'); + for (const entry of await readdirIfPresent(worktreesDir)) { + if (entry.isDirectory()) { + await pushExistingDenies(denies, join(worktreesDir, entry.name)); + } + } + return denies; +} + +/** + * `commondir` and `config.worktree` are read strictly by Git when present — + * `commondir` at every startup, `config.worktree` whenever `worktreeConfig` + * is enabled — and SRT masks a non-existent deny target with an empty + * `/dev/null` bind that Git cannot open, failing every Git command. Denying + * them only where they already exist keeps the mask a read-only bind of the + * real file. `config` and `hooks` stay unconditional: they always exist in a + * real repository. In a repository that already enables `worktreeConfig`, a + * missing `config.worktree` therefore stays creatable, like a new + * `commondir`; the package README lists both with the other residuals of a + * writable workspace. + */ +async function pushExistingDenies(paths: string[], gitDir: string): Promise { + for (const name of ['config.worktree', 'commondir']) { + const candidate = join(gitDir, name); + if (await statIfPresent(candidate)) paths.push(candidate); + } +} + +/** + * Enumerate every submodule Git directory beneath `.git/modules`, following the + * `/modules/` nesting Git uses for recursive submodules. A + * directory holding a `HEAD` file is a Git directory; any other directory is an + * intermediate segment of a submodule path that itself contains slashes. + */ +async function collectSubmoduleGitDirs(modulesDir: string): Promise { + const found: string[] = []; + const stack = [modulesDir]; + while (stack.length > 0) { + const dir = stack.pop()!; + const entries = await readdirIfPresent(dir); + if (entries.some(entry => entry.isFile() && entry.name === 'HEAD')) { + found.push(dir); + if ( + entries.some(entry => entry.isDirectory() && entry.name === 'modules') + ) { + stack.push(join(dir, 'modules')); + } + continue; + } + for (const entry of entries) { + if (entry.isDirectory()) stack.push(join(dir, entry.name)); + } + } + return found; +} + +/** + * Absence is an expected answer while inspecting Git metadata. Any other + * failure (for example `EACCES`) propagates, so metadata that cannot be + * inspected fails the command closed instead of silently dropping a deny. + */ +function isMissing(error: unknown): boolean { + return ( + error instanceof Error && + 'code' in error && + (error.code === 'ENOENT' || error.code === 'ENOTDIR') + ); +} + +async function statIfPresent(path: string): Promise { + try { + return await stat(path); + } catch (error) { + if (isMissing(error)) return undefined; + throw error; + } +} + +async function readdirIfPresent(path: string): Promise { + try { + return await readdir(path, { withFileTypes: true }); + } catch (error) { + if (isMissing(error)) return []; + throw error; + } +} + function boundedUtf8(buffer: Buffer, budget: number): string { let end = Math.min(buffer.byteLength, budget); while (end > 0) { @@ -295,11 +439,24 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox private readonly platform: NodeJS.Platform; private initialized?: Promise; private canonicalRoot?: string; + /** A lane's shared Git directory; replay probes of the lane must read it too. */ + private canonicalCommonGitDir?: string; private runtimeConfig?: SandboxRuntimeConfig; private denyReadPaths: string[] = []; private denyWritePaths: string[] = []; + /** Worker-owned write denies that precede each command's live Git metadata denies. */ + private baseDenyWritePaths: string[] = []; + private get gitEnvironment(): + | typeof TRUSTED_GIT_ENVIRONMENT + | typeof LINKED_WORKTREE_GIT_ENVIRONMENT { + return this.options.linkedWorktree + ? LINKED_WORKTREE_GIT_ENVIRONMENT + : TRUSTED_GIT_ENVIRONMENT; + } private scratchDirectory?: string; private scratchHandle?: FileHandle; + /** A private sibling of scratch, readable but never writable by lane commands. */ + private gitGuardDirectory?: string; private execution?: Promise; private closing?: Promise; private resetFailed = false; @@ -338,6 +495,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox await this.manager.reset().catch(() => { this.resetFailed = true; }); + await this.removeLinkedWorktreeGitGuard().catch(() => undefined); await this.removeScratchDirectory().catch(() => undefined); if (!this.resetFailed) managerOwners.delete(this.manager); this.initialized = undefined; @@ -375,6 +533,14 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox const protectedPaths = await Promise.all( (this.options.protectedPaths ?? []).map(canonicalPath), ); + const resources = this.options.resources ?? []; + await assertEnvironmentResourcesStable(resources); + for (const resource of resources) { + if (isWithin(root, resource.path) || isWithin(resource.path, root) || + protectedPaths.some(path => isWithin(resource.path, path) || isWithin(path, resource.path))) { + throw new WorkspaceToolError('Environment resource overlaps a workspace or worker control path', 'REGISTRATION_INVALID'); + } + } if (protectedPaths.some(path => isWithin(root, path))) { throw new WorkspaceToolError( 'Native sandbox workspace cannot contain worker control files', @@ -424,6 +590,55 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } } + const lane = this.options.linkedWorktree; + if (lane && this.platform === 'win32') { + throw new WorkspaceToolError('Linked worktree Git guard requires a POSIX host', 'COMMAND_UNAVAILABLE'); + } + const commonGitDir = lane ? await canonicalPath(lane.commonGitDir) : undefined; + const checkoutRoot = lane ? await canonicalPath(lane.checkoutRoot) : undefined; + const writableGitPaths = lane + ? await Promise.all(lane.writableGitPaths.map(canonicalPath)) + : []; + if ( + lane && + (commonGitDir == null || + checkoutRoot == null || + !isWithin(checkoutRoot, commonGitDir) || + !isWithin(checkoutRoot, root) || + isWithin(commonGitDir, home) || + protectedPaths.some(path => isWithin(commonGitDir, path)) || + writableGitPaths.some( + (path, index) => + path !== resolve(lane.writableGitPaths[index]!) || + path === commonGitDir || + !isWithin(commonGitDir, path), + )) + ) { + throw new WorkspaceToolError( + 'Linked worktree Git storage is outside its checkout', + 'REGISTRATION_INVALID', + ); + } + const laneGitPaths = commonGitDir ? [commonGitDir] : []; + /** + * On Linux, SRT hides a read-denied directory (such as the worker home) under a + * tmpfs and then re-binds writes before reads, so the read-only bind of the whole + * common Git directory would mask its writable descendants. SRT processes read + * denies shallow-first, re-binding each one's writes on top, so listing every + * existing writable Git directory as a deeper deny restores its write bind after + * the ancestor read bind. The common directory stays a live, read-only host + * directory: nothing can be created at its top level. + */ + const laneWriteRebinds = + this.platform === 'linux' + ? ( + await Promise.all( + writableGitPaths.map(async path => + (await stat(path).catch(() => undefined))?.isDirectory() ? path : undefined, + ), + ) + ).filter((path): path is string => path != null) + : []; const canonicalScratchDirectory = await this.createScratchDirectory(sharedScratchPaths); if ( @@ -435,6 +650,18 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox 'REGISTRATION_INVALID', ); } + if (lane && !canonicalScratchDirectory) { + throw new WorkspaceToolError('Linked worktree Git guard requires private scratch storage', 'COMMAND_UNAVAILABLE'); + } + const gitGuardDirectory = lane + ? await this.createLinkedWorktreeGitGuard(canonicalScratchDirectory!) + : undefined; + // A lane already keeps the shared common Git directory read-only; only + // the non-lane root exposes a writable `.git` whose executable metadata + // SRT's cwd-derived mandatory denies do not reach. + const rootGitMetadataDenies = lane + ? [] + : await collectGitMetadataDenies(root); const commandPolicy = normalizeNativeSrtCommandPolicy( this.options.commandPolicy, ); @@ -456,20 +683,32 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ...sharedScratchPaths.filter(path => deniedInheritedWritablePaths.includes(path), ), + ...laneWriteRebinds, ], allowRead: [ root, + ...resources.map(resource => resource.path), + ...laneGitPaths, + ...(gitGuardDirectory ? [gitGuardDirectory] : []), ...(canonicalScratchDirectory ? [canonicalScratchDirectory] : []), ], allowWrite: [ root, + ...resources.filter(resource => resource.access === 'read-write').map(resource => resource.path), + ...writableGitPaths, ...(canonicalScratchDirectory ? [canonicalScratchDirectory] : []), ], - denyWrite: [...protectedPaths, ...deniedInheritedWritablePaths], + denyWrite: [ + ...protectedPaths, + ...resources.filter(resource => resource.access === 'read-only').map(resource => resource.path), + ...deniedInheritedWritablePaths, + ...rootGitMetadataDenies, + ...(gitGuardDirectory ? [gitGuardDirectory] : []), + ], allowGitConfig: false, }, credentials: { @@ -520,13 +759,14 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox allowAppleEvents: false, enableWeakerNestedSandbox: false, enableWeakerNetworkIsolation: false, - git: { safeDirectories: [root] }, + git: { safeDirectories: checkoutRoot ? [root, checkoutRoot] : [root] }, }; await this.manager.initialize( config, unrestrictedNetwork ? async () => true : undefined, ); this.canonicalRoot = root; + this.canonicalCommonGitDir = commonGitDir; this.runtimeConfig = config; this.denyReadPaths = [ home, @@ -534,9 +774,17 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox deniedInheritedWritablePaths.includes(path), ), ]; + this.baseDenyWritePaths = [ + ...protectedPaths, + ...resources.filter(resource => resource.access === 'read-only').map(resource => resource.path), + ...deniedInheritedWritablePaths, + ]; this.denyWritePaths = [ ...protectedPaths, + ...resources.filter(resource => resource.access === 'read-only').map(resource => resource.path), ...deniedInheritedWritablePaths, + ...rootGitMetadataDenies, + ...(gitGuardDirectory ? [gitGuardDirectory] : []), ]; } @@ -756,8 +1004,15 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ? { filesystem: { allowRead: [ + ...(this.options.resources ?? []).map(resource => resource.path), canonicalWorkspaceRoot ?? this.canonicalRoot!, canonicalDataDirectory, + ...(this.canonicalCommonGitDir + ? [this.canonicalCommonGitDir] + : []), + ...(this.gitGuardDirectory + ? [this.gitGuardDirectory] + : []), ], allowWrite: [ ...(canonicalWorkspaceRoot != null @@ -767,6 +1022,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ], denyRead: this.denyReadPaths, denyWrite: [ + ...(this.options.resources ?? []).map(resource => resource.path), this.canonicalRoot!, ...this.denyWritePaths, ], @@ -839,6 +1095,40 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } } + /** + * The session policy snapshots the root's Git metadata at initialization, but + * the worker outlives that snapshot: a repository, submodule, or linked + * worktree created on the host afterwards must be protected without a + * restart, the way SRT recomputes its own mandatory denies on every wrap. + * Only `denyWrite` differs from the session filesystem policy. Lanes keep + * the whole common Git directory read-only already. Native Windows keeps the + * initialization snapshot, because `srt-win` rejects the per-command + * `allowRead`/`allowWrite` a full filesystem override carries. + */ + private async rootCommandConfig(): Promise< + Partial | undefined + > { + const config = this.runtimeConfig; + const root = this.canonicalRoot; + if ( + !config || + !root || + this.options.linkedWorktree || + this.platform === 'win32' + ) { + return undefined; + } + return { + filesystem: { + ...config.filesystem, + denyWrite: [ + ...this.baseDenyWritePaths, + ...(await collectGitMetadataDenies(root)), + ], + }, + }; + } + private async executeBound( request: WorkspaceExecuteCommandRequest, signal?: AbortSignal, @@ -866,6 +1156,9 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } await this.initialize(); + await assertEnvironmentResourcesStable(this.options.resources ?? []).catch(() => { + throw new WorkspaceToolError('Environment resource changed before command dispatch', 'REGISTRATION_INVALID'); + }); const root = workspaceRoot ?? this.canonicalRoot!; let cwd: string; try { @@ -883,6 +1176,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ReturnType >; try { + const commandConfig = customConfig ?? (await this.rootCommandConfig()); const credentialEnvironment = await this.options.maskedEnvironment?.resolve(signal, cwd); const sandboxedCommand = this.options.maskedEnvironment?.wrapCommand @@ -894,7 +1188,8 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox : request.command; wrapped = await this.withTemporaryHostEnvironment( { - ...TRUSTED_GIT_ENVIRONMENT, + ...environmentResourceVariables(this.options.resources ?? []), + ...this.gitEnvironment, ...(credentialEnvironment ?? {}), ...this.scratchSelectorEnvironment(sandboxScratchDirectory), }, @@ -904,7 +1199,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox this.platform === 'win32' ? undefined : (this.options.shellPath ?? '/bin/bash'), - customConfig, + commandConfig, signal, cwd, { commandId, commandText: request.command }, @@ -997,17 +1292,21 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox cwd, env: { ...wrapped.env, + ...environmentResourceVariables(this.options.resources ?? []), ...this.scratchEnvironment(), ...trustedEnvironment, - ...TRUSTED_GIT_CONFIG_ENTRIES, + ...this.gitEnvironment, GIT_CONFIG_COUNT: wrapped.env.GIT_CONFIG_COUNT ?? - TRUSTED_GIT_CONFIG_COUNT, + this.gitEnvironment.GIT_CONFIG_COUNT, GIT_CONFIG_GLOBAL: this.platform === 'win32' ? 'NUL' : '/dev/null', GIT_CONFIG_NOSYSTEM: '1', + ...(this.gitGuardDirectory ? { + PATH: `${this.gitGuardDirectory}:${wrapped.env.PATH || this.environment.PATH || '/usr/bin:/bin'}`, + } : {}), }, detached: this.platform !== 'win32', shell: false, @@ -1026,28 +1325,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } let settled = false; let timedOut = false; - let outputBytes = 0; - let truncated = false; - const stdout: Buffer[] = []; - const stderr: Buffer[] = []; - const append = (target: Buffer[], chunk: Buffer): void => { - const remaining = outputLimit - outputBytes; - if (remaining <= 0) { - truncated = true; - return; - } - const accepted = chunk.subarray(0, remaining); - target.push(accepted); - outputBytes += accepted.byteLength; - if (accepted.byteLength !== chunk.byteLength) - truncated = true; - }; - child.stdout.on('data', (chunk: Buffer) => - append(stdout, chunk), - ); - child.stderr.on('data', (chunk: Buffer) => - append(stderr, chunk), - ); + const stdout = new OutputBuffer(outputLimit); + const stderr = new OutputBuffer(outputLimit); + child.stdout.on('data', (chunk: Buffer) => stdout.append(chunk)); + child.stderr.on('data', (chunk: Buffer) => stderr.append(chunk)); const abort = (): void => { if (settled) return; this.killCommandTree(child); @@ -1096,29 +1377,17 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); return; } - const stdoutValue = boundedUtf8( - Buffer.concat(stdout), - outputLimit, - ); - const stderrBudget = Math.max( - 0, - outputLimit - Buffer.byteLength(stdoutValue), - ); - const rawStderr = Buffer.concat(stderr).toString('utf8'); - let annotatedStderr = rawStderr; try { - annotatedStderr = - this.manager.annotateStderrWithSandboxFailures( - commandId, - rawStderr, + // SRT appends violations to its input. Capture them in the same bounded stderr window. + stderr.append( + Buffer.from( + this.manager.annotateStderrWithSandboxFailures(commandId, ''), + ), ); } catch { // Preserve the bounded child error if optional violation annotation fails. } - const stderrValue = boundedUtf8( - Buffer.from(annotatedStderr), - stderrBudget, - ); + const output = renderCommandOutput(stdout, stderr, outputLimit); resolvePromise({ protocolVersion: BRIDGE_PROTOCOL_VERSION, operation: 'execute_command', @@ -1128,11 +1397,7 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ? null : this.protocolExitCode(code), ...(childSignal ? { signal: childSignal } : {}), - stdout: stdoutValue, - stderr: stderrValue, - truncated: - truncated || - Buffer.byteLength(annotatedStderr) > stderrBudget, + ...output, timedOut, }); }); @@ -1238,6 +1503,24 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox ); } + private async createLinkedWorktreeGitGuard(scratchDirectory: string): Promise { + if (this.gitGuardDirectory) { + throw new Error('Linked worktree Git guard cleanup is still pending'); + } + // Unlike scratch, this sibling directory is not in filesystem.allowWrite. + const directory = await mkdtemp(join(dirname(scratchDirectory), 'librechat-code-git-')); + this.gitGuardDirectory = directory; + await assertPrivateStorageAncestors(directory); + await writeLinkedWorktreeGitGuard(directory); + return directory; + } + + private async removeLinkedWorktreeGitGuard(): Promise { + if (!this.gitGuardDirectory) return; + await rm(this.gitGuardDirectory, { recursive: true, force: true }); + this.gitGuardDirectory = undefined; + } + private async removeScratchDirectory(): Promise { const scratchDirectory = this.scratchDirectory; if (!scratchDirectory) return; @@ -1281,6 +1564,10 @@ export class NativeSrtWorkspaceCommandSandbox implements WorkspaceCommandSandbox } this.initialized = undefined; this.canonicalRoot = undefined; - await this.removeScratchDirectory(); + try { + await this.removeLinkedWorktreeGitGuard(); + } finally { + await this.removeScratchDirectory(); + } } } diff --git a/packages/code/src/output.test.ts b/packages/code/src/output.test.ts new file mode 100644 index 00000000..f78b0993 --- /dev/null +++ b/packages/code/src/output.test.ts @@ -0,0 +1,353 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { OutputBuffer, renderCommandOutput } from './output.js'; + +function collect( + content: Buffer, + budget: number, + chunkSize = content.length || 1 +): OutputBuffer { + const output = new OutputBuffer(budget); + for (let offset = 0; offset < content.length; offset += chunkSize) { + output.append(content.subarray(offset, offset + chunkSize)); + } + return output; +} + +function assertBounded(text: string, budget: number): void { + assert.ok(Buffer.byteLength(text) <= budget); + assert.equal(Buffer.from(text).toString('utf8'), text); +} + +test('small output stays byte-for-byte unchanged, including split UTF-8 and exact limits', () => { + for (const content of ['', '1234567', 'hello\n世界🌍\n']) { + const bytes = Buffer.from(content); + const budget = Math.max(1, bytes.length); + for (const chunkSize of [1, 2, 7, 64]) { + assert.deepEqual(collect(bytes, budget, chunkSize).render(budget), { + text: content, + truncated: false, + }); + } + } +}); + +test('a rolling suffix keeps the last summary and reports the exact dropped-byte count', () => { + const budget = 128; + const content = Buffer.from( + 'START\n' + 'x'.repeat(10_000) + '\n42 tests passed\n' + ); + for (const chunkSize of [1, 3, 63, 64, 65, 127, 1024, content.length]) { + const result = collect(content, budget, chunkSize).render(budget); + assert.equal(result.truncated, true); + assert.ok(result.text.startsWith('START\n')); + assert.ok(result.text.endsWith('\n42 tests passed\n')); + const marker = /\n\[\.\.\. (\d+) bytes omitted \.\.\.\]\n/.exec( + result.text + ); + assert.ok(marker); + assert.equal( + Number(marker[1]), + content.length - + Buffer.byteLength(result.text.replace(marker[0], '')) + ); + assertBounded(result.text, budget); + } +}); + +test('either quiet stream donates its budget, and stdout cannot starve stderr', () => { + const budget = 256; + const empty = collect(Buffer.alloc(0), budget); + const exact = collect(Buffer.alloc(budget, 'a'), budget, 1); + assert.deepEqual(renderCommandOutput(exact, empty, budget), { + stdout: 'a'.repeat(budget), + stderr: '', + truncated: false, + }); + assert.deepEqual(renderCommandOutput(empty, exact, budget), { + stdout: '', + stderr: 'a'.repeat(budget), + truncated: false, + }); + const stdout = collect( + Buffer.from('OUT\n' + 'x'.repeat(4096) + '\nstdout summary'), + budget + ); + const stderr = collect( + Buffer.from('ERR\n' + 'y'.repeat(4096) + '\nstderr summary'), + budget + ); + const result = renderCommandOutput(stdout, stderr, budget); + assert.ok(result.stdout.startsWith('OUT\n')); + assert.ok(result.stderr.startsWith('ERR\n')); + assert.ok(result.stdout.endsWith('stdout summary')); + assert.ok(result.stderr.endsWith('stderr summary')); + assert.ok(result.stdout.includes('bytes omitted')); + assert.ok(result.stderr.includes('bytes omitted')); + assert.ok( + Buffer.byteLength(result.stdout) + Buffer.byteLength(result.stderr) <= + budget + ); + assert.equal(result.truncated, true); + + const shortError = collect(Buffer.from('late failure'), budget); + assert.equal( + renderCommandOutput(stdout, shortError, budget).stderr, + 'late failure' + ); + assert.equal( + renderCommandOutput(shortError, stdout, budget).stdout, + 'late failure' + ); +}); + +test('partly filled buffers retain final summaries on either stream', () => { + const capacity = 256; + const summary = 'late stderr summary\n'; + const content = Buffer.from('y'.repeat(129 - summary.length) + summary); + for (const chunkSize of [1, 7, 128, content.length]) { + const quiet = collect(content, capacity, chunkSize); + const noisy = collect(Buffer.alloc(20_000, 'x'), capacity, chunkSize); + for (const [stdout, stderr, stream] of [ + [noisy, quiet, 'stderr'], + [quiet, noisy, 'stdout'], + ] as const) { + const result = renderCommandOutput(stdout, stderr, capacity); + assert.ok(result[stream].endsWith(summary)); + assert.ok( + Buffer.byteLength(result.stdout) + + Buffer.byteLength(result.stderr) <= + capacity + ); + } + } +}); + +test('rendered windows match logical stream ends across capture and allowance boundaries', () => { + for (const capacity of [32, 63, 64, 127, 128, 255, 256, 257]) { + const half = Math.floor(capacity / 2); + for (const length of [ + 0, + 1, + half - 1, + half, + half + 1, + capacity - 1, + capacity, + capacity + 1, + capacity * 4, + ]) { + const content = Buffer.from( + Array.from({ length }, (_, index) => + String.fromCharCode(65 + (index % 26)) + ).join('') + ); + for (const budget of [ + 0, + 1, + Math.floor(half / 2), + half, + capacity - 1, + capacity, + ]) { + let expected = content.toString('utf8'); + if (length > budget) { + const marker = (bytes: number): string => + `\n[... ${bytes} bytes omitted ...]\n`; + const markerBytes = Buffer.byteLength(marker(length)); + const includeMarker = budget >= markerBytes; + const remaining = includeMarker + ? budget - markerBytes + : budget; + const prefix = Math.floor(remaining / 2); + const suffix = Math.ceil(remaining / 2); + expected = + content.subarray(0, prefix).toString('utf8') + + (includeMarker + ? marker(length - prefix - suffix) + : '') + + content.subarray(length - suffix).toString('utf8'); + } + for (const chunkSize of [1, half, capacity + 7]) { + assert.deepEqual( + collect(content, capacity, chunkSize).render(budget), + { + text: expected, + truncated: length > budget, + }, + `capacity ${capacity}, length ${length}, budget ${budget}, chunk ${chunkSize}` + ); + } + } + } + } +}); + +test('UTF-8 summaries survive capture transitions and split code-point chunks', () => { + const summary = 'end 世界🌍\n'; + for (const capacity of [128, 255, 256, 257]) { + const half = Math.floor(capacity / 2); + for (const length of [ + half - 1, + half, + half + 1, + capacity - 1, + capacity, + capacity + 1, + ]) { + const content = Buffer.from( + 'x'.repeat(length - Buffer.byteLength(summary)) + summary + ); + for (const chunkSize of [1, 2, content.length]) { + const quiet = collect(content, capacity, chunkSize); + const noisy = collect( + Buffer.alloc(capacity * 4, 'y'), + capacity, + chunkSize + ); + for (const [stdout, stderr, stream] of [ + [noisy, quiet, 'stderr'], + [quiet, noisy, 'stdout'], + ] as const) { + const result = renderCommandOutput( + stdout, + stderr, + capacity + ); + assert.ok(result[stream].endsWith(summary)); + assert.ok(!result[stream].includes('\ufffd')); + assert.ok( + Buffer.byteLength(result.stdout) + + Buffer.byteLength(result.stderr) <= + capacity + ); + } + } + } + } +}); + +test('budgeting includes UTF-8 boundaries and marker overhead for every small limit', () => { + const content = Buffer.from('世界🌍 café Ελληνικά\n'.repeat(100)); + for (let budget = 1; budget <= 256; budget += 1) { + const result = collect(content, budget, 1).render(budget); + assertBounded(result.text, budget); + assert.equal(result.truncated, true); + assert.ok( + !result.text.includes('\ufffd'), + `split code point at budget ${budget}` + ); + const marker = /\n\[\.\.\. (\d+) bytes omitted \.\.\.\]\n/.exec( + result.text + ); + if (marker) { + assert.equal( + Number(marker[1]), + content.length - + Buffer.byteLength(result.text.replace(marker[0], '')) + ); + } + const streams = renderCommandOutput( + collect(content, budget, 7), + collect(content, budget, 3), + budget + ); + assert.ok( + Buffer.byteLength(streams.stdout) + + Buffer.byteLength(streams.stderr) <= + budget + ); + assert.ok( + !streams.stdout.includes('\ufffd') && + !streams.stderr.includes('\ufffd') + ); + } +}); + +test('malformed output cannot expand past the byte budget during decoding', () => { + for (const content of [ + Buffer.alloc(1024, 0xff), + Buffer.from([0xf0, 0x80, 0xff, 0xc2, 0xa2, 0xe0, 0xa0]), + Buffer.from('a\ufffdb'.repeat(300)), + ]) { + for (let budget = 1; budget <= 128; budget += 1) { + const result = collect(content, budget, 1).render(budget); + assertBounded(result.text, budget); + if (content.length > budget) assert.equal(result.truncated, true); + } + } +}); + +test('short decoded output is preserved when replacement characters fit the combined budget', () => { + const budget = 32; + assert.deepEqual( + renderCommandOutput( + collect(Buffer.from([0xff, 0xff]), budget), + collect(Buffer.from('child failure'), budget), + budget + ), + { stdout: '\ufffd\ufffd', stderr: 'child failure', truncated: false } + ); +}); + +test('quiet-stream donation counts decoded bytes beside noisy output', () => { + for (const bytes of [ + Buffer.from([0xff, 0xff]), + Buffer.from([0x80, 0x80]), + Buffer.from([0xf0, 0x9f]), + Buffer.from([0x61, 0xff, 0x62]), + ]) { + const decoded = bytes.toString('utf8'); + for (const budget of [16, 31, 32, 63, 64, 128]) { + const quiet = collect(bytes, budget, 1); + const noisy = collect(Buffer.alloc(budget * 4, 'x'), budget, 7); + for (const [stdout, stderr, stream] of [ + [noisy, quiet, 'stderr'], + [quiet, noisy, 'stdout'], + ] as const) { + const result = renderCommandOutput(stdout, stderr, budget); + assert.equal(result[stream], decoded); + assert.equal(result.truncated, true); + assert.ok( + Buffer.byteLength(result.stdout) + + Buffer.byteLength(result.stderr) <= + budget + ); + } + } + } +}); + +test('tiny budgets still flag truncation and give stderr the odd byte', () => { + for (let budget = 1; budget < 32; budget += 1) { + const result = renderCommandOutput( + collect(Buffer.from('a'.repeat(256)), budget), + collect(Buffer.from('b'.repeat(256)), budget), + budget + ); + assert.equal(result.truncated, true); + assert.ok(result.stderr.length > 0); + assert.ok( + Buffer.byteLength(result.stdout) + + Buffer.byteLength(result.stderr) <= + budget + ); + } +}); + +test('copies bounded windows instead of retaining or repeatedly concatenating source chunks', () => { + const budget = 128; + const output = new OutputBuffer(budget); + const oversized = Buffer.alloc(1024 * 1024, 'a'); + output.append(oversized); + oversized.fill('z'); + const byte = Buffer.from('b'); + for (let i = 0; i < 100_000; i += 1) output.append(byte); + byte[0] = 0x7a; + const result = output.render(budget); + assert.ok(result.text.startsWith('a'.repeat(40))); + assert.ok(result.text.endsWith('b'.repeat(40))); + assert.ok(!result.text.includes('z')); + assert.equal(output.bytes, 1024 * 1024 + 100_000); + assertBounded(result.text, budget); +}); diff --git a/packages/code/src/output.ts b/packages/code/src/output.ts new file mode 100644 index 00000000..4cb34211 --- /dev/null +++ b/packages/code/src/output.ts @@ -0,0 +1,182 @@ +interface OutputWindow { + text: string; + bytes: number; +} + +function utf8Window( + buffer: Buffer, + length: number, + tail: boolean +): OutputWindow { + let start = tail ? buffer.length - length : 0; + let end = tail ? buffer.length : length; + if (tail) { + while (start < end && (buffer[start] & 0xc0) === 0x80) start += 1; + } else if (end > 0) { + let last = end - 1; + while (last > 0 && (buffer[last] & 0xc0) === 0x80) last -= 1; + const lead = buffer[last]; + const width = + lead >= 0xc2 && lead <= 0xdf + ? 2 + : lead >= 0xe0 && lead <= 0xef + ? 3 + : lead >= 0xf0 && lead <= 0xf4 + ? 4 + : 1; + if (end - last < width) end = last; + } + return { + text: buffer.subarray(start, end).toString('utf8'), + bytes: end - start, + }; +} + +function boundedWindow( + buffer: Buffer, + budget: number, + tail: boolean +): OutputWindow { + const length = Math.min(buffer.length, budget); + const window = utf8Window(buffer, length, tail); + if (Buffer.byteLength(window.text) <= budget) return window; + + // Malformed bytes expand to replacement characters. Valid UTF-8 takes the fast path. + let low = 0; + let high = length; + while (low < high) { + const middle = Math.ceil((low + high) / 2); + if ( + Buffer.byteLength(utf8Window(buffer, middle, tail).text) <= budget + ) { + low = middle; + } else { + high = middle - 1; + } + } + return utf8Window(buffer, low, tail); +} + +/** Copies a prefix and rolling suffix, never retaining child-process chunk buffers. */ +export class OutputBuffer { + private readonly headCapacity: number; + private readonly tailCapacity: number; + private head?: Buffer; + private tail?: Buffer; + private headLength = 0; + private tailLength = 0; + private tailOffset = 0; + private totalBytes = 0; + + constructor(capacity: number) { + this.headCapacity = Math.floor(capacity / 2); + this.tailCapacity = capacity - this.headCapacity; + } + + get bytes(): number { + return this.totalBytes; + } + + append(chunk: Buffer): void { + this.totalBytes += chunk.length; + const headBytes = Math.min( + chunk.length, + this.headCapacity - this.headLength + ); + if (headBytes > 0) { + this.head ??= Buffer.allocUnsafe(this.headCapacity); + chunk.copy(this.head, this.headLength, 0, headBytes); + this.headLength += headBytes; + } + const remaining = chunk.length - headBytes; + if (remaining === 0) return; + this.tail ??= Buffer.allocUnsafe(this.tailCapacity); + if (remaining >= this.tailCapacity) { + chunk.copy(this.tail, 0, chunk.length - this.tailCapacity); + this.tailOffset = 0; + this.tailLength = this.tailCapacity; + return; + } + const first = Math.min(remaining, this.tailCapacity - this.tailOffset); + chunk.copy(this.tail, this.tailOffset, headBytes, headBytes + first); + chunk.copy(this.tail, 0, headBytes + first); + this.tailOffset = (this.tailOffset + remaining) % this.tailCapacity; + this.tailLength = Math.min( + this.tailCapacity, + this.tailLength + remaining + ); + } + + private suffix(): Buffer { + if (!this.tail) return Buffer.alloc(0); + if (this.tailLength < this.tailCapacity) + return this.tail.subarray(0, this.tailLength); + if (this.tailOffset === 0) return this.tail; + return Buffer.concat([ + this.tail.subarray(this.tailOffset), + this.tail.subarray(0, this.tailOffset), + ]); + } + + render(budget: number): { text: string; truncated: boolean } { + const head = this.head?.subarray(0, this.headLength) ?? Buffer.alloc(0); + const tail = this.suffix(); + // Before overflow the buffers are adjacent; either window may cross their boundary. + const contiguous = + this.totalBytes === head.length + tail.length + ? tail.length === 0 + ? head + : Buffer.concat([head, tail]) + : undefined; + if (contiguous && this.totalBytes <= budget) { + const text = contiguous.toString('utf8'); + if (Buffer.byteLength(text) <= budget) + return { text, truncated: false }; + } + const marker = (bytes: number): string => + `\n[... ${bytes} bytes omitted ...]\n`; + const markerBytes = Buffer.byteLength(marker(this.totalBytes)); + const includeMarker = budget >= markerBytes; + const contentBudget = includeMarker ? budget - markerBytes : budget; + const prefix = boundedWindow( + contiguous ?? head, + Math.floor(contentBudget / 2), + false + ); + const suffix = boundedWindow( + contiguous ?? tail, + Math.ceil(contentBudget / 2), + true + ); + const omitted = this.totalBytes - prefix.bytes - suffix.bytes; + return { + text: + prefix.text + + (includeMarker ? marker(omitted) : '') + + suffix.text, + truncated: omitted > 0, + }; + } +} + +/** Each stream can use the whole budget when the other is quiet; stderr cannot be starved. */ +export function renderCommandOutput( + stdout: OutputBuffer, + stderr: OutputBuffer, + budget: number +): { + stdout: string; + stderr: string; + truncated: boolean; +} { + let err = stderr.render(Math.ceil(budget / 2)); + const out = stdout.render(budget - Buffer.byteLength(err.text)); + if (!out.truncated) { + err = stderr.render(budget - Buffer.byteLength(out.text)); + } + return { + stdout: out.text, + stderr: err.text, + truncated: out.truncated || err.truncated, + }; +} diff --git a/packages/code/src/process-lock.test.ts b/packages/code/src/process-lock.test.ts index 3594a1a9..77091db3 100644 --- a/packages/code/src/process-lock.test.ts +++ b/packages/code/src/process-lock.test.ts @@ -17,25 +17,60 @@ test( const child = spawn( process.execPath, [ + '--expose-gc', '--input-type=module', '-e', ` + import { setTimeout as delay } from 'node:timers/promises'; import { withProcessLock } from ${JSON.stringify(moduleUrl)}; await withProcessLock(${JSON.stringify(path)}, async () => { - process.stdout.write('locked'); - await new Promise(() => { setInterval(() => {}, 1000); }); + setImmediate(() => { + global.gc(); + setImmediate(() => process.send('locked')); + }); + // The timer retains its resolver and the lock-owning async frame. + await delay(60_000); }); `, ], - { stdio: ['ignore', 'pipe', 'pipe'] } + { stdio: ['ignore', 'pipe', 'pipe', 'ipc'] } ); + let stdout = ''; + let stderr = ''; + child.stdout!.setEncoding('utf8').on('data', (chunk: string) => { + stdout = (stdout + chunk).slice(-4096); + }); + child.stderr!.setEncoding('utf8').on('data', (chunk: string) => { + stderr = (stderr + chunk).slice(-4096); + }); const closed = once(child, 'close'); t.after(async () => { child.kill('SIGKILL'); - await closed; - await rm(directory, { recursive: true, force: true }); + try { + await closed; + } finally { + t.diagnostic( + JSON.stringify({ + stdout, + stderr, + exitCode: child.exitCode, + signal: child.signalCode, + }) + ); + await rm(directory, { recursive: true, force: true }); + } }); - await once(child.stdout!, 'data'); + await Promise.race([ + once(child, 'message', { signal: t.signal }).then(([marker]) => { + assert.equal(marker, 'locked'); + t.diagnostic('Lock owner ready: locked (after GC)'); + }), + closed.then(([code, signal]) => { + throw new Error( + `Lock owner exited before readiness: ${code}/${signal}` + ); + }), + ]); let entered = false; await assert.rejects( withProcessLock( @@ -47,8 +82,10 @@ test( ) ); assert.equal(entered, false); + assert.equal(child.exitCode, null); + assert.equal(child.signalCode, null); child.kill('SIGKILL'); - await closed; + assert.deepEqual(await closed, [null, 'SIGKILL']); await withProcessLock( path, async () => { diff --git a/packages/code/src/process-lock.ts b/packages/code/src/process-lock.ts index aeca561a..6e456d50 100644 --- a/packages/code/src/process-lock.ts +++ b/packages/code/src/process-lock.ts @@ -5,6 +5,7 @@ import { setTimeout as delay } from 'node:timers/promises'; const LOCK_EX = 2; const LOCK_NB = 4; const LOCK_UN = 8; +export class ProcessLockBusyError extends Error {} let binding: | Promise<{ flock: (fd: number, operation: number) => number; @@ -26,7 +27,8 @@ async function lockBinding() { export async function withProcessLock( path: string, operation: () => Promise, - signal?: AbortSignal + signal?: AbortSignal, + wait = true, ): Promise { signal?.throwIfAborted(); if (process.platform !== 'darwin' && process.platform !== 'linux') { @@ -48,6 +50,7 @@ export async function withProcessLock( `Conversation worktree lock failed with errno ${errno}` ); } + if (!wait) throw new ProcessLockBusyError('Process lock is active'); await delay(50, undefined, { signal }); } signal?.throwIfAborted(); @@ -57,3 +60,9 @@ export async function withProcessLock( await handle.close(); } } + +/** Non-blocking maintenance admission. Never masks operation errors as lock contention. */ +export async function tryWithProcessLock(path: string, operation: () => Promise, signal?: AbortSignal): Promise<{ acquired: true; value: T } | { acquired: false }> { + try { return { acquired: true, value: await withProcessLock(path, operation, signal, false) }; } + catch (error) { if (error instanceof ProcessLockBusyError) return { acquired: false }; throw error; } +} diff --git a/packages/code/src/projects.ts b/packages/code/src/projects.ts index 1e2fcbd3..e91694f3 100644 --- a/packages/code/src/projects.ts +++ b/packages/code/src/projects.ts @@ -4,6 +4,8 @@ import { lstat, opendir, realpath } from 'node:fs/promises'; import { isAbsolute, relative, resolve, sep } from 'node:path'; import { promisify } from 'node:util'; +import { boundedBranch, boundedHead } from './protocol.js'; + const exec = promisify(execFile); const skipped = new Set(['node_modules', 'vendor']); @@ -221,16 +223,8 @@ export async function discoverProjects( ); const path = rel.split(sep).join('/') || '.'; const normalizedRemote = remote ? projectRemote(remote) : null; - const validBranch = - branch && - branch.length <= 256 && - !/[\x00-\x1f\x7f]/.test(branch) - ? branch - : null; - const validHead = - head && /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(head) - ? head - : null; + const validBranch = boundedBranch(branch); + const validHead = boundedHead(head); if ( (remote !== null && normalizedRemote === null) || (branch !== null && validBranch === null) || diff --git a/packages/code/src/protocol.test.ts b/packages/code/src/protocol.test.ts index 5c26c462..25c00cf3 100644 --- a/packages/code/src/protocol.test.ts +++ b/packages/code/src/protocol.test.ts @@ -8,15 +8,30 @@ import { isSupportedBridgeArtifactName, isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, + isWorkspaceToolErrorCode, isWorkspaceToolRequest, isWorkspaceToolResult, + WORKSPACE_TOOL_ERROR_CODE_FALLBACKS, workspaceIsolationKey, + workspaceIsolationKeysConflict, + workspaceIsolationParent, } from './protocol.js'; import type { WorkspaceEditFileRequest, WorkspacePreviewEditRequest, } from './protocol.js'; +test('every added workspace error code falls back to a legacy code', () => { + assert.equal(isWorkspaceToolErrorCode('NOT_FOUND'), true); + assert.equal(isWorkspaceToolErrorCode('MISSING'), false); + for (const [code, legacy] of Object.entries(WORKSPACE_TOOL_ERROR_CODE_FALLBACKS)) { + assert.equal(isWorkspaceToolErrorCode(code), true); + assert.equal(isWorkspaceToolErrorCode(legacy), true); + assert.equal(legacy in WORKSPACE_TOOL_ERROR_CODE_FALLBACKS, false); + } + assert.equal(WORKSPACE_TOOL_ERROR_CODE_FALLBACKS.NOT_FOUND, 'INVALID_PATH'); +}); + test('accepts gateway directory markers as artifacts', () => { assert.equal(isSupportedBridgeArtifactName('.dirkeep'), true); assert.equal(isSupportedBridgeArtifactName('nested/.dirkeep'), true); @@ -543,6 +558,164 @@ test('workspace mutations accept bounded UTF-8 requests and exact result shapes' ); }); +test('edit matching and replaceAll are opt-in and reported only when requested', () => { + const tolerant = { + protocolVersion: 1 as const, + operation: 'edit_file' as const, + workspaceId: 'primary', + path: 'notes.txt', + matching: 'tolerant' as const, + edits: [ + { oldText: 'hello', newText: 'goodbye' }, + { oldText: 'world', newText: 'BYOM', replaceAll: true }, + ], + }; + const result = { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + replacements: 2, + bytesWritten: 12, + }; + const matches = [ + { strategy: 'line-trimmed', occurrences: 1 }, + { strategy: 'exact', occurrences: 3 }, + ]; + assert.equal(isWorkspaceToolRequest(tolerant), true); + assert.equal(isWorkspaceToolRequest({ ...tolerant, operation: 'preview_edit' }), true); + assert.equal(isWorkspaceToolRequest({ ...tolerant, matching: 'fuzzy' }), false); + assert.equal( + isWorkspaceToolRequest({ + ...tolerant, + edits: [{ oldText: 'a', newText: 'b', replaceAll: 'yes' }], + }), + false, + ); + assert.equal( + isWorkspaceToolRequest({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + oldText: 'a', + newText: 'b', + replaceAll: true, + }), + false, + 'replaceAll is only accepted per batch edit', + ); + + assert.equal(isWorkspaceToolResult(tolerant, { ...result, matches }), true); + assert.equal(isWorkspaceToolResult(tolerant, result), false, 'an opted-in request must report matches'); + assert.equal( + isWorkspaceToolResult(tolerant, { ...result, matches: matches.slice(0, 1) }), + false, + ); + assert.equal( + isWorkspaceToolResult(tolerant, { + ...result, + matches: [matches[0], { strategy: 'exact', occurrences: 0 }], + }), + false, + ); + assert.equal( + isWorkspaceToolResult(tolerant, { + ...result, + matches: [{ strategy: 'exact', occurrences: 2 }, matches[1]], + }), + false, + 'only replaceAll edits may replace more than one location', + ); + + const exactReplaceAll = { + ...tolerant, + matching: undefined, + edits: [{ oldText: 'a', newText: 'b', replaceAll: true }], + }; + delete exactReplaceAll.matching; + assert.equal( + isWorkspaceToolResult(exactReplaceAll, { + ...result, + replacements: 1, + matches: [{ strategy: 'line-trimmed', occurrences: 2 }], + }), + false, + 'without tolerant matching every edit matches exactly', + ); + + const legacy = { ...tolerant, edits: [{ oldText: 'a', newText: 'b' }] }; + delete (legacy as { matching?: string }).matching; + assert.equal( + isWorkspaceToolResult(legacy, { + ...result, + replacements: 1, + matches: [{ strategy: 'exact', occurrences: 1 }], + }), + false, + 'legacy requests never receive a matches field', + ); +}); + +test('edit features advertise any unique subset of the known features', () => { + const valid = { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: 1, + operations: ['read_file', 'edit_file'], + workspaces: [{ id: 'primary' }], + editFileModes: ['single', 'batch'], + }, + }; + const withFeatures = (editFileFeatures: unknown) => ({ + ...valid, + workspaceTools: { ...valid.workspaceTools, editFileFeatures }, + }); + for (const features of [ + ['expected_base_sha256'], + ['tolerant_match'], + ['expected_base_sha256', 'tolerant_match', 'replace_all'], + ]) { + assert.equal(isValidBridgeWorkerCapabilities(withFeatures(features)), true, features.join(',')); + } + for (const features of [[], ['fuzzy'], ['replace_all', 'replace_all']]) { + assert.equal(isValidBridgeWorkerCapabilities(withFeatures(features)), false, features.join(',')); + } +}); + +test('preview-only workers may advertise tolerant matching and replace-all, but not edit-only hashes', () => { + const previewOnly = { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceTools: { + protocolVersion: 1, + operations: ['read_file', 'preview_edit'], + workspaces: [{ id: 'primary' }], + editFileModes: ['single', 'batch'], + }, + }; + for (const features of [ + ['tolerant_match'], + ['replace_all'], + ['tolerant_match', 'replace_all'], + ]) { + assert.equal(isValidBridgeWorkerCapabilities({ + ...previewOnly, + workspaceTools: { ...previewOnly.workspaceTools, editFileFeatures: features }, + }), true, features.join(',')); + } + assert.equal(isValidBridgeWorkerCapabilities({ + ...previewOnly, + workspaceTools: { + ...previewOnly.workspaceTools, + editFileFeatures: ['expected_base_sha256', 'tolerant_match'], + }, + }), false); +}); + test('workspace commands require bounded sandbox inputs and outputs', () => { const request = { protocolVersion: 1 as const, @@ -816,3 +989,62 @@ test('bridge artifact policy and media types match the hardened gateway contract assert.equal(bridgeArtifactMediaType('reports/result.json'), 'application/json'); assert.equal(bridgeArtifactMediaType('Dockerfile'), 'application/octet-stream'); }); + +test('linked-worktree lanes nest beneath their checkout key and conflict only with it', () => { + const instanceId = 'a'.repeat(64); + const lane = workspaceIsolationKey('repo', undefined, 'task-a'); + const sibling = workspaceIsolationKey('repo', undefined, 'task-b'); + const nested = workspaceIsolationKey('repo', instanceId, 'task-a'); + assert.notEqual(lane, workspaceIsolationKey('repo')); + assert.notEqual(lane, workspaceIsolationKey('repo\0task-a')); + assert.notEqual(nested, lane); + assert.equal(workspaceIsolationParent(lane), 'repo'); + assert.equal(workspaceIsolationParent(nested), workspaceIsolationKey('repo', instanceId)); + assert.equal(workspaceIsolationParent('repo'), undefined); + assert.equal(workspaceIsolationParent(workspaceIsolationKey('repo', instanceId)), undefined); + assert.equal(workspaceIsolationKeysConflict(lane, 'repo'), true); + assert.equal(workspaceIsolationKeysConflict('repo', lane), true); + assert.equal(workspaceIsolationKeysConflict(lane, lane), true); + assert.equal(workspaceIsolationKeysConflict(lane, sibling), false); + assert.equal(workspaceIsolationKeysConflict(nested, 'repo'), false); + assert.equal(workspaceIsolationKeysConflict(nested, workspaceIsolationKey('repo', instanceId)), true); +}); + +test('workspace requests accept only a single safe worktree name', () => { + const base = { protocolVersion: 1, operation: 'read_file', workspaceId: 'repo', path: 'README.md' }; + for (const worktree of ['task-a', 'fix_16464', 'v2.0']) { + assert.equal(isWorkspaceToolRequest({ ...base, worktree }), true, worktree); + } + for (const worktree of ['', '.hidden', '..', 'a/b', 'a\\b', 'task.lock', 'x'.repeat(129), 7, null]) { + assert.equal(isWorkspaceToolRequest({ ...base, worktree }), false, String(worktree)); + } + const programmatic = (workspace_worktree: string) => ({ + headers: {}, + body: { + language: 'bash', + version: '5.2.0', + session_id: 'session', + files: [{ name: 'main.sh', content: 'true' }], + workspace_worktree, + }, + }); + assert.equal(isBridgeWorkspaceProgrammaticRequest(programmatic('task-a')), true); + assert.equal(isBridgeWorkspaceProgrammaticRequest(programmatic('../x')), false); +}); + +test('workspace capabilities advertise linked-worktree scopes exactly', () => { + const capabilities = (workspaceScopes: unknown) => ({ + statefulWorkspace: false, + sandboxProfile: 'anthropic-srt', + runtimes: [], + workspaceTools: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'repo', workspaceScopes }], + }, + }); + assert.equal(isValidBridgeWorkerCapabilities(capabilities(['git_linked_worktree'])), true); + assert.equal(isValidBridgeWorkerCapabilities(capabilities(['git_worktree'])), false); + assert.equal(isValidBridgeWorkerCapabilities(capabilities([])), false); + assert.equal(isValidBridgeWorkerCapabilities(capabilities('git_linked_worktree')), false); +}); diff --git a/packages/code/src/protocol.ts b/packages/code/src/protocol.ts index c678a1af..1451986d 100644 --- a/packages/code/src/protocol.ts +++ b/packages/code/src/protocol.ts @@ -259,14 +259,51 @@ export function bridgeArtifactMediaType(name: string): string { export type BridgeProtocolVersion = typeof BRIDGE_PROTOCOL_VERSION; -/** Collision-free identity shared by scheduling and worker quarantine state. */ +/** One path segment naming a linked worktree at `/.worktrees/`. */ +export const BRIDGE_LINKED_WORKTREE_NAME_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; + +export function isValidLinkedWorktreeName(value: unknown): value is string { + return ( + typeof value === 'string' && + BRIDGE_LINKED_WORKTREE_NAME_PATTERN.test(value) && + !value.endsWith('.lock') + ); +} + +const LINKED_WORKTREE_KEY_PREFIX = '\0linked-worktree\0'; + +/** Collision-free identity shared by scheduling and worker quarantine state. + * A linked worktree lane nests beneath the key of the checkout that owns it. */ export function workspaceIsolationKey( workspaceId: string, instanceId?: string, + worktree?: string, ): string { - return instanceId === undefined + const base = instanceId === undefined ? workspaceId : `\0git-worktree\0${workspaceId}\0${instanceId}`; + return worktree === undefined + ? base + : `${LINKED_WORKTREE_KEY_PREFIX}${base}\0${worktree}`; +} + +/** The checkout key a linked-worktree lane nests beneath, or undefined for a root key. + * Scheduling treats a lane and its parent as conflicting; sibling lanes do not. */ +export function workspaceIsolationParent(key: string): string | undefined { + if (!key.startsWith(LINKED_WORKTREE_KEY_PREFIX)) return undefined; + const separator = key.lastIndexOf('\0'); + return separator <= LINKED_WORKTREE_KEY_PREFIX.length + ? undefined + : key.slice(LINKED_WORKTREE_KEY_PREFIX.length, separator); +} + +/** Two isolation keys may not execute concurrently when either nests the other. */ +export function workspaceIsolationKeysConflict(left: string, right: string): boolean { + return ( + left === right || + workspaceIsolationParent(left) === right || + workspaceIsolationParent(right) === left + ); } export type BridgeWorkspaceToolOperation = @@ -280,8 +317,73 @@ export type BridgeWorkspaceToolOperation = export type WorkspaceWriteFileMode = 'replace' | 'create'; export type WorkspaceEditFileMode = 'single' | 'batch'; -export type WorkspaceEditFileFeature = 'expected_base_sha256'; +export type WorkspaceEditFileFeature = + | 'expected_base_sha256' + | 'tolerant_match' + | 'replace_all'; +/** Every edit feature this protocol version defines, for capability validation. */ +export const WORKSPACE_EDIT_FILE_FEATURES: readonly WorkspaceEditFileFeature[] = [ + 'expected_base_sha256', + 'tolerant_match', + 'replace_all', +]; +/** `tolerant` falls back from exact matching to whitespace-tolerant strategies. */ +export type WorkspaceEditMatching = 'exact' | 'tolerant'; +export type WorkspaceEditMatchStrategy = + | 'exact' + | 'line-trimmed' + | 'whitespace-normalized' + | 'indentation-flexible'; +const WORKSPACE_EDIT_MATCH_STRATEGIES = new Set([ + 'exact', + 'line-trimmed', + 'whitespace-normalized', + 'indentation-flexible', +]); + +/** A branch name reportable to LibreChat: 1 to 256 chars, no control characters. */ +export function boundedBranch(value: unknown): string | null { + return typeof value === 'string' && + value.length > 0 && + value.length <= 256 && + // Git forbids only ASCII controls in ref names; the header must also never show Unicode + // controls, format characters (bidi overrides, zero-width) or line and paragraph separators. + !/[\x00-\x1f\x7f\p{Cc}\p{Cf}\p{Zl}\p{Zp}]/u.test(value) + ? value + : null; +} + +/** A commit id reportable to LibreChat: 40 (SHA-1) or 64 (SHA-256) lowercase hex chars. */ +export function boundedHead(value: unknown): string | null { + return typeof value === 'string' && /^[a-f0-9]{40}(?:[a-f0-9]{24})?$/.test(value) + ? value + : null; +} + export type WorkspaceListFileFeature = 'after_path'; +/** `lane_git`: `execute_command` results may carry `laneGit`. Sent only when Code API echoes it. */ +export type WorkspaceCommandResultFeature = 'lane_git'; + +/** Git state of the lane a command ran in. Never carries paths, remotes or repository content. */ +export interface WorkspaceLaneGit { + /** Checked-out branch, or null when HEAD is detached or the name is not reportable. */ + branch: string | null; + /** HEAD commit (40 or 64 lowercase hex), or null before the first commit or when unreportable. */ + head: string | null; +} + +export function isWorkspaceLaneGit(value: unknown): value is WorkspaceLaneGit { + if (typeof value !== 'object' || value === null || Array.isArray(value)) return false; + const state = value as Record; + const keys = Object.keys(state); + return ( + keys.length === 2 && + keys.includes('branch') && + keys.includes('head') && + (state.branch === null || (state.branch !== undefined && boundedBranch(state.branch) === state.branch)) && + (state.head === null || (state.head !== undefined && boundedHead(state.head) === state.head)) + ); +} export type WorkspaceProgrammaticLanguage = 'bash'; export interface BridgeWorkspaceDescriptor { @@ -292,6 +394,9 @@ export interface BridgeWorkspaceDescriptor { operations?: BridgeWorkspaceToolOperation[]; /** Worker-owned isolation schemes available beneath this selected root. */ workspaceInstances?: ['git_worktree']; + /** Scheduling scopes available beneath this root. `git_linked_worktree` gives each + * verified `.worktrees/` linked worktree its own lane. */ + workspaceScopes?: ['git_linked_worktree']; environment?: { fingerprint: string; repo?: string; @@ -312,6 +417,8 @@ export interface BridgeWorkspaceToolCapabilities { editFileFeatures?: WorkspaceEditFileFeature[]; /** Omitted by workers that cannot continue a bounded file listing. */ listFileFeatures?: WorkspaceListFileFeature[]; + /** Omitted by workers that do not report lane Git state on command results. */ + commandResultFeatures?: WorkspaceCommandResultFeature[]; /** Languages that can execute PTC replay inside a selected workspace. */ programmaticLanguages?: WorkspaceProgrammaticLanguage[]; } @@ -321,6 +428,8 @@ export interface WorkspaceReadFileRequest { operation: 'read_file'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; path: string; startLine?: number; maxLines?: number; @@ -364,6 +473,8 @@ export interface WorkspaceSearchTextRequest { operation: 'search_text'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; query: string; path?: string; maxResults?: number; @@ -389,6 +500,8 @@ export interface WorkspaceListFilesRequest { operation: 'list_files'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; path?: string; maxResults?: number; /** Continue strictly after this canonical path from a previous page. */ @@ -410,6 +523,8 @@ export interface WorkspaceWriteFileRequest { operation: 'write_file'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; path: string; content: string; /** False requires an atomic create and refuses to replace an existing file. */ @@ -430,9 +545,13 @@ interface WorkspaceEditFileRequestBase { operation: 'edit_file'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; path: string; /** Refuses the mutation unless current file bytes match this preview revision. */ expectedBaseSha256?: string; + /** Requires the `tolerant_match` edit feature. Omitted means `exact`. */ + matching?: WorkspaceEditMatching; } export interface WorkspaceSingleEditFileRequest @@ -459,6 +578,15 @@ export type WorkspaceEditFileRequest = export interface WorkspaceTextEdit { oldText: string; newText: string; + /** Replaces every match instead of requiring exactly one. Requires `replace_all`. */ + replaceAll?: boolean; +} + +/** How one edit matched. Present only when the request set `matching` or `replaceAll`. */ +export interface WorkspaceEditMatch { + strategy: WorkspaceEditMatchStrategy; + /** Locations replaced; always 1 unless the edit set `replaceAll`. */ + occurrences: number; } export interface WorkspaceEditFileResult { @@ -466,8 +594,10 @@ export interface WorkspaceEditFileResult { operation: 'edit_file'; workspaceId: string; path: string; + /** Number of edits applied, which is always the number requested. */ replacements: number; bytesWritten: number; + matches?: WorkspaceEditMatch[]; } interface WorkspacePreviewEditRequestBase { @@ -475,7 +605,11 @@ interface WorkspacePreviewEditRequestBase { operation: 'preview_edit'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; path: string; + /** Requires the `tolerant_match` edit feature. Omitted means `exact`. */ + matching?: WorkspaceEditMatching; } export interface WorkspaceSinglePreviewEditRequest @@ -506,6 +640,7 @@ export interface WorkspacePreviewEditResult { baseSha256: string; replacements: number; bytesWritten: number; + matches?: WorkspaceEditMatch[]; } export interface WorkspaceExecuteCommandRequest { @@ -513,6 +648,8 @@ export interface WorkspaceExecuteCommandRequest { operation: 'execute_command'; workspaceId: string; workspaceInstanceId?: string; + /** Linked worktree lane at `.worktrees/`; paths and cwd are relative to it. */ + worktree?: string; /** Shell source evaluated only inside the selected sandbox runtime. */ command: string; /** Portable path relative to the workspace root; defaults to '.'. */ @@ -533,6 +670,8 @@ export interface WorkspaceExecuteCommandResult { stderr: string; truncated: boolean; timedOut: boolean; + /** Present only when `commandResultFeatures` negotiated `lane_git` and Git was readable. */ + laneGit?: WorkspaceLaneGit; } export type WorkspaceToolRequest = @@ -558,6 +697,7 @@ const WORKSPACE_READ_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'path', 'startLine', 'maxLines', @@ -567,6 +707,7 @@ const WORKSPACE_SEARCH_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'query', 'path', 'maxResults', @@ -576,6 +717,7 @@ const WORKSPACE_LIST_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'path', 'maxResults', 'afterPath', @@ -585,6 +727,7 @@ const WORKSPACE_WRITE_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'path', 'content', 'overwrite', @@ -594,29 +737,35 @@ const WORKSPACE_EDIT_REQUEST_KEYS = new Set([ 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'path', 'oldText', 'newText', 'edits', 'expectedBaseSha256', + 'matching', ]); const WORKSPACE_PREVIEW_EDIT_REQUEST_KEYS = new Set([ 'protocolVersion', 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'path', 'oldText', 'newText', 'edits', + 'matching', ]); -const WORKSPACE_TEXT_EDIT_KEYS = new Set(['oldText', 'newText']); +const WORKSPACE_TEXT_EDIT_KEYS = new Set(['oldText', 'newText', 'replaceAll']); +const WORKSPACE_EDIT_MATCH_KEYS = new Set(['strategy', 'occurrences']); const WORKSPACE_COMMAND_REQUEST_KEYS = new Set([ 'environmentAction', 'protocolVersion', 'operation', 'workspaceId', 'workspaceInstanceId', + 'worktree', 'command', 'cwd', 'timeoutMs', @@ -663,6 +812,7 @@ const WORKSPACE_EDIT_RESULT_KEYS = new Set([ 'path', 'replacements', 'bytesWritten', + 'matches', ]); const WORKSPACE_PREVIEW_EDIT_RESULT_KEYS = new Set([ 'protocolVersion', @@ -674,6 +824,7 @@ const WORKSPACE_PREVIEW_EDIT_RESULT_KEYS = new Set([ 'baseSha256', 'replacements', 'bytesWritten', + 'matches', ]); const WORKSPACE_COMMAND_RESULT_KEYS = new Set([ 'protocolVersion', @@ -685,6 +836,7 @@ const WORKSPACE_COMMAND_RESULT_KEYS = new Set([ 'stderr', 'truncated', 'timedOut', + 'laneGit', ]); const WORKSPACE_SEARCH_MATCH_KEYS = new Set(['path', 'line', 'column', 'text']); @@ -730,6 +882,12 @@ export interface BridgeWorkerRegistrationResponse { supportedWorkspaceProgrammaticLanguages?: WorkspaceProgrammaticLanguage[]; /** Workspace isolation schemes this Code API understands and can route. */ supportedWorkspaceInstanceTypes?: ['git_worktree']; + /** Scheduling scopes this Code API can admit as independent lanes. */ + supportedWorkspaceScopes?: ['git_linked_worktree']; + /** Command result fields this Code API accepts in settlements. */ + supportedWorkspaceCommandResultFeatures?: WorkspaceCommandResultFeature[]; + /** Added workspace tool error codes this Code API accepts in settlements. */ + supportedWorkspaceToolErrorCodes?: WorkspaceToolErrorCode[]; } /** Administrator-visible liveness for a configured worker. Credentials, @@ -759,6 +917,32 @@ export interface BridgeWorkerCredentialResponse { expiresAt: string; } +/** The enrolled machine signs this request before Code API issues a challenge. */ +export interface BridgeRecoveryChallengeRequest { + protocolVersion: BridgeProtocolVersion; + operation: 'credential.challenge'; + serverId: string; + workerId: string; + timestamp: string; + nonce: string; + signature: string; +} + +/** A short-lived, single-use challenge for an already enrolled machine key. */ +export interface BridgeRecoveryChallengeResponse { + protocolVersion: BridgeProtocolVersion; + operation: 'credential.recover'; + serverId: string; + workerId: string; + enrollmentGeneration: string; + challenge: string; + expiresAt: string; +} + +export interface BridgeRecoveryRequest extends BridgeRecoveryChallengeResponse { + signature: string; +} + export interface BridgeSandboxRequest { body: TBody; headers: Record; @@ -777,6 +961,8 @@ export interface BridgeWorkspaceProgrammaticBody { language: 'bash'; version: string; workspace_instance_id?: string; + /** Linked worktree lane at `.worktrees/` beneath the selected checkout. */ + workspace_worktree?: string; /** Stable identity shared by every replay iteration of one execution. */ execution_id?: string; /** Declared replay tools; zero allows the worker to skip the probe pass. */ @@ -842,6 +1028,7 @@ export interface BridgeRejectedSettlement { export type WorkspaceToolErrorCode = | 'INVALID_PATH' + | 'NOT_FOUND' | 'INVALID_REQUEST' | 'READ_LIMIT_EXCEEDED' | 'WRITE_LIMIT_EXCEEDED' @@ -860,6 +1047,7 @@ export type WorkspaceToolErrorCode = const WORKSPACE_TOOL_ERROR_CODES = new Set([ 'INVALID_PATH', + 'NOT_FOUND', 'INVALID_REQUEST', 'READ_LIMIT_EXCEEDED', 'WRITE_LIMIT_EXCEEDED', @@ -877,6 +1065,15 @@ const WORKSPACE_TOOL_ERROR_CODES = new Set([ 'COMMAND_DISABLED', ]); +/** + * Codes added after the original settlement contract, each with the legacy code + * an older Code API accepts in its place. A worker reports an added code only + * after registration advertises it, and falls back if a settlement is refused. + */ +export const WORKSPACE_TOOL_ERROR_CODE_FALLBACKS: Readonly< + Partial> +> = { NOT_FOUND: 'INVALID_PATH' }; + export function isWorkspaceToolErrorCode( value: unknown, ): value is WorkspaceToolErrorCode { @@ -944,6 +1141,8 @@ export function isBridgeWorkspaceProgrammaticRequest( (body.workspace_instance_id !== undefined && (typeof body.workspace_instance_id !== 'string' || !/^[a-f0-9]{64}$/.test(body.workspace_instance_id))) || + (body.workspace_worktree !== undefined && + !isValidLinkedWorktreeName(body.workspace_worktree)) || (body.execution_id !== undefined && (typeof body.execution_id !== 'string' || !/^[A-Za-z0-9_-]{1,128}$/.test(body.execution_id))) || @@ -1119,6 +1318,44 @@ function isWithinRequestedPath(candidate: string, requested?: string): boolean { ); } +/** Whether a request asked for per-edit match reporting (and so must receive it). */ +export function workspaceEditRequestReportsMatches( + request: WorkspaceEditFileRequest | WorkspacePreviewEditRequest, +): boolean { + return ( + request.matching !== undefined || + (request.edits?.some((edit) => edit.replaceAll !== undefined) ?? false) + ); +} + +function isValidWorkspaceEditMatches( + request: WorkspaceEditFileRequest | WorkspacePreviewEditRequest, + matches: unknown, +): boolean { + if (!workspaceEditRequestReportsMatches(request)) return matches === undefined; + const edits: WorkspaceTextEdit[] = request.edits ?? [ + { oldText: request.oldText ?? '', newText: request.newText ?? '' }, + ]; + return ( + Array.isArray(matches) && + matches.length === edits.length && + matches.every((match: unknown, index) => { + if (typeof match !== 'object' || match === null) return false; + const candidate = match as Record; + return ( + hasOnlyKeys(candidate, WORKSPACE_EDIT_MATCH_KEYS) && + WORKSPACE_EDIT_MATCH_STRATEGIES.has( + candidate.strategy as WorkspaceEditMatchStrategy, + ) && + (request.matching === 'tolerant' || candidate.strategy === 'exact') && + Number.isSafeInteger(candidate.occurrences) && + Number(candidate.occurrences) >= 1 && + (edits[index]?.replaceAll === true || candidate.occurrences === 1) + ); + }) + ); +} + function isValidWorkspaceEditRequest( request: Record, ): boolean { @@ -1129,6 +1366,13 @@ function isValidWorkspaceEditRequest( ) { return false; } + if ( + request.matching !== undefined && + request.matching !== 'exact' && + request.matching !== 'tolerant' + ) { + return false; + } const edits = hasBatch ? request.edits : [{ oldText: request.oldText, newText: request.newText }]; @@ -1159,7 +1403,9 @@ function isValidWorkspaceEditRequest( candidate.oldText || typeof candidate.newText !== 'string' || Buffer.from(candidate.newText).toString('utf8') !== - candidate.newText + candidate.newText || + (candidate.replaceAll !== undefined && + typeof candidate.replaceAll !== 'boolean') ) { return false; } @@ -1196,7 +1442,8 @@ export function isWorkspaceToolRequest( !isValidBridgeWorkerId(request.workspaceId) || (request.workspaceInstanceId !== undefined && (typeof request.workspaceInstanceId !== 'string' || - !/^[a-f0-9]{64}$/.test(request.workspaceInstanceId))) + !/^[a-f0-9]{64}$/.test(request.workspaceInstanceId))) || + (request.worktree !== undefined && !isValidLinkedWorktreeName(request.worktree)) ) { return false; } @@ -1467,7 +1714,8 @@ export function isWorkspaceToolResult( result.replacements === replacements && Number.isSafeInteger(result.bytesWritten) && Number(result.bytesWritten) >= 0 && - Number(result.bytesWritten) <= BRIDGE_WORKSPACE_WRITE_MAX_BYTES + Number(result.bytesWritten) <= BRIDGE_WORKSPACE_WRITE_MAX_BYTES && + isValidWorkspaceEditMatches(request, result.matches) ); } @@ -1488,7 +1736,8 @@ export function isWorkspaceToolResult( Number(result.bytesWritten) === new TextEncoder().encode(content).byteLength + (result.hasUtf8Bom ? 3 : 0) && - Number(result.bytesWritten) <= BRIDGE_WORKSPACE_WRITE_MAX_BYTES + Number(result.bytesWritten) <= BRIDGE_WORKSPACE_WRITE_MAX_BYTES && + isValidWorkspaceEditMatches(request, result.matches) ); } @@ -1518,6 +1767,7 @@ export function isWorkspaceToolResult( /^SIG[A-Z0-9]+$/.test(result.signal))) && typeof result.truncated === 'boolean' && typeof result.timedOut === 'boolean' && + (result.laneGit === undefined || isWorkspaceLaneGit(result.laneGit)) && (result.exitCode === null ? result.timedOut === true || result.signal !== undefined : result.timedOut === false && result.signal === undefined) @@ -1612,9 +1862,20 @@ export function isValidBridgeWorkspaceToolCapabilities( if ( capabilities.editFileFeatures !== undefined && (!Array.isArray(capabilities.editFileFeatures) || - capabilities.editFileFeatures.length !== 1 || - !capabilities.operations.includes('edit_file') || - capabilities.editFileFeatures[0] !== 'expected_base_sha256') + capabilities.editFileFeatures.length < 1 || + capabilities.editFileFeatures.length > + WORKSPACE_EDIT_FILE_FEATURES.length || + (!capabilities.operations.includes('edit_file') && + !capabilities.operations.includes('preview_edit')) || + !capabilities.editFileFeatures.every((feature: unknown) => + WORKSPACE_EDIT_FILE_FEATURES.includes( + feature as WorkspaceEditFileFeature, + ) && + (feature !== 'expected_base_sha256' || + (capabilities.operations as string[]).includes('edit_file')), + ) || + new Set(capabilities.editFileFeatures).size !== + capabilities.editFileFeatures.length) ) { return false; } @@ -1629,6 +1890,16 @@ export function isValidBridgeWorkspaceToolCapabilities( return false; } + if ( + capabilities.commandResultFeatures !== undefined && + (!Array.isArray(capabilities.commandResultFeatures) || + capabilities.commandResultFeatures.length !== 1 || + !capabilities.operations.includes('execute_command') || + capabilities.commandResultFeatures[0] !== 'lane_git') + ) { + return false; + } + if ( capabilities.programmaticLanguages !== undefined && (!Array.isArray(capabilities.programmaticLanguages) || @@ -1650,6 +1921,7 @@ export function isValidBridgeWorkspaceToolCapabilities( key !== 'name' && key !== 'operations' && key !== 'workspaceInstances' && + key !== 'workspaceScopes' && key !== 'instructions' && key !== 'environment', ) || @@ -1660,6 +1932,10 @@ export function isValidBridgeWorkspaceToolCapabilities( (!Array.isArray(descriptor.workspaceInstances) || descriptor.workspaceInstances.length !== 1 || descriptor.workspaceInstances[0] !== 'git_worktree')) || + (descriptor.workspaceScopes !== undefined && + (!Array.isArray(descriptor.workspaceScopes) || + descriptor.workspaceScopes.length !== 1 || + descriptor.workspaceScopes[0] !== 'git_linked_worktree')) || (descriptor.instructions !== undefined && (!Array.isArray(descriptor.instructions) || descriptor.instructions.length > 1 || !descriptor.instructions.every(isRepositoryInstructionDescriptor))) || (descriptor.environment !== undefined && !isValidCodeEnvironmentDescriptor(descriptor.environment)) || diff --git a/packages/code/src/root-access.ts b/packages/code/src/root-access.ts index 4aeb6100..2f049bcb 100644 --- a/packages/code/src/root-access.ts +++ b/packages/code/src/root-access.ts @@ -62,10 +62,16 @@ const openAt = nativeOpenAt const renameAt = bind( 'int renameat(int fromfd, const char *from, int tofd, const char *to)', ); +const renameExclusiveAt = bind(process.platform === 'darwin' + ? 'int renameatx_np(int fromfd, const char *from, int tofd, const char *to, unsigned int flags)' + : 'int renameat2(int fromfd, const char *from, int tofd, const char *to, unsigned int flags)'); const linkAt = bind( 'int linkat(int fromfd, const char *from, int tofd, const char *to, int flags)', ); const unlinkAt = bind('int unlinkat(int dirfd, const char *path, int flags)'); +const mkdirAt = bind('int mkdirat(int dirfd, const char *path, unsigned int mode)'); +const symlinkAt = bind('int symlinkat(const char *target, int dirfd, const char *path)'); +const readlinkAt = bind('int readlinkat(int dirfd, const char *path, void *buffer, size_t size)'); const getPath = process.platform === 'darwin' ? bind('int fcntl(int fd, int command, ...)') @@ -328,13 +334,14 @@ export class WorkspaceRootAccess { } } - install(from: string, to: string, link: boolean): void { + install(from: string, to: string, link: boolean, exclusive = false): void { const source = this.parent(from); let target: ReturnType | undefined; try { target = this.parent(to); const result = link ? linkAt!(source.fd, source.name, target.fd, target.name, 0) + : exclusive ? renameExclusiveAt!(source.fd, source.name, target.fd, target.name, process.platform === 'darwin' ? 4 : 1) : renameAt!(source.fd, source.name, target.fd, target.name); if (result !== 0) throw nativeError(); } finally { @@ -343,10 +350,27 @@ export class WorkspaceRootAccess { } } - unlink(path: string): void { + entry(path: string, operation: 'mkdir' | 'symlink' | 'readlink', target?: string, mode = 0o700): string | void { const parent = this.parent(path); try { - if (unlinkAt!(parent.fd, parent.name, 0) !== 0) throw nativeError(); + if (operation === 'readlink') { + const buffer = Buffer.alloc(4097); + const length = readlinkAt!(parent.fd, parent.name, buffer, buffer.length); + if (length < 0) throw nativeError(); + if (length === buffer.length) throw new Error('Workspace link exceeds its bounded contract'); + return buffer.subarray(0, length).toString(); + } + const result = operation === 'mkdir' + ? mkdirAt!(parent.fd, parent.name, mode) + : symlinkAt!(target!, parent.fd, parent.name); + if (result !== 0) throw nativeError(); + } finally { closeSync(parent.fd); } + } + + unlink(path: string, directory = false): void { + const parent = this.parent(path); + try { + if (unlinkAt!(parent.fd, parent.name, directory ? (process.platform === 'darwin' ? 0x80 : 0x200) : 0) !== 0) throw nativeError(); } finally { closeSync(parent.fd); } @@ -354,6 +378,14 @@ export class WorkspaceRootAccess { } const context = new AsyncLocalStorage(); +const deferredCleanup = new WeakSet(); +/** Interrupted I/O must not hold a request open while Node drains its handles. */ +export function deferWorkspaceRootCleanup(): void { + const access = context.getStore(); + if (access) deferredCleanup.add(access); +} +/** Whether filesystem adapters in this call are anchored to a held root descriptor. */ +export const holdsWorkspaceRoot = (): boolean => context.getStore() != null; export async function withWorkspaceRoot( root: string, identity: WorkspaceRootIdentity | undefined, @@ -364,7 +396,9 @@ export async function withWorkspaceRoot( try { return await context.run(access, action); } finally { - await access.close(); + const closing = access.close(); + if (deferredCleanup.delete(access)) void closing.catch(() => {}); + else await closing; } } @@ -448,6 +482,11 @@ export const rename = async (from: string, to: string): Promise => { if (access) access.install(from, to, false); else await fs.rename(from, to); }; +export const renameExclusive = async (from: string, to: string): Promise => { + const access = context.getStore(); + if (!access) throw new Error('Exclusive installation requires a selected root'); + access.install(from, to, false, true); +}; export const link = async (from: string, to: string): Promise => { const access = context.getStore(); if (access) access.install(from, to, true); @@ -458,3 +497,35 @@ export const unlink = async (path: string): Promise => { if (access) access.unlink(path); else await fs.unlink(path); }; + +/** Enumeration and entry mutation retain the selected root descriptor. */ +export const readdir = async (path: string, maxEntries = 200_000): Promise => { + const access = context.getStore(); + const handle = access ? await access.openFile(path, constants.O_RDONLY | constants.O_DIRECTORY | constants.O_NOFOLLOW) : undefined; + try { + const entries: string[] = []; + const directory = await fs.opendir(handle ? descriptorPath(handle.fd) : path); + for await (const entry of directory) { + if (entries.length >= maxEntries) throw new Error('Dependency directory exceeds maxFiles'); + entries.push(entry.name); + } + return entries; + } finally { await handle?.close(); } +}; +export const readlink = async (path: string): Promise => + context.getStore()?.entry(path, 'readlink') as string ?? fs.readlink(path); +export const mkdir = async (path: string, mode = 0o700): Promise => { + const access = context.getStore(); + if (access) access.entry(path, 'mkdir', undefined, mode); + else await fs.mkdir(path, { mode }); +}; +export const symlink = async (target: string, path: string): Promise => { + const access = context.getStore(); + if (access) access.entry(path, 'symlink', target); + else await fs.symlink(target, path); +}; +export const rmdir = async (path: string): Promise => { + const access = context.getStore(); + if (access) access.unlink(path, true); + else await fs.rmdir(path); +}; diff --git a/packages/code/src/root-git-denies-live.test.ts b/packages/code/src/root-git-denies-live.test.ts new file mode 100644 index 00000000..12dc820c --- /dev/null +++ b/packages/code/src/root-git-denies-live.test.ts @@ -0,0 +1,140 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, realpath, rm, stat, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { NativeSrtWorkspaceCommandSandbox } from './native-sandbox.js'; +import { resolveNativeSrtCommandPolicy } from './native-policy.js'; + +const execFileAsync = promisify(execFile); +const IDENTITY = ['-c', 'user.name=t', '-c', 'user.email=t@example.com', '-c', 'commit.gpgsign=false']; + +async function git(cwd: string, ...args: string[]): Promise { + return (await execFileAsync('git', [...IDENTITY, ...args], { cwd })).stdout.trim(); +} + +async function snapshot(paths: string[]): Promise> { + const entries = await Promise.all( + paths.map(async (path) => [path, await readFile(path, 'utf8').catch(() => null)] as const), + ); + return Object.fromEntries(entries); +} + +async function exists(path: string): Promise { + return stat(path).then(() => true, () => false); +} + +/** + * SRT's Linux mandatory denies for `.git/hooks` and `.git/config` are computed + * from the worker process's own current directory, which is never the registered + * workspace root. This exercises the sandbox from a cwd outside the root and + * asserts the root's executable Git metadata is unwritable anyway — hooks, + * config, an existing per-worktree config, and a submodule's config cannot be + * tampered, so nothing an attacker writes runs the next time git is invoked on + * the host. + */ +test('real SRT denies a non-lane root its own Git hooks and config even when the worker cwd is elsewhere', { + skip: process.env.LIBRECHAT_CODE_LIVE_SRT_TESTS !== '1', + timeout: 90_000, +}, async (t) => { + const outside = await realpath(await mkdtemp(join(tmpdir(), 'git-denies-cwd-'))); + const base = await realpath(await mkdtemp(join(tmpdir(), 'git-denies-'))); + t.after(() => rm(base, { recursive: true, force: true })); + t.after(() => rm(outside, { recursive: true, force: true })); + + // A standalone repo used only as a local submodule source. + const upstream = join(base, 'sub-src'); + await mkdir(upstream); + await git(upstream, 'init', '-q', '-b', 'main'); + await writeFile(join(upstream, 's.txt'), 'sub\n'); + await git(upstream, 'add', 's.txt'); + await git(upstream, 'commit', '-qm', 'sub'); + + const root = join(base, 'repo'); + await mkdir(root); + await git(root, 'init', '-q', '-b', 'main'); + await writeFile(join(root, 'tracked.txt'), 'checkout\n'); + await git(root, 'add', 'tracked.txt'); + await git(root, 'commit', '-qm', 'init'); + await git(root, '-c', 'protocol.file.allow=always', 'submodule', 'add', '-q', upstream, 'sub'); + await git(root, 'commit', '-qm', 'add submodule'); + await mkdir(join(root, '.worktrees')); + await git(root, 'worktree', 'add', '-q', '.worktrees/wt', '-b', 'wt'); + // An existing per-worktree config for the linked worktree; --worktree needs + // the repository extension enabled first. + await git(root, 'config', 'extensions.worktreeConfig', 'true'); + await git(join(root, '.worktrees', 'wt'), 'config', '--worktree', 'core.testflag', 'true'); + + const gitDir = join(root, '.git'); + const submoduleGitDir = join(gitDir, 'modules', 'sub'); + const worktreeGitDir = join(gitDir, 'worktrees', 'wt'); + const guarded = [ + join(gitDir, 'hooks', 'post-checkout'), + join(gitDir, 'config'), + join(submoduleGitDir, 'config'), + join(submoduleGitDir, 'hooks', 'post-checkout'), + join(worktreeGitDir, 'config.worktree'), + join(worktreeGitDir, 'commondir'), + ]; + for (const path of guarded) { + assert.ok(await exists(path) || path.endsWith('post-checkout'), `precondition: ${path}`); + } + const before = await snapshot(guarded); + + const sandbox = new NativeSrtWorkspaceCommandSandbox({ + workspaceRoot: root, + commandPolicy: resolveNativeSrtCommandPolicy('trusted-vm'), + environment: { PATH: process.env.PATH, LANG: 'C.UTF-8' }, + }); + t.after(() => sandbox.close()); + const run = (command: string) => + sandbox.execute({ + protocolVersion: 1, + operation: 'execute_command', + workspaceId: 'root', + command, + timeoutMs: 30_000, + maxOutputBytes: 16_384, + }); + + // The worker process runs from outside the registered root, the way the + // systemd user unit does (its cwd is the worker home, not the workspace). + const previousCwd = process.cwd(); + process.chdir(outside); + t.after(() => process.chdir(previousCwd)); + + for (const path of guarded) { + const attempt = await run(`printf tampered > '${path}'`); + assert.notEqual(attempt.exitCode, 0, `expected a write to ${path} to be denied`); + } + // Planting a fresh hook file (not just overwriting one) must also be blocked. + const plantHook = await run( + `printf '#!/bin/sh\\ntouch ${join(base, 'PWNED')}\\n' > .git/hooks/pre-commit && chmod +x .git/hooks/pre-commit`, + ); + assert.notEqual(plantHook.exitCode, 0); + + // Metadata the host creates while the worker runs is covered from the next + // command, not only what existed when the sandbox initialized. + await git(root, 'worktree', 'add', '-q', '.worktrees/late', '-b', 'late'); + const lateCommondir = join(gitDir, 'worktrees', 'late', 'commondir'); + const lateBefore = await readFile(lateCommondir, 'utf8'); + const lateAttempt = await run(`printf tampered > '${lateCommondir}'`); + assert.notEqual(lateAttempt.exitCode, 0, 'expected the new worktree metadata to be denied'); + assert.equal(await readFile(lateCommondir, 'utf8'), lateBefore); + + // A benign commit in the workspace must still succeed: the guard makes Git + // metadata read-only, not the checkout or the writable object store. + const committed = await run( + `printf work > work.txt && git ${IDENTITY.join(' ')} add work.txt && git ${IDENTITY.join(' ')} commit -qm work`, + ); + assert.equal(committed.exitCode, 0, committed.stderr); + + await sandbox.close(); + + assert.equal(await exists(join(base, 'PWNED')), false); + assert.deepEqual(await snapshot(guarded), before); + assert.equal(await git(root, 'log', '-1', '--format=%s'), 'work'); +}); diff --git a/packages/code/src/snapshot-lifecycle.test.ts b/packages/code/src/snapshot-lifecycle.test.ts new file mode 100644 index 00000000..8074fd11 --- /dev/null +++ b/packages/code/src/snapshot-lifecycle.test.ts @@ -0,0 +1,274 @@ +import assert from 'node:assert/strict'; +import test from 'node:test'; +import { + mkdtemp, + mkdir, + realpath, + writeFile, + readFile, + rm, + lstat, + utimes, + symlink, +} from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { + loadDependencySnapshot, + parseDependencySnapshot, +} from './dependency-snapshots.js'; +import { + assertPreparationSpace, + parseEnvironmentStorage, + parseSnapshotLifecycle, + pruneDependencySnapshots, + SNAPSHOT_MANIFEST, + STAGING_MANIFEST, +} from './snapshot-lifecycle.js'; +import { withProcessLock } from './process-lock.js'; +import { prepareCodeEnvironment } from './environment-preparation.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; + +async function fixture(t: test.TestContext) { + const root = await realpath( + await mkdtemp(join(tmpdir(), 'snapshot-lifecycle-')), + ); + t.after(() => rm(root, { recursive: true, force: true })); + const path = join(root, 'store'); + await mkdir(path, { mode: 0o700 }); + const store = await loadDependencySnapshot( + parseDependencySnapshot({ + store: path, + paths: ['node_modules'], + lifecycle: { maxStoreBytes: 10, maxEntries: 2, retentionMs: 1000 }, + }), + ); + const add = async (key: string, bytes = 4, age = 0) => { + const directory = join(store.store, key); + await mkdir(directory, { mode: 0o700 }); + await writeFile( + join(directory, SNAPSHOT_MANIFEST), + JSON.stringify({ + version: 1, + key, + bytes, + files: 1, + createdAt: Date.now(), + }), + { mode: 0o600 }, + ); + const lastUsed = new Date(Date.now() - age); + await utimes(join(directory, SNAPSHOT_MANIFEST), lastUsed, lastUsed); + return directory; + }; + return { + root, + store, + add, + a: 'a'.repeat(64), + b: 'b'.repeat(64), + c: 'c'.repeat(64), + }; +} + +test('storage policy is opt-in, bounded and fails before a setup command', async t => { + assert.equal(parseEnvironmentStorage({}).minFreeBytes, 5 * 1024 ** 3); + assert.throws(() => parseEnvironmentStorage({ minFreeBytes: -1 })); + assert.throws(() => parseSnapshotLifecycle({ retentionMs: Infinity })); + await assertPreparationSpace( + '/', + { minFreeBytes: 5, setupReserveBytes: 2 }, + async () => 7n, + ); + await assert.rejects( + assertPreparationSpace( + '/', + { minFreeBytes: 5, setupReserveBytes: 2 }, + async () => 6n, + ), + /no setup command was started/, + ); + const { root } = await fixture(t); + const checkout = join(root, 'checkout'); + await mkdir(checkout); + const identity = await captureWorkspaceRootIdentity(checkout); + let executions = 0; + await assert.rejects( + prepareCodeEnvironment({ + root: checkout, + identity, + setup: { command: 'install', timeoutMs: 1000 }, + receiptPath: join(root, 'receipt'), + context: '', + storage: { + minFreeBytes: Number.MAX_SAFE_INTEGER, + setupReserveBytes: 0, + }, + execute: async () => { + executions++; + return { exitCode: 0, timedOut: false }; + }, + }), + /deferred/, + ); + assert.equal(executions, 0); +}); + +test('low space reclaims once before deferring setup', async t => { + const policy = { minFreeBytes: 5, setupReserveBytes: 2 }; + let free = 6n; + let reclaims = 0; + await assertPreparationSpace('/', policy, async () => free, async () => { + reclaims++; + free = 7n; + }); + assert.equal(reclaims, 1); + await assertPreparationSpace('/', policy, async () => free, async () => { + reclaims++; + }); + assert.equal(reclaims, 1, 'enough space never triggers reclamation'); + await assert.rejects( + assertPreparationSpace('/', policy, async () => 6n, async () => { + reclaims++; + }), + /no setup command was started/, + ); + assert.equal(reclaims, 2); + + const { root } = await fixture(t); + const checkout = join(root, 'checkout'); + await mkdir(checkout); + let executions = 0; + let reclaimed = 0; + await assert.rejects( + prepareCodeEnvironment({ + root: checkout, + identity: await captureWorkspaceRootIdentity(checkout), + setup: { command: 'install', timeoutMs: 1000 }, + receiptPath: join(root, 'receipt'), + context: '', + storage: { minFreeBytes: Number.MAX_SAFE_INTEGER, setupReserveBytes: 0 }, + reclaimSpace: async () => { + reclaimed++; + }, + execute: async () => { + executions++; + return { exitCode: 0, timedOut: false }; + }, + }), + /deferred/, + ); + assert.equal(reclaimed, 1); + assert.equal(executions, 0); +}); + +test('dry-run is non-destructive; cleanup removes only expired owned entries and retains unknown or linked data', async t => { + const { store, add, a, b, c, root } = await fixture(t); + const old = await add(a, 4, 10_000); + await add(b); + await mkdir(join(store.store, 'unrecognized')); + await writeFile(join(store.store, 'unrecognized', 'source'), 'keep'); + await symlink(root, join(store.store, c)); + const preview = await pruneDependencySnapshots(store, { dryRun: true }); + assert.deepEqual(preview.removed, [a]); + assert.equal((await lstat(old)).isDirectory(), true); + const actual = await pruneDependencySnapshots(store); + assert.deepEqual(actual.removed, [a]); + await assert.rejects(lstat(old), { code: 'ENOENT' }); + assert.equal( + await readFile(join(store.store, 'unrecognized', 'source'), 'utf8'), + 'keep', + ); + assert.equal((await lstat(join(store.store, c))).isSymbolicLink(), true); + assert.equal(actual.retainedBytes, 4); +}); + +test('cleanup preserves active keys and current preparation even when over budget', async t => { + const { store, add, a, b } = await fixture(t); + await add(a, 4, 10_000); + await add(b, 4, 10_000); + await withProcessLock(join(store.store, `${a}.lock`), async () => { + const result = await pruneDependencySnapshots(store, { currentKey: b }); + assert.deepEqual(result.removed, []); + assert.deepEqual(result.skippedActive, [a]); + await assert.rejects( + pruneDependencySnapshots(store, { + currentKey: b, + incomingBytes: 10, + }), + /budget is full/, + ); + }); + assert.equal((await lstat(join(store.store, a))).isDirectory(), true); +}); + +test('publication budgets evict LRU entries and serialize commits without deleting source', async t => { + const { store, add, a, b, c, root } = await fixture(t); + await add(a, 4, 500); + await add(b, 4, 100); + const source = join(root, 'dirty-worktree'); + await mkdir(source); + await writeFile(join(source, 'uncommitted'), 'keep'); + let published = false; + const result = await pruneDependencySnapshots(store, { + currentKey: c, + incomingBytes: 4, + incomingEntries: 1, + publish: async () => { + published = true; + }, + }); + assert.deepEqual(result.removed, [a]); + assert.equal(published, true); + assert.equal(await readFile(join(source, 'uncommitted'), 'utf8'), 'keep'); +}); + +test('operator CLI previews by default and applies only after explicit flag', async t => { + const { store, add, a, root } = await fixture(t); + await add(a, 4, 10_000); + const checkout = join(root, 'checkout'); + await mkdir(checkout); + const config = join(root, 'environment.yaml'); + await writeFile( + config, + `name: app\nroot: ${checkout}\nsetup:\n command: npm ci\n reuse:\n inputs: [package-lock.json]\n checkCommand: test -d node_modules\n snapshot:\n store: ${store.store}\n paths: [node_modules]\n lifecycle:\n retentionMs: 1000\n`, + { mode: 0o600 }, + ); + const run = promisify(execFile); + const cli = new URL('./cli.js', import.meta.url).pathname; + const args = [cli, 'prune-environment-storage', '--environment', config]; + const preview = await run(process.execPath, args); + assert.deepEqual(JSON.parse(preview.stdout).removed, [a]); + assert.equal((await lstat(join(store.store, a))).isDirectory(), true); + const applied = await run(process.execPath, [...args, '--apply']); + assert.deepEqual(JSON.parse(applied.stdout).removed, [a]); + await assert.rejects(lstat(join(store.store, a)), { code: 'ENOENT' }); +}); + +test('abandoned staging is reclaimed only with an ownership manifest and a free key lock', async t => { + const { store, a } = await fixture(t); + const name = `.staging-${a}-abcdef`; + const directory = join(store.store, name); + await mkdir(directory, { mode: 0o700 }); + await writeFile( + join(directory, STAGING_MANIFEST), + JSON.stringify({ + version: 1, + key: a, + bytes: 0, + files: 1, + createdAt: Date.now(), + }), + { mode: 0o600 }, + ); + await withProcessLock(join(store.store, `${a}.lock`), async () => { + const active = await pruneDependencySnapshots(store); + assert.deepEqual(active.skippedActive, [name]); + assert.deepEqual(active.removed, []); + }); + const result = await pruneDependencySnapshots(store); + assert.deepEqual(result.removed, [name]); + await assert.rejects(lstat(directory), { code: 'ENOENT' }); +}); diff --git a/packages/code/src/snapshot-lifecycle.ts b/packages/code/src/snapshot-lifecycle.ts new file mode 100644 index 00000000..66ed39dd --- /dev/null +++ b/packages/code/src/snapshot-lifecycle.ts @@ -0,0 +1,324 @@ +import { constants } from 'node:fs'; +import { lstat, open, rm, statfs } from 'node:fs/promises'; +import { join } from 'node:path'; +import { withProcessLock, tryWithProcessLock } from './process-lock.js'; +import { matchesWorkspaceRoot } from './root-identity.js'; +import { readdir, withWorkspaceRoot } from './root-access.js'; +import type { DependencySnapshotStore } from './dependency-snapshots.js'; + +export interface SnapshotLifecyclePolicy { + maxStoreBytes: number; + maxEntries: number; + retentionMs: number; + scanLimit: number; +} +export interface EnvironmentStoragePolicy { + minFreeBytes: number; + setupReserveBytes: number; +} +function positive(value: unknown, zero = false): value is number { + return ( + typeof value === 'number' && + Number.isSafeInteger(value) && + value >= (zero ? 0 : 1) + ); +} +export function parseSnapshotLifecycle( + value: unknown, +): SnapshotLifecyclePolicy { + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Invalid snapshot lifecycle'); + const v = value as Record; + const policy = { + maxStoreBytes: v.maxStoreBytes ?? 20 * 1024 ** 3, + maxEntries: v.maxEntries ?? 8, + retentionMs: v.retentionMs ?? 5 * 24 * 60 * 60 * 1000, + scanLimit: v.scanLimit ?? 4096, + }; + if ( + Object.keys(v).some(key => !Object.hasOwn(policy, key)) || + !Object.values(policy).every(value => positive(value)) + ) + throw new Error('Invalid snapshot lifecycle'); + return policy as SnapshotLifecyclePolicy; +} +export function parseEnvironmentStorage( + value: unknown, +): EnvironmentStoragePolicy { + if (!value || typeof value !== 'object' || Array.isArray(value)) + throw new Error('Invalid environment storage policy'); + const v = value as Record; + const policy = { + minFreeBytes: v.minFreeBytes ?? 5 * 1024 ** 3, + setupReserveBytes: v.setupReserveBytes ?? 2 * 1024 ** 3, + }; + if ( + Object.keys(v).some(key => !Object.hasOwn(policy, key)) || + !Object.values(policy).every(value => positive(value, true)) || + !Number.isSafeInteger( + (policy.minFreeBytes as number) + + (policy.setupReserveBytes as number), + ) + ) + throw new Error('Invalid environment storage policy'); + return policy as EnvironmentStoragePolicy; +} +/** + * Soft admission for managed preparation, not a reservation or arbitrary-write + * quota. Below the floor, `reclaim` (when supplied) may free reproducible + * storage once before the space is measured again. + */ +export async function assertPreparationSpace( + root: string, + policy: EnvironmentStoragePolicy, + available: (root: string) => Promise = async path => { + const status = await statfs(path, { bigint: true }); + return status.bavail * status.bsize; + }, + reclaim?: () => Promise, +): Promise { + const floor = + BigInt(policy.minFreeBytes) + BigInt(policy.setupReserveBytes); + if ((await available(root)) >= floor) return; + if (reclaim) { + await reclaim(); + if ((await available(root)) >= floor) return; + } + throw new Error( + 'Managed environment preparation deferred: free disk is below the configured floor plus setup reserve. Clean reproducible artifacts or expand storage; no setup command was started.', + ); +} + +export const SNAPSHOT_MANIFEST = '.snapshot.json'; +export const STAGING_MANIFEST = '.staging.json'; +export class SnapshotBudgetFullError extends Error {} +export interface SnapshotManifest { + version: 1; + key: string; + bytes: number; + files: number; + createdAt: number; +} +async function manifest( + store: DependencySnapshotStore, + key: string, + name = key, + file = SNAPSHOT_MANIFEST, +): Promise { + const directory = join(store.store, name); + const status = await lstat(directory); + if (!status.isDirectory() || status.isSymbolicLink() || status.mode & 0o077) + return; + const handle = await open( + join(directory, file), + constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK, + ).catch((e: NodeJS.ErrnoException) => { + if (e.code === 'ENOENT' || e.code === 'ELOOP') return undefined; + throw e; + }); + if (!handle) return; + try { + const metadata = await handle.stat(); + if ( + !metadata.isFile() || + metadata.size > 4096 || + metadata.mode & 0o077 || + (process.getuid && metadata.uid !== process.getuid()) + ) + return; + const buffer = Buffer.alloc(4097); + const { bytesRead } = await handle.read(buffer, 0, buffer.length, 0); + if (bytesRead > 4096) return; + let value: unknown; + try { + value = JSON.parse(buffer.subarray(0, bytesRead).toString()); + } catch { + return; + } + if (!value || typeof value !== 'object' || Array.isArray(value)) return; + const entry = value as Record; + if ( + entry.version !== 1 || + entry.key !== key || + !positive(entry.bytes, true) || + !positive(entry.files) || + !positive(entry.createdAt) + ) + return; + return value as SnapshotManifest; + } finally { + await handle.close(); + } +} +export interface SnapshotPruneResult { + dryRun: boolean; + removed: string[]; + skippedActive: string[]; + unknown: string[]; + retainedBytes: number; + retainedEntries: number; +} + +/** Only worker-published manifests authorize deletion. Source worktrees and unknown entries never qualify. */ +export async function pruneDependencySnapshots( + store: DependencySnapshotStore, + options: { + dryRun?: boolean; + currentKey?: string; + incomingBytes?: number; + incomingEntries?: number; + now?: number; + signal?: AbortSignal; + publish?: () => Promise; + } = {}, +): Promise { + const policy = store.lifecycle; + if (!policy) throw new Error('Snapshot lifecycle is not configured'); + return withProcessLock( + join(store.store, '.maintenance.lock'), + async () => { + if (!(await matchesWorkspaceRoot(store.store, store.identity))) + throw new Error( + 'Dependency snapshot store changed before cleanup', + ); + const result: SnapshotPruneResult = { + dryRun: options.dryRun === true, + removed: [], + skippedActive: [], + unknown: [], + retainedBytes: 0, + retainedEntries: 0, + }; + const candidates: { + key: string; + data: SnapshotManifest; + lastUsed: number; + }[] = []; + const names = await withWorkspaceRoot( + store.store, + store.identity, + () => readdir(store.store, policy.scanLimit), + ); + for (const key of names) { + options.signal?.throwIfAborted(); + const staging = /^\.staging-([a-f0-9]{64})-[A-Za-z0-9]+$/.exec( + key, + ); + if (staging) { + const owner = staging[1]; + const data = await manifest( + store, + owner, + key, + STAGING_MANIFEST, + ); + if (!data) { + result.unknown.push(key); + continue; + } + if (owner === options.currentKey) continue; + const lock = await tryWithProcessLock( + join(store.store, `${owner}.lock`), + async () => { + if ( + !(await manifest( + store, + owner, + key, + STAGING_MANIFEST, + )) + ) + return; + if (!options.dryRun) + await rm(join(store.store, key), { + recursive: true, + force: false, + }); + result.removed.push(key); + }, + options.signal, + ); + if (!lock.acquired) result.skippedActive.push(key); + continue; + } + if (!/^[a-f0-9]{64}$/.test(key)) { + if ( + !/^[a-f0-9]{64}\.lock$/.test(key) && + key !== '.maintenance.lock' + ) + result.unknown.push(key); + continue; + } + const data = await manifest(store, key); + if (!data) { + result.unknown.push(key); + continue; + } + candidates.push({ + key, + data, + lastUsed: ( + await lstat(join(store.store, key, SNAPSHOT_MANIFEST)) + ).mtimeMs, + }); + result.retainedBytes += data.bytes; + result.retainedEntries++; + } + const now = options.now ?? Date.now(); + candidates.sort( + (a, b) => a.lastUsed - b.lastUsed || a.key.localeCompare(b.key), + ); + for (const candidate of candidates) { + options.signal?.throwIfAborted(); + if (candidate.key === options.currentKey) continue; + if ( + now - candidate.lastUsed < policy.retentionMs && + result.retainedBytes + (options.incomingBytes ?? 0) <= + policy.maxStoreBytes && + result.retainedEntries + (options.incomingEntries ?? 0) <= + policy.maxEntries + ) + continue; + const admission = await tryWithProcessLock( + join(store.store, `${candidate.key}.lock`), + async () => { + const fresh = await manifest(store, candidate.key); + const updated = await lstat( + join(store.store, candidate.key, SNAPSHOT_MANIFEST), + ); + if ( + !fresh || + fresh.createdAt !== candidate.data.createdAt || + updated.mtimeMs !== candidate.lastUsed + ) + return; + if (!options.dryRun) + await rm(join(store.store, candidate.key), { + recursive: true, + force: false, + }); + result.removed.push(candidate.key); + result.retainedBytes -= candidate.data.bytes; + result.retainedEntries--; + }, + options.signal, + ); + if (!admission.acquired) + result.skippedActive.push(candidate.key); + } + if ( + !options.dryRun && + (result.retainedBytes + (options.incomingBytes ?? 0) > + policy.maxStoreBytes || + result.retainedEntries + (options.incomingEntries ?? 0) > + policy.maxEntries) + ) + throw new SnapshotBudgetFullError( + 'Dependency snapshot budget is full; active or unknown data was not deleted', + ); + if (!options.dryRun) await options.publish?.(); + return result; + }, + options.signal, + ); +} diff --git a/packages/code/src/storage.ts b/packages/code/src/storage.ts index 1cbb7093..904a8562 100644 --- a/packages/code/src/storage.ts +++ b/packages/code/src/storage.ts @@ -1,4 +1,5 @@ import { createHash, randomBytes } from 'node:crypto'; +import { constants } from 'node:fs'; import { chmod, lstat, @@ -201,11 +202,15 @@ async function assertOwnedByWorker(path: string): Promise { async function readGuardedFile( path: string, exposed: (mode: string) => string, + maxBytes = Infinity, ): Promise { await assertReadPathPrivate(path); const handle = await open(path, 'r'); try { const stats = await handle.stat(); + if (!stats.isFile() || stats.size > maxBytes) { + throw new BridgeProtocolError('Invalid private storage file size or type'); + } const self = process.getuid?.(); if (self !== undefined && !isTrustedOwner(stats.uid, self)) { throw new BridgeProtocolError( @@ -423,6 +428,44 @@ export async function saveBridgeIdentity( path: string, identity: PairedBridgeWorkerIdentity, ): Promise { + return savePrivateJson(path, identity); +} + +/** Preparation receipts are worker state, never a workspace-authored success marker. */ +export async function prepareEnvironmentPreparationDirectory(path: string): Promise { + await assertPrivateStorageAncestors(path, true); + await ensureDurableDirectory(path); + const handle = await open(path, constants.O_RDONLY | constants.O_NOFOLLOW); + try { + const metadata = await handle.stat(); + if (!metadata.isDirectory() || (metadata.mode & 0o077) !== 0) + throw new BridgeProtocolError('Environment preparation directory must be owner-only'); + await assertPrivateStorageAcl(handle, path, true); + } finally { await handle.close(); } + return assertPrivateStorageAncestors(path); +} + +export async function loadEnvironmentPreparationKey(path: string): Promise { + let content: string; + try { + content = await readGuardedFile(path, () => 'Environment preparation receipt must be owner-only', 256); + } catch (error) { + if (isMissingPathError(error)) return undefined; + throw error; + } + try { + const value: unknown = JSON.parse(content); + if (isRecord(value) && value.version === 1 && typeof value.key === 'string' && /^[a-f0-9]{64}$/.test(value.key)) return value.key; + } catch { /* Invalid receipts are cache misses, not readiness evidence. */ } + return undefined; +} + +export async function saveEnvironmentPreparationKey(path: string, key: string): Promise { + if (!/^[a-f0-9]{64}$/.test(key)) throw new Error('Invalid environment preparation key'); + await savePrivateJson(path, { version: 1, key }); +} + +async function savePrivateJson(path: string, value: unknown): Promise { assertPrivateStorageSupported(); await assertWriteContainerPrivate(path); await mkdir(dirname(path), { recursive: true, mode: 0o700 }); @@ -435,13 +478,14 @@ export async function saveBridgeIdentity( await removePrivateStorageAcl(file, path); await file.chmod(0o600); await assertOwnerOnlyFile(file, path); - await file.writeFile(`${JSON.stringify(identity, null, 2)}\n`, 'utf8'); + await file.writeFile(`${JSON.stringify(value, null, 2)}\n`, 'utf8'); await file.sync(); } finally { await file.close(); } await rename(temporaryPath, path); await chmod(path, 0o600); + await syncParentDirectory(path); } catch (error) { await rm(temporaryPath, { force: true }); throw error; diff --git a/packages/code/src/worker-slots.test.ts b/packages/code/src/worker-slots.test.ts index fda6c659..72052974 100644 --- a/packages/code/src/worker-slots.test.ts +++ b/packages/code/src/worker-slots.test.ts @@ -377,3 +377,204 @@ test('programmatic work on an independent workspace bypasses another root cleanu 'previous', ); }); + +test('sibling linked-worktree lanes run past each other, but a lane and its checkout wait for one another', async () => { + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'fixture', + runtimes: [], + }, + }); + const internals = worker as unknown as { + activeWorkspaceAssignments: Map }>; + executeOwned: (assignment: BridgeAssignment) => Promise; + }; + const executed: string[] = []; + internals.executeOwned = async (assignment) => { + executed.push(assignment.assignmentId); + }; + const assignment = (assignmentId: string, worktree?: string) => + ({ + assignmentId, + executionKind: 'workspace_tool', + remainingMs: 1_000, + request: { + protocolVersion: 1, + workspaceId: 'repo', + operation: 'read_file', + path: 'README.md', + ...(worktree ? { worktree } : {}), + }, + }) as BridgeAssignment; + let releaseLane!: () => void; + internals.activeWorkspaceAssignments.set(workspaceIsolationKey('repo', undefined, 'task-a'), { + id: 'lane-a', + done: new Promise((resolve) => { + releaseLane = resolve; + }), + }); + + await worker.executeAndSettle(assignment('lane-b', 'task-b')); + assert.deepEqual(executed, ['lane-b']); + + const checkout = worker.executeAndSettle(assignment('checkout')); + await new Promise((resolve) => setTimeout(resolve, 5)); + assert.deepEqual(executed, ['lane-b']); + internals.activeWorkspaceAssignments.delete(workspaceIsolationKey('repo', undefined, 'task-a')); + releaseLane(); + await checkout; + assert.deepEqual(executed, ['lane-b', 'checkout']); + + let releaseCheckout!: () => void; + internals.activeWorkspaceAssignments.set(workspaceIsolationKey('repo'), { + id: 'root', + done: new Promise((resolve) => { + releaseCheckout = resolve; + }), + }); + const lane = worker.executeAndSettle(assignment('lane-c', 'task-c')); + await new Promise((resolve) => setTimeout(resolve, 5)); + assert.deepEqual(executed, ['lane-b', 'checkout']); + internals.activeWorkspaceAssignments.delete(workspaceIsolationKey('repo')); + releaseCheckout(); + await lane; + assert.deepEqual(executed, ['lane-b', 'checkout', 'lane-c']); +}); + +test('a linked worktree lane fence is reset through its own guard and isolation key', async () => { + const bodies: Record[] = []; + const guarded: string[] = []; + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + incarnationId: 'incarnation-reset', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceLeaseSlots: 2, + requiresReadyConfirmation: true, + workspaceTools: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'repo', workspaceScopes: ['git_linked_worktree'] }, { id: 'plain' }], + }, + }, + workspaceQuarantines: new Map( + ['repo', 'plain'].map((id) => [ + id, + { + assertAvailable: async () => undefined, + arm: async () => undefined, + clear: async () => undefined, + quarantine: async () => undefined, + }, + ]), + ), + workspaceTools: { + capabilities: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [{ id: 'repo', workspaceScopes: ['git_linked_worktree'] }, { id: 'plain' }], + }, + async execute() { + throw new Error('must not execute'); + }, + }, + linkedWorktreeQuarantineResolver: (workspaceId, worktree) => { + guarded.push(`${workspaceId}/${worktree}`); + return { + assertAvailable: async () => undefined, + arm: async () => undefined, + clear: async () => undefined, + quarantine: async () => undefined, + }; + }, + fetchImpl: async (url, init) => { + assert.ok(new URL(String(url)).pathname.endsWith('/workspaces/reset')); + bodies.push(JSON.parse(String(init?.body))); + return Response.json({ protocolVersion: 1, reset: true }); + }, + }); + + await worker.resetNativeWorkspace('repo', undefined, undefined, 'task-a'); + + assert.deepEqual(guarded, ['repo/task-a']); + assert.equal( + bodies[0]?.runtimeSessionId, + `native-workspace:${workspaceIsolationKey('repo', undefined, 'task-a')}`, + ); + await assert.rejects( + worker.resetNativeWorkspace('plain', undefined, undefined, 'task-a'), + /worktree lanes/, + ); + await assert.rejects( + worker.resetNativeWorkspace('repo', undefined, 'a'.repeat(64), 'task-a'), + /worktree lanes/, + ); + await assert.rejects( + worker.resetNativeWorkspace('repo', undefined, undefined, '../task-a'), + /worktree lanes/, + ); + assert.equal(bodies.length, 1); +}); + +test('a checkout waits on quarantined linked worktrees beneath it', async () => { + const quarantinedLanes = new Set(['task-b']); + const worker = new BridgeWorker({ + codeApiUrl: 'http://localhost:1', + token: 'fixture', + workerId: 'worker', + sandboxEndpoint: 'http://localhost:2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'fixture', + runtimes: [], + }, + linkedWorktreeQuarantineResolver: (_workspaceId, worktree) => ({ + assertAvailable: async () => { + if (quarantinedLanes.has(worktree)) throw new Error(`lane ${worktree} is quarantined`); + }, + arm: async () => undefined, + clear: async () => undefined, + quarantine: async () => undefined, + }), + linkedWorktreeNames: async () => ['task-a', 'task-b'], + }); + const internals = worker as unknown as { + quarantinedWorkspaces: Set; + assertLaneFamilyAvailable: (key: string, assignment: BridgeAssignment) => Promise; + }; + const assignment = (worktree?: string) => + ({ + assignmentId: 'probe', + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + workspaceId: 'repo', + operation: 'read_file', + path: 'README.md', + ...(worktree ? { worktree } : {}), + }, + }) as BridgeAssignment; + const check = (worktree?: string) => + internals.assertLaneFamilyAvailable( + workspaceIsolationKey('repo', undefined, worktree), + assignment(worktree), + ); + + await assert.rejects(check(), /task-b is quarantined/); + await check('task-a'); + + quarantinedLanes.clear(); + await check(); + internals.quarantinedWorkspaces.add(workspaceIsolationKey('repo', undefined, 'task-c')); + await assert.rejects(check(), /linked worktree/); +}); diff --git a/packages/code/src/worker.ts b/packages/code/src/worker.ts index 2a5896fd..49b96eb1 100644 --- a/packages/code/src/worker.ts +++ b/packages/code/src/worker.ts @@ -6,14 +6,19 @@ import { BridgeProtocolError, bridgeWorkerPath, isBridgeWorkspaceProgrammaticRequest, + isValidLinkedWorktreeName, isWorkspaceToolResult, workspaceIsolationKey, + workspaceIsolationKeysConflict, + workspaceIsolationParent, + WORKSPACE_TOOL_ERROR_CODE_FALLBACKS, } from './protocol.js'; import { EndpointRuntimeSupervisor } from './runtime.js'; import { signBridgeRequest } from './identity.js'; import { isWorkspaceToolRequest, WorkspaceToolError } from './workspace.js'; import type { + WorkspaceCommandResultFeature, BridgeAssignment, BridgeLeaseResponse, BridgeSandboxRequest, @@ -25,6 +30,7 @@ import type { BridgeWorkspaceToolOperation, BridgeWorkspaceProgrammaticRequest, RepositoryInstructionDescriptor, + WorkspaceToolErrorCode, } from './protocol.js'; import type { RuntimeLease, RuntimeSupervisor } from './runtime.js'; import type { WorkspaceToolExecutor } from './workspace.js'; @@ -62,6 +68,15 @@ export interface BridgeWorkerOptions { ) => | WorkspaceMutationQuarantine | Promise; + /** Resolve a durable guard for a linked-worktree lane beneath a registered root. */ + linkedWorktreeQuarantineResolver?: ( + workspaceId: string, + worktree: string, + ) => + | WorkspaceMutationQuarantine + | Promise; + /** Names of the linked worktrees currently beneath a registered root. */ + linkedWorktreeNames?: (workspaceId: string) => Promise; leaseWaitMs?: number; leaseTransportGraceMs?: number; registrationTransportTimeoutMs?: number; @@ -161,6 +176,19 @@ function errorCode(value: object): string | undefined { return undefined; } +/** Edit conflicts can include source excerpts, so disclosure needs read or full-preview access. */ +function workspaceCanReadSource( + capabilities: BridgeWorkerCapabilities['workspaceTools'], + workspaceId: string, +): boolean { + const workspace = capabilities?.workspaces.find((entry) => entry.id === workspaceId); + if (!workspace || !capabilities) return false; + return (['read_file', 'preview_edit'] as const).some( + (operation) => capabilities.operations.includes(operation) && + (workspace.operations == null || workspace.operations.includes(operation)), + ); +} + function workspaceCapabilitiesMatch( advertised: NonNullable, executor: NonNullable, @@ -191,6 +219,11 @@ function workspaceCapabilitiesMatch( (feature, index) => feature === executor.listFileFeatures?.[index], ) ?? executor.listFileFeatures == null) && + advertised.commandResultFeatures?.length === executor.commandResultFeatures?.length && + (advertised.commandResultFeatures?.every( + (feature, index) => feature === executor.commandResultFeatures?.[index], + ) ?? + executor.commandResultFeatures == null) && advertised.programmaticLanguages?.length === executor.programmaticLanguages?.length && (advertised.programmaticLanguages?.every( @@ -223,7 +256,13 @@ function workspaceCapabilitiesMatch( (instanceType, instanceIndex) => instanceType === executor.workspaces[index]?.workspaceInstances?.[instanceIndex], - ) ?? executor.workspaces[index]?.workspaceInstances == null), + ) ?? executor.workspaces[index]?.workspaceInstances == null) && + workspace.workspaceScopes?.length === + executor.workspaces[index]?.workspaceScopes?.length && + (workspace.workspaceScopes?.every( + (scope, scopeIndex) => + scope === executor.workspaces[index]?.workspaceScopes?.[scopeIndex], + ) ?? executor.workspaces[index]?.workspaceScopes == null), ) ); } @@ -239,7 +278,9 @@ function registrationCompatibleCapabilities( ) && workspaceTools.workspaces.every( (workspace) => - workspace.operations == null && workspace.workspaceInstances == null, + workspace.operations == null && + workspace.workspaceInstances == null && + workspace.workspaceScopes == null, )) ) { return capabilities; @@ -263,6 +304,7 @@ function registrationCompatibleCapabilities( const { operations: _operations, workspaceInstances: _workspaceInstances, + workspaceScopes: _workspaceScopes, ...compatibleWorkspace } = workspace; return [{ ...compatibleWorkspace, ...(workspace.environment ? { @@ -278,6 +320,7 @@ function registrationCompatibleCapabilities( editFileModes: _editFileModes, editFileFeatures: _editFileFeatures, listFileFeatures: _listFileFeatures, + commandResultFeatures: _commandResultFeatures, programmaticLanguages: _programmaticLanguages, ...compatibleWorkspaceTools } = workspaceTools; @@ -355,6 +398,10 @@ function supportedWorkspaceCapabilities( ) ? { workspaceInstances: workspace.workspaceInstances } : { workspaceInstances: undefined }), + ...(workspace.workspaceScopes != null && + registration.supportedWorkspaceScopes?.includes('git_linked_worktree') + ? { workspaceScopes: workspace.workspaceScopes } + : { workspaceScopes: undefined }), ...(workspace.operations ? { operations: workspaceOperations } : {}), ...(workspace.environment && !workspaceOperations.includes('execute_command') ? { environment: { ...workspace.environment, actions: [] }, @@ -363,11 +410,15 @@ function supportedWorkspaceCapabilities( }); if (workspaces.length === 0) return undefined; const editFileFeatures = desired.editFileFeatures?.filter((feature) => - registration.supportedWorkspaceEditFileFeatures?.includes(feature), + registration.supportedWorkspaceEditFileFeatures?.includes(feature) && + (feature !== 'expected_base_sha256' || operations.includes('edit_file')), ); const listFileFeatures = desired.listFileFeatures?.filter((feature) => registration.supportedWorkspaceListFileFeatures?.includes(feature), ); + const commandResultFeatures = desired.commandResultFeatures?.filter((feature) => + registration.supportedWorkspaceCommandResultFeatures?.includes(feature), + ); const programmaticLanguages = desired.programmaticLanguages?.filter( (language) => registration.supportedWorkspaceProgrammaticLanguages?.includes(language), @@ -377,6 +428,7 @@ function supportedWorkspaceCapabilities( editFileModes: _editFileModes, editFileFeatures: _editFileFeatures, listFileFeatures: _listFileFeatures, + commandResultFeatures: _commandResultFeatures, programmaticLanguages: _programmaticLanguages, ...compatibleDesired } = desired; @@ -392,12 +444,15 @@ function supportedWorkspaceCapabilities( ...(supportsEditRequests && editFileModes?.length ? { editFileModes } : {}), - ...(operations.includes('edit_file') && editFileFeatures?.length + ...(supportsEditRequests && editFileFeatures?.length ? { editFileFeatures } : {}), ...(operations.includes('list_files') && listFileFeatures?.length ? { listFileFeatures } : {}), + ...(operations.includes('execute_command') && commandResultFeatures?.length + ? { commandResultFeatures } + : {}), ...(operations.includes('execute_command') && programmaticLanguages?.length ? { programmaticLanguages } @@ -425,6 +480,13 @@ export class BridgeWorker { private registrationCapabilities: BridgeWorkerCapabilities; private activeCapabilities: BridgeWorkerCapabilities; private instructionMetadataSupported = true; + /** Added error codes the current Code API registration accepts in settlements. */ + private settlementErrorCodes: ReadonlySet = new Set(); + /** Whether the registration that is actually active advertised this command result feature. */ + commandResultFeatureActive(feature: WorkspaceCommandResultFeature): boolean { + return this.activeCapabilities.workspaceTools?.commandResultFeatures?.includes(feature) === true; + } + private registrationTtlMs = DEFAULT_REGISTRATION_TTL_MS; private lastRegisteredAtMs = 0; private maintenanceOnly = false; @@ -506,7 +568,8 @@ export class BridgeWorker { ) === true && options.workspaceMutationQuarantine == null && options.workspaceQuarantines == null && - options.workspaceQuarantineResolver == null + options.workspaceQuarantineResolver == null && + options.linkedWorktreeQuarantineResolver == null ) { throw new BridgeProtocolError( 'Workspace mutation capabilities require durable quarantine storage', @@ -522,6 +585,17 @@ export class BridgeWorker { 'Workspace instance capabilities require a durable quarantine resolver', ); } + if ( + options.capabilities.workspaceTools?.workspaces.some( + (root) => (root.workspaceScopes?.length ?? 0) > 0, + ) && + (options.linkedWorktreeQuarantineResolver == null || + (options.capabilities.workspaceLeaseSlots ?? 1) < 2) + ) { + throw new BridgeProtocolError( + 'Linked worktree lanes require a durable quarantine resolver and at least two lease slots', + ); + } if ((options.capabilities.workspaceLeaseSlots ?? 1) > 1) { if ( options.capabilities.requiresReadyConfirmation !== true || @@ -752,6 +826,11 @@ export class BridgeWorker { } this.registrationTtlMs = registration.leaseTtlMs; this.activeCapabilities = this.registrationCapabilities; + this.settlementErrorCodes = new Set( + Array.isArray(registration.supportedWorkspaceToolErrorCodes) + ? registration.supportedWorkspaceToolErrorCodes + : [], + ); await this.options.onRegistered?.(registration); if ( !this.maintenanceOnly && @@ -823,18 +902,33 @@ export class BridgeWorker { workspaceId: string, signal?: AbortSignal, workspaceInstanceId?: string, + worktree?: string, ): Promise { const workspace = this.options.capabilities.workspaceTools?.workspaces.find( (root) => root.id === workspaceId, ); - const key = workspaceIsolationKey(workspaceId, workspaceInstanceId); - const guard = - workspaceInstanceId == null - ? this.options.workspaceQuarantines?.get(workspaceId) - : await this.options.workspaceQuarantineResolver?.( - workspaceId, - workspaceInstanceId, - ); + if ( + worktree != null && + (workspaceInstanceId != null || + !isValidLinkedWorktreeName(worktree) || + workspace?.workspaceScopes?.includes('git_linked_worktree') !== true) + ) { + throw new BridgeProtocolError( + 'Linked worktree reset requires a registered checkout with worktree lanes', + ); + } + const key = workspaceIsolationKey(workspaceId, workspaceInstanceId, worktree); + let guard: WorkspaceMutationQuarantine | undefined; + if (worktree != null) { + guard = await this.options.linkedWorktreeQuarantineResolver?.(workspaceId, worktree); + } else if (workspaceInstanceId != null) { + guard = await this.options.workspaceQuarantineResolver?.( + workspaceId, + workspaceInstanceId, + ); + } else { + guard = this.options.workspaceQuarantines?.get(workspaceId); + } if ( !guard || this.activeWorkspaceAssignments.size > 0 || @@ -1313,8 +1407,11 @@ export class BridgeWorker { ): Promise { const root = this.assignmentWorkspaceId(assignment); const waitingAt = Date.now(); - while (root != null && this.activeWorkspaceAssignments.has(root)) { - const active = this.activeWorkspaceAssignments.get(root)!; + for ( + let active = root == null ? undefined : this.conflictingActiveAssignment(root); + root != null && active != null; + active = this.conflictingActiveAssignment(root) + ) { if (active.id === assignment.assignmentId) throw new BridgeProtocolError( 'Code API replayed an active workspace assignment', @@ -1386,6 +1483,50 @@ export class BridgeWorker { } } + /** An active assignment on the same key, the key's parent checkout, or a lane beneath it. */ + private conflictingActiveAssignment( + key: string, + ): { id: string; done: Promise } | undefined { + for (const [activeKey, active] of this.activeWorkspaceAssignments) { + if (workspaceIsolationKeysConflict(activeKey, key)) return active; + } + return undefined; + } + + /** A lane may not run while the checkout it belongs to is quarantined. */ + /** + * A lane may not start while its checkout is quarantined, and a checkout may + * not start while any linked worktree beneath it is: root commands can reach + * `.worktrees/*`, including `git worktree remove`. + */ + private async assertLaneFamilyAvailable( + workspaceKey: string, + assignment: BridgeAssignment, + ): Promise { + const workspaceId = this.assignmentBaseWorkspaceId(assignment); + const parent = workspaceIsolationParent(workspaceKey); + if (parent != null) { + if (this.quarantinedWorkspaces.has(parent)) { + throw new Error('Parent workspace requires an explicit quarantine reset'); + } + if (workspaceId != null && parent === workspaceId) { + await this.options.workspaceQuarantines?.get(workspaceId)?.assertAvailable(); + } + return; + } + const resolveLaneGuard = this.options.linkedWorktreeQuarantineResolver; + if (workspaceId == null || workspaceKey !== workspaceId || resolveLaneGuard == null) return; + for (const key of this.quarantinedWorkspaces) { + if (workspaceIsolationParent(key) === workspaceKey) { + throw new Error('A linked worktree in this workspace requires an explicit quarantine reset'); + } + } + const names = (await this.options.linkedWorktreeNames?.(workspaceId)) ?? []; + await Promise.all( + names.map(async (name) => (await resolveLaneGuard(workspaceId, name)).assertAvailable()), + ); + } + private workspaceGuard( assignment: BridgeAssignment, ): @@ -1394,6 +1535,10 @@ export class BridgeWorker { | undefined { const workspaceId = this.assignmentBaseWorkspaceId(assignment); const instanceId = this.assignmentWorkspaceInstanceId(assignment); + const worktree = this.assignmentWorktree(assignment); + if (workspaceId != null && worktree != null) { + return this.options.linkedWorktreeQuarantineResolver?.(workspaceId, worktree); + } if (workspaceId != null && instanceId != null) { return this.options.workspaceQuarantineResolver?.( workspaceId, @@ -1412,7 +1557,27 @@ export class BridgeWorker { const workspaceId = this.assignmentBaseWorkspaceId(assignment); if (workspaceId == null) return undefined; const instanceId = this.assignmentWorkspaceInstanceId(assignment); - return workspaceIsolationKey(workspaceId, instanceId); + return workspaceIsolationKey( + workspaceId, + instanceId, + this.assignmentWorktree(assignment), + ); + } + + private assignmentWorktree(assignment: BridgeAssignment): string | undefined { + if ( + assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) + ) { + return assignment.request.worktree; + } + if ( + assignment.executionKind === 'workspace_programmatic' && + isBridgeWorkspaceProgrammaticRequest(assignment.request) + ) { + return assignment.request.body.workspace_worktree; + } + return undefined; } private assignmentBaseWorkspaceId( @@ -1580,9 +1745,11 @@ export class BridgeWorker { } if ( this.options.workspaceQuarantines != null || - this.options.workspaceQuarantineResolver != null + this.options.workspaceQuarantineResolver != null || + this.options.linkedWorktreeQuarantineResolver != null ) { await guard?.assertAvailable(); + await this.assertLaneFamilyAvailable(workspaceKey, assignment); } } catch (error) { throw new BridgeWorkspaceQuarantinedError( @@ -1615,6 +1782,14 @@ export class BridgeWorker { 'Workspace instance type is not advertised', ); } + if ( + workspaceRequest.worktree != null && + workspace.workspaceScopes?.includes('git_linked_worktree') !== true + ) { + throw new BridgeProtocolError( + 'Linked worktree lanes are not advertised for workspace', + ); + } if ( workspace.operations != null && !workspace.operations.includes(workspaceRequest.operation) @@ -1652,9 +1827,13 @@ export class BridgeWorker { ); } if ( - workspaceRequest.operation === 'edit_file' && - workspaceRequest.expectedBaseSha256 !== undefined && - !advertised.editFileFeatures?.includes('expected_base_sha256') + (workspaceRequest.operation === 'edit_file' && + workspaceRequest.expectedBaseSha256 !== undefined && + !advertised.editFileFeatures?.includes('expected_base_sha256')) || + (workspaceRequest.matching !== undefined && + !advertised.editFileFeatures?.includes('tolerant_match')) || + (workspaceRequest.edits?.some((edit) => edit.replaceAll !== undefined) === true && + !advertised.editFileFeatures?.includes('replace_all')) ) { throw new BridgeProtocolError( 'Workspace edit feature is not advertised', @@ -1694,6 +1873,7 @@ export class BridgeWorker { payload = await this.options.workspaceTools.execute( workspaceRequest, executionController.signal, + { deadlineAtMs: localDeadlineAtMs }, ); if ( workspaceRequest.operation === 'list_files' && @@ -1704,6 +1884,14 @@ export class BridgeWorker { payload; payload = compatiblePayload; } + if ( + workspaceRequest.operation === 'execute_command' && + !advertised.commandResultFeatures?.includes('lane_git') && + 'laneGit' in payload + ) { + const { laneGit: _laneGit, ...compatiblePayload } = payload; + payload = compatiblePayload; + } workspaceMutationApplied = isMutation; if (isMutation && !isWorkspaceToolResult(workspaceRequest, payload)) { throw new BridgeProtocolError( @@ -1739,9 +1927,11 @@ export class BridgeWorker { } if ( this.options.workspaceQuarantines != null || - this.options.workspaceQuarantineResolver != null + this.options.workspaceQuarantineResolver != null || + this.options.linkedWorktreeQuarantineResolver != null ) { await guard?.assertAvailable(); + await this.assertLaneFamilyAvailable(workspaceKey, assignment); } } catch (error) { throw new BridgeWorkspaceQuarantinedError( @@ -1772,6 +1962,14 @@ export class BridgeWorker { 'Workspace instance type is not advertised', ); } + if ( + assignment.request.body.workspace_worktree != null && + workspace.workspaceScopes?.includes('git_linked_worktree') !== true + ) { + throw new BridgeProtocolError( + 'Linked worktree lanes are not advertised for workspace', + ); + } this.mutationGuardArmed = true; try { this.armedWorkspaces.add(workspaceKey); @@ -1924,11 +2122,16 @@ export class BridgeWorker { ...((assignment.executionKind === 'workspace_tool' || assignment.executionKind === 'workspace_programmatic') && error instanceof WorkspaceToolError - ? { errorCode: error.code } + ? { errorCode: this.settlementErrorCode(error.code) } : {}), - error: (error instanceof Error - ? error.message - : 'Sandbox execution failed' + error: (assignment.executionKind === 'workspace_tool' && + isWorkspaceToolRequest(assignment.request) && + assignment.request.operation === 'edit_file' && + !workspaceCanReadSource(this.activeCapabilities.workspaceTools, assignment.request.workspaceId) + ? 'Workspace edit failed; source diagnostics require read access' + : error instanceof Error + ? error.message + : 'Sandbox execution failed' ).slice(0, MAX_SETTLEMENT_ERROR_LENGTH), }; } @@ -2308,6 +2511,12 @@ export class BridgeWorker { ); } + /** An older Code API refuses a settlement carrying a code it does not know. */ + private settlementErrorCode(code: WorkspaceToolErrorCode): WorkspaceToolErrorCode { + if (this.settlementErrorCodes.has(code)) return code; + return WORKSPACE_TOOL_ERROR_CODE_FALLBACKS[code] ?? code; + } + private async settleWithRetry( assignment: BridgeAssignment, settlement: BridgeSettlement, @@ -2359,6 +2568,20 @@ export class BridgeWorker { } catch (error) { lastError = error; if (signal?.aborted) break; + const legacyErrorCode = + settlement.status === 'rejected' && settlement.errorCode != null + ? WORKSPACE_TOOL_ERROR_CODE_FALLBACKS[settlement.errorCode] + : undefined; + if ( + settlement.status === 'rejected' && + legacyErrorCode != null && + error instanceof BridgeProtocolError && + error.status === 400 + ) { + /** A replica that predates the added code can still serve this settlement. */ + settlement = { ...settlement, errorCode: legacyErrorCode }; + continue; + } if ( error instanceof BridgeProtocolError && error.status != null && diff --git a/packages/code/src/workspace-cli.test.ts b/packages/code/src/workspace-cli.test.ts index 1de68818..52743bae 100644 --- a/packages/code/src/workspace-cli.test.ts +++ b/packages/code/src/workspace-cli.test.ts @@ -147,6 +147,47 @@ test('CLI requires concurrent native slots for conversation worktrees', async (t assert.match(result.stderr, /at least two workspace lease slots/i); }); +test('CLI requires concurrent native slots and no conversation worktrees for linked worktree lanes', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-lanes-')); + const workspaceRoot = join(root, 'workspace'); + await mkdir(workspaceRoot); + t.after(() => rm(root, { recursive: true, force: true })); + const run = (...extra: string[]) => + spawnSync( + process.execPath, + [ + fileURLToPath(new URL('./cli.js', import.meta.url)), + 'run', + '--worker-dir', + workspaceRoot, + '--allow-workspace-writes', + '--allow-workspace-commands', + '--linked-worktree-lanes', + ...extra, + ], + { + encoding: 'utf8', + env: { + ...process.env, + LIBRECHAT_CODE_URL: 'http://127.0.0.1:1/v1', + LIBRECHAT_CODE_WORKER_TOKEN: 'worker-secret', + LIBRECHAT_CODE_WORKER_ID: 'engineering-vm', + }, + }, + ); + const serial = run(); + assert.notEqual(serial.status, 0); + assert.match(serial.stderr, /Linked worktree lanes require .*at least two workspace lease slots/i); + const combined = run( + '--workspace-lease-slots', + '2', + '--conversation-worktree-root', + join(root, 'conversations'), + ); + assert.notEqual(combined.status, 0); + assert.match(combined.stderr, /cannot be combined with conversation worktrees/i); +}); + test('CLI advertises explicitly enabled writes without exposing the workspace root', async (t) => { const root = await mkdtemp(join(tmpdir(), 'librechat-code-cli-')); const workspaceRoot = join(root, ' '); diff --git a/packages/code/src/workspace-read-io.test.ts b/packages/code/src/workspace-read-io.test.ts new file mode 100644 index 00000000..2fdd2b90 --- /dev/null +++ b/packages/code/src/workspace-read-io.test.ts @@ -0,0 +1,147 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import * as fs from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import { promisify } from 'node:util'; + +const execFileAsync = promisify(execFile); + +// Isolate libuv saturation from the test runner and sibling tests. +const queuedRead = ` +import assert from 'node:assert/strict'; +import * as fs from 'node:fs'; +import * as fsp from 'node:fs/promises'; +import { execFileSync } from 'node:child_process'; +import { getEventListeners } from 'node:events'; +import { join } from 'node:path'; +import { mock } from 'node:test'; +import { LocalWorkspaceTools } from ${JSON.stringify( + new URL('./workspace.js', import.meta.url).href +)}; +import { captureWorkspaceRootIdentity } from ${JSON.stringify( + new URL('./root-identity.js', import.meta.url).href +)}; +import { WorkspaceRootAccess } from ${JSON.stringify( + new URL('./root-access.js', import.meta.url).href +)}; +const [root, cause, held] = process.argv.slice(1); +const tools = await LocalWorkspaceTools.create({ workspaces: [{ + id: 'primary', root, + ...(held === 'true' ? { identity: await captureWorkspaceRootIdentity(root) } : {}), +}] }); +const fifo = join(root, 'pool'); +execFileSync('mkfifo', [fifo]); +const pipe = fs.openSync(fifo, fs.constants.O_RDWR); +const probe = await fsp.open(join(root, 'file'), 'r'); +const prototype = Object.getPrototypeOf(probe); +await probe.close(); +const originalRead = prototype.read; +const controller = new AbortController(); +let entered, blocker, pendingIo, physicalFd, readSettled = false; +const started = new Promise(resolve => { entered = resolve; }); +const closing = []; +const descriptors = []; +let elapsed = 0; +const now = performance.now(); +mock.method(performance, 'now', () => now + elapsed); +function trackClose(handle) { + const originalClose = handle.close; + mock.method(handle, 'close', function (...args) { + descriptors.push(this.fd); + const promise = originalClose.apply(this, args); + closing.push(promise); + return promise; + }); +} +const originalOpen = WorkspaceRootAccess.open; +mock.method(WorkspaceRootAccess, 'open', async (...args) => { + const access = await originalOpen(...args); + trackClose(access.handle); + return access; +}); +mock.method(prototype, 'read', function (...args) { + physicalFd = this.fd; + trackClose(this); + // This actual pipe read occupies the sole libuv worker until explicitly drained. + blocker = new Promise((resolve, reject) => fs.read(pipe, Buffer.alloc(1), 0, 1, null, + error => error ? reject(error) : resolve())); + // Invoke Node's original FileHandle.read, including its active-I/O references. + pendingIo = originalRead.apply(this, args).finally(() => { readSettled = true; }); + if (cause === 'deadline') elapsed = 10_000; + if (cause === 'early-deadline') elapsed = 9_999.75; + entered(); + return pendingIo; +}); +let released = false; +function drain() { + if (!released) { released = true; fs.writeSync(pipe, Buffer.from('x')); } +} +const wallStart = Date.now(); +const pending = tools.execute({ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'file' }, controller.signal); +const rejected = assert.rejects(pending, error => error.code === (cause === 'abort' ? 'EXECUTION_ABORTED' : 'READ_LIMIT_EXCEEDED')); +await started; +if (cause === 'abort') controller.abort(); +// Always drain, even if a regression prevents bounded settlement. +const fallback = setTimeout(drain, 2_000); +try { + await rejected; + assert.equal(released, false, 'request settled before pool was drained'); + if (cause === 'early-deadline') assert.ok(performance.now() < now + 10_000, 'timer settled before the monotonic deadline'); + assert.equal(readSettled, false, 'real descriptor read remained pending'); + assert.ok(fs.fstatSync(physicalFd).isFile(), 'Node still owns the physical descriptor'); + assert.equal(closing.length, held === 'true' ? 2 : 1, 'file and held-root closes initiated'); + assert.deepEqual(getEventListeners(controller.signal, 'abort'), []); + console.log(JSON.stringify({ cause, held: held === 'true', settledBeforeDrain: true, readSettled, elapsedMs: Date.now() - wallStart })); +} finally { + clearTimeout(fallback); + drain(); + await blocker; + await pendingIo; + await Promise.all(closing); + for (const fd of descriptors) assert.throws(() => fs.fstatSync(fd), { code: 'EBADF' }); + mock.restoreAll(); + fs.closeSync(pipe); +} +`; + +for (const cause of ['abort', 'deadline', 'early-deadline'] as const) { + for (const held of [false, true]) { + test(`${cause} settles before real queued I/O drains (${ + held ? 'held' : 'legacy' + } root)`, async t => { + if (!['linux', 'darwin'].includes(process.platform)) + return t.skip('requires POSIX FIFO and descriptor access'); + const root = await fs.realpath( + await fs.mkdtemp(join(tmpdir(), 'workspace-read-io-')) + ); + t.after(() => fs.rm(root, { recursive: true, force: true })); + await fs.writeFile(join(root, 'file'), 'line\n'.repeat(300_000)); + const { stdout } = await execFileAsync( + process.execPath, + [ + '--input-type=module', + '--eval', + queuedRead, + root, + cause, + String(held), + ], + { + // Node 20 can bypass the pool for regular files via io_uring. + env: { + ...process.env, + UV_THREADPOOL_SIZE: '1', + UV_USE_IO_URING: '0', + }, + timeout: 10_000, + } + ); + const observed = JSON.parse(stdout); + assert.equal(observed.settledBeforeDrain, true); + assert.equal(observed.readSettled, false); + t.diagnostic(stdout.trim()); + }); + } +} diff --git a/packages/code/src/workspace-read.test.ts b/packages/code/src/workspace-read.test.ts new file mode 100644 index 00000000..1c6a7f96 --- /dev/null +++ b/packages/code/src/workspace-read.test.ts @@ -0,0 +1,488 @@ +import assert from 'node:assert/strict'; +import * as fs from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import test from 'node:test'; +import type { TestContext } from 'node:test'; +import type { FileHandle } from 'node:fs/promises'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; +import { WorkspaceRootAccess } from './root-access.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; +import { readRepositoryInstructions } from './instructions.js'; +import { + BRIDGE_WORKSPACE_READ_MAX_BYTES as MAX_BYTES, + isWorkspaceToolResult, +} from './protocol.js'; +import type { WorkspaceReadFileRequest } from './protocol.js'; + +async function fixture(t: TestContext, held = false) { + const parent = await fs.realpath( + await fs.mkdtemp(join(tmpdir(), 'workspace-read-')) + ); + t.after(() => fs.rm(parent, { recursive: true, force: true })); + const root = join(parent, 'root'); + await fs.mkdir(root); + const tools = await LocalWorkspaceTools.create({ + workspaces: [ + { + id: 'primary', + root, + writable: true, + ...(held + ? { identity: await captureWorkspaceRootIdentity(root) } + : {}), + }, + ], + repositoryInstructions: true, + }); + const read = async ( + startLine?: number, + maxLines?: number, + signal?: AbortSignal, + path = 'file' + ) => { + const request: WorkspaceReadFileRequest = { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path, + ...(startLine !== undefined ? { startLine } : {}), + ...(maxLines !== undefined ? { maxLines } : {}), + }; + const result = await tools.execute(request, signal); + assert.ok( + isWorkspaceToolResult(request, result), + 'current protocol accepts the window' + ); + assert.equal(result.operation, 'read_file'); + return result; + }; + const probe = await fs.open(join(root, 'file'), 'w+'); + const prototype = Object.getPrototypeOf(probe) as FileHandle; + await probe.close(); + return { parent, root, tools, read, prototype }; +} + +function hasCode(code: string) { + return (error: unknown) => + error instanceof WorkspaceToolError && error.code === code; +} + +test('small windows from large files use bounded descriptor reads, not readFile', async t => { + const { root, read, prototype } = await fixture(t, true); + await fs.writeFile( + join(root, 'file'), + 'first\nsecond\n' + 'tail\n'.repeat(300_000) + ); + const originalRead = prototype.read; + let bytesRead = 0; + t.mock.method(prototype, 'readFile', () => assert.fail('whole-file read')); + t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + const result = await originalRead.apply(this, args); + bytesRead += result.bytesRead; + return result; + } + ); + const result = await read(1, 2); + assert.equal(result.content, 'first\nsecond'); + assert.equal(result.nextStartLine, 3); + assert.ok(bytesRead <= 64 * 1024 + 3); +}); + +test('sequential line and byte pagination reconstructs large content exactly', async t => { + const { root, read } = await fixture(t); + const lines = Array.from( + { length: 1_201 }, + (_, i) => `${i}: ${'é🦊'.repeat(300)}` + ); + await fs.writeFile(join(root, 'file'), '\ufeff' + lines.join('\n') + '\n'); + const collected: string[] = []; + let start = 1; + for (;;) { + const result = await read(start, 500); + assert.ok(Buffer.byteLength(result.content) <= MAX_BYTES); + collected.push(...result.content.split('\n')); + assert.equal(result.endLine, collected.length); + if (!result.truncated) break; + assert.ok(result.nextStartLine! > start); + start = result.nextStartLine!; + } + assert.deepEqual(collected, lines); + const defaults = await read(); + assert.equal(defaults.endLine, 200); + assert.equal(defaults.nextStartLine, 201); +}); + +test('byte ceilings include separators and never split or repeat an oversized line', async t => { + const { root, read } = await fixture(t); + const exact = 'a'.repeat(MAX_BYTES); + await fs.writeFile( + join(root, 'file'), + exact + '\nx\n' + 'z'.repeat(MAX_BYTES + 1) + ); + const first = await read(1, 500); + assert.equal(first.content, exact); + assert.equal(first.nextStartLine, 2); + const second = await read(2, 500); + assert.equal(second.content, 'x'); + assert.equal(second.nextStartLine, 3); + await assert.rejects(read(3), error => { + assert.ok(hasCode('READ_LIMIT_EXCEEDED')(error)); + assert.match( + (error as Error).message, + /line 3.*later startLine.*search_text/ + ); + assert.ok((error as Error).message.length < 300); + return true; + }); + await fs.writeFile( + join(root, 'file'), + 'a'.repeat(MAX_BYTES - 2) + '\nb\nc' + ); + const separators = await read(1, 500); + assert.equal(Buffer.byteLength(separators.content), MAX_BYTES); + assert.equal(separators.endLine, 2); + assert.equal(separators.nextStartLine, 3); +}); + +test('EOF, empty lines, CRLF, BOM-only files, and starts beyond EOF retain their contract', async t => { + const { root, read } = await fixture(t); + for (const [source, expected] of [ + ['', ['']], + ['\ufeff', ['']], + ['\n', ['']], + ['\n\n', ['', '']], + ['a\n', ['a']], + ['a\r\nb\r\n', ['a\r', 'b\r']], + ['a\nb', ['a', 'b']], + ['\ufeff\ufeffa', ['\ufeffa']], + ] as const) { + await fs.writeFile(join(root, 'file'), source); + for (let line = 1; line <= expected.length; line++) { + const result = await read(line, 1); + assert.equal(result.content, expected[line - 1]); + assert.equal(result.endLine, line); + assert.equal(result.truncated, line < expected.length); + } + const past = await read(expected.length + 1); + assert.equal(past.content, ''); + assert.equal(past.endLine, expected.length); + assert.equal(past.truncated, false); + } +}); + +test('streaming decoding handles short reads, split UTF-8, CRLF, and UTF-16 surrogates', async t => { + const { root, read, prototype } = await fixture(t); + const originalRead = prototype.read; + t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + buffer: Buffer, + offset: number, + length: number, + position: number + ) { + return originalRead.call(this, { + buffer, + offset, + length: Math.min(length, 1), + position, + }); + } + ); + const text = 'é🦊\r\n\ufeffsecond\n'; + const little = Buffer.from(text, 'utf16le'); + const big = Buffer.from(little).swap16(); + for (const source of [ + Buffer.from('\ufeff' + text), + Buffer.concat([Buffer.from([0xff, 0xfe]), little]), + Buffer.concat([Buffer.from([0xfe, 0xff]), big]), + ]) { + await fs.writeFile(join(root, 'file'), source); + const result = await read(); + assert.equal(result.content, 'é🦊\r\n\ufeffsecond'); + assert.equal(result.endLine, 2); + assert.equal(result.truncated, false); + } +}); + +test('UTF-8 replacement decoding and binary bytes are bounded text, not file validation', async t => { + const { root, read } = await fixture(t); + await fs.writeFile( + join(root, 'file'), + Buffer.from([0x61, 0x0a, 0xff, 0x00, 0xe2, 0x82]) + ); + const first = await read(1, 1); + assert.equal(first.content, 'a'); + assert.equal(first.truncated, true); + const second = await read(2); + assert.equal(second.content, '\ufffd\0\ufffd'); + await fs.writeFile(join(root, 'file'), Buffer.alloc(MAX_BYTES / 2, 0xff)); + await assert.rejects(read(), hasCode('READ_LIMIT_EXCEEDED')); +}); + +test('scanning skips huge earlier lines without accumulating them', async t => { + const { root, read } = await fixture(t); + await fs.writeFile( + join(root, 'file'), + 'x'.repeat(4 * MAX_BYTES) + '\nrequested\n' + ); + assert.equal((await read(2, 1)).content, 'requested'); +}); + +test('cancellation during scanning and collection closes every read handle', async t => { + for (const startLine of [1, 300_000]) { + const { root, read, prototype } = await fixture(t); + await fs.writeFile(join(root, 'file'), 'line\n'.repeat(300_000)); + const controller = new AbortController(); + const originalRead = prototype.read; + const handles = new Set(); + let calls = 0; + const mock = t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + handles.add(this); + const result = await originalRead.apply(this, args); + if (++calls === 2) controller.abort(); + return result; + } + ); + await assert.rejects( + read(startLine, 500, controller.signal), + hasCode('EXECUTION_ABORTED') + ); + assert.ok(handles.size > 0); + for (const handle of handles) assert.equal(handle.fd, -1); + mock.mock.restore(); + } +}); + +test('far-away starts hit the scan deadline and close the descriptor', async t => { + const { root, read, prototype } = await fixture(t); + await fs.writeFile(join(root, 'file'), 'line\n'.repeat(300_000)); + const originalRead = prototype.read; + const now = performance.now(); + let elapsed = 0; + let handle: FileHandle | undefined; + t.mock.method(performance, 'now', () => now + elapsed); + t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + handle = this; + const result = await originalRead.apply(this, args); + elapsed += 6_000; + return result; + } + ); + await assert.rejects(read(300_000), hasCode('READ_LIMIT_EXCEEDED')); + assert.equal(handle?.fd, -1); +}); + +test('growth is capped at the opened extent and truncation produces a valid EOF window', async t => { + for (const change of ['grow', 'truncate'] as const) { + const { root, read, prototype } = await fixture(t); + const path = join(root, 'file'); + await fs.writeFile(path, 'first\nsecond\n'); + const originalRead = prototype.read; + let calls = 0; + const mock = t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + const result = await originalRead.apply(this, args); + if (++calls === 1) { + if (change === 'grow') + await fs.appendFile(path, 'appended\n'); + else await fs.truncate(path, 3); + } + return result; + } + ); + const result = await read(); + assert.equal( + result.content, + change === 'grow' ? 'first\nsecond' : 'fir' + ); + assert.equal(result.truncated, false); + mock.mock.restore(); + } +}); + +test('file and root replacement cannot redirect a held streaming read; descriptors close', async t => { + const { parent, root, read, prototype } = await fixture(t, true); + const path = join(root, 'file'); + await fs.writeFile(path, 'original\n' + 'inside\n'.repeat(200_000)); + await fs.writeFile(join(parent, 'secret'), 'outside secret'); + const originalRead = prototype.read; + const originalOpen = WorkspaceRootAccess.open; + let rootHandle: FileHandle | undefined; + let fileHandle: FileHandle | undefined; + t.mock.method( + WorkspaceRootAccess, + 'open', + async (...args: Parameters) => { + const access = await originalOpen(...args); + rootHandle = access.handle; + return access; + } + ); + let calls = 0; + t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + fileHandle = this; + const result = await originalRead.apply(this, args); + if (++calls === 1) { + await fs.rename(path, join(root, 'old-file')); + await fs.symlink(join(parent, 'secret'), path); + await fs.rename(root, join(parent, 'old-root')); + await fs.mkdir(root); + await fs.writeFile(path, 'replacement'); + } + return result; + } + ); + const result = await read(2, 500); + assert.equal(result.content, Array(500).fill('inside').join('\n')); + assert.equal(result.nextStartLine, 502); + assert.equal(fileHandle?.fd, -1); + assert.equal(rootHandle?.fd, -1); + await assert.rejects(read(), hasCode('REGISTRATION_INVALID')); +}); + +test('oversized lines and I/O failures close file and held-root descriptors', async t => { + const { root, read, prototype } = await fixture(t, true); + const originalRead = prototype.read; + const originalOpen = WorkspaceRootAccess.open; + let fileHandle: FileHandle | undefined; + let rootHandle: FileHandle | undefined; + t.mock.method( + WorkspaceRootAccess, + 'open', + async (...args: Parameters) => { + const access = await originalOpen(...args); + rootHandle = access.handle; + return access; + } + ); + t.mock.method( + prototype, + 'read', + async function ( + this: FileHandle, + ...args: Parameters + ) { + fileHandle = this; + return originalRead.apply(this, args); + } + ); + await fs.writeFile(join(root, 'file'), 'x'.repeat(MAX_BYTES + 1)); + await assert.rejects(read(), hasCode('READ_LIMIT_EXCEEDED')); + assert.equal(fileHandle?.fd, -1); + assert.equal(rootHandle?.fd, -1); + t.mock.method(prototype, 'read', async function (this: FileHandle) { + fileHandle = this; + throw Object.assign(new Error('read failed'), { code: 'EIO' }); + }); + await assert.rejects(read(), hasCode('INVALID_PATH')); + assert.equal(fileHandle?.fd, -1); + assert.equal(rootHandle?.fd, -1); +}); + +test('search, preview, edit, and digest-checked instruction limits remain separate', async t => { + const { root, tools } = await fixture(t, true); + await fs.writeFile(join(root, 'file'), 'needle\n'.repeat(200_000)); + const search = await tools.execute({ + protocolVersion: 1, + operation: 'search_text', + workspaceId: 'primary', + path: 'file', + query: 'needle', + }); + assert.equal(search.operation, 'search_text'); + assert.deepEqual(search.matches, []); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation, + workspaceId: 'primary', + path: 'file', + oldText: 'needle', + newText: 'changed', + }), + hasCode( + operation === 'preview_edit' + ? 'READ_LIMIT_EXCEEDED' + : 'WRITE_LIMIT_EXCEEDED' + ) + ); + } + await fs.writeFile( + join(root, 'AGENTS.md'), + 'instructions\n'.repeat(200_000) + ); + const snapshot = await readRepositoryInstructions(root); + assert.ok(snapshot); + const request: WorkspaceReadFileRequest = { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'AGENTS.md', + instructionSha256: snapshot.descriptor.sha256, + }; + const result = await tools.execute(request); + assert.ok(isWorkspaceToolResult(request, result)); + assert.equal(result.operation, 'read_file'); + assert.equal(result.content, snapshot.content); + assert.equal(result.truncated, true); + assert.equal(result.nextStartLine, undefined); + await fs.writeFile(join(root, 'AGENTS.md'), 'changed'); + await assert.rejects(tools.execute(request), hasCode('INVALID_PATH')); +}); + +test('byte-budget pagination reconstructs every complete line', async t => { + const { root, read } = await fixture(t); + const lines = Array.from( + { length: 1_201 }, + (_, i) => `${i}: ${'é🦊'.repeat(400)}` + ); + await fs.writeFile(join(root, 'file'), lines.join('\n') + '\n'); + const collected: string[] = []; + let startLine = 1; + for (;;) { + const result = await read(startLine, 500); + const selected = result.content.split('\n'); + assert.ok(Buffer.byteLength(result.content) <= MAX_BYTES); + if (result.truncated) + assert.ok(selected.length < 500, 'bytes forced continuation'); + collected.push(...selected); + if (!result.truncated) break; + assert.ok(result.nextStartLine! > startLine); + startLine = result.nextStartLine!; + } + assert.deepEqual(collected, lines); +}); diff --git a/packages/code/src/workspace-worker.test.ts b/packages/code/src/workspace-worker.test.ts index 96a5d1f0..99fcdd02 100644 --- a/packages/code/src/workspace-worker.test.ts +++ b/packages/code/src/workspace-worker.test.ts @@ -2,6 +2,7 @@ import assert from 'node:assert/strict'; import test from 'node:test'; import { BridgeProtocolError } from './protocol.js'; +import type { WorkspaceToolRequest } from './protocol.js'; import { BridgeWorker, BridgeWorkspaceQuarantinedError } from './worker.js'; import { SandboxWorkspaceTools, WorkspaceToolError } from './workspace.js'; @@ -236,6 +237,112 @@ test('worker omits pagination fields until Code API negotiates them', async () = }); }); +async function settleMissingFile( + registeredCodes: string[] | undefined, + rejectSettlement: (body: Record) => boolean = () => false, +): Promise<{ settlements: Array>; failure?: unknown }> { + const settlements: Array> = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { + throw new WorkspaceToolError('Workspace path does not exist', 'NOT_FOUND'); + }, + }, + fetchImpl: async (input, init) => { + if (!String(input).endsWith('/settle')) { + return Response.json({ + protocolVersion: 1, + workerId: 'vm-1', + incarnationId, + registeredAt: new Date().toISOString(), + leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file'], + ...(registeredCodes ? { supportedWorkspaceToolErrorCodes: registeredCodes } : {}), + }); + } + const body = JSON.parse(String(init?.body)) as Record; + settlements.push(body); + if (rejectSettlement(body)) { + return Response.json({ error: 'Invalid bridge settlement' }, { status: 400 }); + } + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + + await worker.register(); + const failure = await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'assignment-missing-file', + workerId: 'vm-1', + incarnationId, + generation: 1, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'primary', + path: 'logs/pending.log', + }, + }).then(() => undefined, (error: unknown) => error); + return { settlements, failure }; +} + +test('worker reports NOT_FOUND only to a Code API that advertised it', async () => { + const { settlements: supported } = await settleMissingFile(['NOT_FOUND']); + assert.deepEqual( + supported.map(({ status, errorCode, error }) => ({ status, errorCode, error })), + [{ status: 'rejected', errorCode: 'NOT_FOUND', error: 'Workspace path does not exist' }], + ); + + for (const registeredCodes of [undefined, [], ['SOMETHING_ELSE']]) { + const { settlements: legacy } = await settleMissingFile(registeredCodes); + assert.deepEqual( + legacy.map(({ status, errorCode, error }) => ({ status, errorCode, error })), + [{ status: 'rejected', errorCode: 'INVALID_PATH', error: 'Workspace path does not exist' }], + ); + } +}); + +test('worker resends a refused NOT_FOUND settlement with its legacy code', async () => { + const { settlements, failure } = await settleMissingFile( + ['NOT_FOUND'], + (body) => body.errorCode === 'NOT_FOUND', + ); + + assert.equal(failure, undefined); + assert.deepEqual( + settlements.map(({ errorCode }) => errorCode), + ['NOT_FOUND', 'INVALID_PATH'], + ); +}); + +test('worker does not resend a refused legacy settlement', async () => { + const { settlements, failure } = await settleMissingFile(undefined, () => true); + + assert.deepEqual(settlements.map(({ errorCode }) => errorCode), ['INVALID_PATH']); + assert.ok(failure instanceof BridgeProtocolError); + assert.equal(failure.status, 400); +}); + test('worker omits restricted workspaces that legacy registration would widen', async () => { const registrations: Array<{ operations: string[]; @@ -659,6 +766,49 @@ test('worker advertises only edit modes and features negotiated by Code API', as ]); }); +test('worker negotiates preview-only tolerant features and drops edit-only hashes', async () => { + const registrations: Array> = []; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['read_file' as const, 'preview_edit' as const, 'edit_file' as const], + workspaces: [{ id: 'primary' }], + editFileModes: ['single' as const, 'batch' as const], + editFileFeatures: ['expected_base_sha256' as const, 'tolerant_match' as const, 'replace_all' as const], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', + token: 'worker-secret', + workerId: 'vm-1', + incarnationId, + sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'native-srt', + runtimes: ['bash'], + workspaceTools: workspaceCapabilities, + }, + workspaceTools: { + capabilities: workspaceCapabilities, + async execute() { throw new Error('must not execute'); }, + }, + workspaceMutationQuarantine: mutationQuarantine(), + fetchImpl: async (_input, init) => { + const capabilities = JSON.parse(String(init?.body)).capabilities.workspaceTools; + registrations.push(capabilities); + return Response.json({ + protocolVersion: 1, workerId: 'vm-1', incarnationId, + registeredAt: new Date().toISOString(), leaseTtlMs: 60_000, + supportedWorkspaceToolOperations: ['read_file', 'preview_edit'], + supportedWorkspaceEditFileModes: ['single', 'batch'], + supportedWorkspaceEditFileFeatures: ['tolerant_match', 'replace_all', 'expected_base_sha256'], + }); + }, + }); + await worker.register(); + assert.deepEqual(registrations.at(-1)?.operations, ['read_file', 'preview_edit']); + assert.deepEqual(registrations.at(-1)?.editFileFeatures, ['tolerant_match', 'replace_all']); +}); + test('worker drops file operations when no request mode is compatible', async () => { const registrations: Array> = []; const workspaceCapabilities = { @@ -3165,6 +3315,98 @@ test('worker rejects workspace operations outside its advertised capability', as assert.match(String(settlement?.error), /operation is not advertised/i); }); +for (const [operation, extras] of [ + ['preview_edit', { matching: 'tolerant', oldText: 'a', newText: 'b' }], + ['preview_edit', { matching: 'exact', oldText: 'a', newText: 'b' }], + ['preview_edit', { edits: [{ oldText: 'a', newText: 'b', replaceAll: false }] }], + ['edit_file', { edits: [{ oldText: 'a', newText: 'b', replaceAll: true }] }], +] as const) { + test(`worker refuses unnegotiated ${operation} feature before dispatch`, async () => { + let executions = 0; + let armed = 0; + let settlement: Record | undefined; + const workspaceCapabilities = { + protocolVersion: 1 as const, + operations: ['preview_edit' as const, 'edit_file' as const], + editFileModes: ['single' as const, 'batch' as const], + workspaces: [{ id: 'primary' }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', + incarnationId, sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: true, sandboxProfile: 'native-srt', runtimes: ['bash'], + workspaceTools: workspaceCapabilities }, + workspaceTools: { capabilities: workspaceCapabilities, async execute() { + executions++; + throw new Error('must not execute'); + } }, + workspaceMutationQuarantine: mutationQuarantine(undefined, () => armed++), + fetchImpl: async (_input, init) => { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: `assignment-${operation}-${String('matching' in extras ? extras.matching : 'replace')}`, + workerId: 'vm-1', incarnationId, generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', ...extras } as WorkspaceToolRequest, + }); + assert.equal(executions, 0); + assert.equal(armed, 0); + assert.equal(settlement?.status, 'rejected'); + assert.match(String(settlement?.error), /edit feature is not advertised/i); + }); +} + +for (const [label, operations, allowed, mayRead] of [ + ['edit-only worker', ['edit_file'], ['edit_file'], false], + ['read denied by workspace', ['read_file', 'edit_file'], ['edit_file'], false], + ['read permitted by workspace', ['read_file', 'edit_file'], ['read_file', 'edit_file'], true], + ['preview permits full file reads', ['preview_edit', 'edit_file'], ['preview_edit', 'edit_file'], true], +] as const) { + test(`edit diagnostics respect ${label} authorization`, async () => { + const source = 'confidential-source-line'; + let settlement: Record | undefined; + const capabilities = { + protocolVersion: 1 as const, + operations: [...operations], + editFileModes: ['single' as const], + workspaces: [{ id: 'primary', operations: [...allowed] }], + }; + const worker = new BridgeWorker({ + codeApiUrl: 'https://code.example/v1', token: 'worker-secret', workerId: 'vm-1', + incarnationId, sandboxEndpoint: 'http://127.0.0.1:2000/api/v2', + capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'], + workspaceTools: capabilities }, + workspaceTools: { capabilities, mutationFailuresAreAtomic: true, async execute() { + throw new WorkspaceToolError(`old_text was not found; closest line: "${source}"`, 'EDIT_CONFLICT'); + } }, + workspaceMutationQuarantine: mutationQuarantine(), + fetchImpl: async (_url, init) => { + settlement = JSON.parse(String(init?.body)) as Record; + return Response.json({ protocolVersion: 1, accepted: true }); + }, + }); + await worker.executeAndSettle({ + protocolVersion: 1, + assignmentId: 'edit-source-authorization', + workerId: 'vm-1', incarnationId, generation: 4, + leaseToken: 'lease-token-that-is-long-enough-for-testing', + expiresAt: new Date(Date.now() + 5_000).toISOString(), + executionKind: 'workspace_tool', + request: { protocolVersion: 1, operation: 'edit_file', workspaceId: 'primary', + path: 'notes.txt', oldText: 'missing', newText: 'replacement' }, + }); + assert.equal(settlement?.status, 'rejected'); + assert.equal(settlement?.errorCode, 'EDIT_CONFLICT'); + assert.equal(String(settlement?.error).includes(source), mayRead); + }); +} + test('worker rejects legacy replacement writes outside its advertised mode', async () => { let executions = 0; let settlement: Record | undefined; diff --git a/packages/code/src/workspace.test.ts b/packages/code/src/workspace.test.ts index 3dd7d2f4..787fbe7d 100644 --- a/packages/code/src/workspace.test.ts +++ b/packages/code/src/workspace.test.ts @@ -15,6 +15,7 @@ import { unlink, writeFile, } from 'node:fs/promises'; +import { existsSync } from 'node:fs'; import { tmpdir } from 'node:os'; import { join, sep } from 'node:path'; import test from 'node:test'; @@ -28,6 +29,7 @@ import { SandboxWorkspaceTools, WorkspaceToolError, } from './workspace.js'; +import { captureWorkspaceRootIdentity } from './root-identity.js'; import { BRIDGE_WORKSPACE_COMMAND_MAX_BYTES, @@ -939,7 +941,7 @@ test('writable workspaces create, replace, and exactly edit files', async (t) => ], writeFileModes: ['replace', 'create'], editFileModes: ['single', 'batch'], - editFileFeatures: ['expected_base_sha256'], + editFileFeatures: ['expected_base_sha256', 'tolerant_match', 'replace_all'], listFileFeatures: ['after_path'], }); await tools.execute({ @@ -1351,7 +1353,393 @@ test('exact edits reject missing or repeated text without changing the file', as assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), 'aaa'); }); -test('writes reject symlink targets and missing parent directories', async (t) => { +test('a failed edit batch reports every failing edit and writes nothing', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'alpha\nbeta\nbeta\ngamma\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + edits: [ + { oldText: 'alpha', newText: 'ALPHA' }, + { oldText: 'beta', newText: 'BETA' }, + { oldText: 'delta', newText: 'DELTA' }, + ], + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'EDIT_CONFLICT' && + /^2 of 3 workspace edits did not apply/.test(error.message) && + /Edit 2: old_text matched 2 locations at lines 2, 3/.test(error.message) && + /Edit 3: old_text was not found/.test(error.message), + ); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); +}); + +test('long preview and edit batch errors expose every failing edit position without writing', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-batch-positions-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = `KEEP\nconst shared = '${'x'.repeat(180)}';\n`; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ workspaces: [{ id: 'primary', root, writable: true }] }); + const edits = Array.from({ length: 60 }, (_, index) => index % 2 === 0 + ? { oldText: `const shared missing_${index}`, newText: 'replacement' } + : { oldText: 'KEEP', newText: 'KEEP' }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', edits, + }), (error: unknown) => { + assert.ok(error instanceof WorkspaceToolError); + assert.equal(error.code, 'EDIT_CONFLICT'); + assert.equal(error.mutationMayHaveCommitted, false); + assert.ok(error.message.length <= 3_000); + assert.deepEqual( + [...error.message.matchAll(/\nEdit (\d+):/g)].map((match) => Number(match[1])), + Array.from({ length: 30 }, (_, index) => 2 * index + 1), + ); + return true; + }); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + +test('tolerant edits and replaceAll report how each edit matched', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + await writeFile(join(root, 'app.ts'), 'if (ok) { \r\n run();\r\n}\r\nrun();\r\n'); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + const preview = await tools.execute({ + protocolVersion: 1, + operation: 'preview_edit', + workspaceId: 'primary', + path: 'app.ts', + matching: 'tolerant', + edits: [{ oldText: 'if (ok) {\n run();\n}', newText: 'if (ok) {\n go();\n}' }], + }); + assert.equal(preview.operation, 'preview_edit'); + assert.deepEqual(preview.operation === 'preview_edit' && preview.matches, [ + { strategy: 'line-trimmed', occurrences: 1 }, + ]); + + const edit = await tools.execute({ + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'app.ts', + matching: 'tolerant', + edits: [ + { oldText: 'if (ok) {\n run();\n}', newText: 'if (ok) {\n go();\n}' }, + { oldText: 'run();', newText: 'stop();', replaceAll: true }, + ], + }); + assert.deepEqual(edit, { + protocolVersion: 1, + operation: 'edit_file', + workspaceId: 'primary', + path: 'app.ts', + replacements: 2, + bytesWritten: 32, + matches: [ + { strategy: 'line-trimmed', occurrences: 1 }, + { strategy: 'exact', occurrences: 1 }, + ], + }); + assert.equal( + await readFile(join(root, 'app.ts'), 'utf8'), + 'if (ok) {\r\n go();\r\n}\r\nstop();\r\n', + ); +}); + +test('preview and edit peel equivalent newline wrappers without doubling line breaks or losing BOM', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-newlines-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = '\uFEFFheader\r\n \tfoo bar\r\nnext\r\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const request = { + protocolVersion: 1 as const, + operation: 'preview_edit' as const, + workspaceId: 'primary', + path: 'notes.txt', + matching: 'tolerant' as const, + edits: [ + { oldText: '\n foo bar\r\n', newText: '\r\n baz qux\n' }, + { oldText: 'next', newText: 'done' }, + ], + }; + const preview = await tools.execute(request); + assert.equal(preview.operation, 'preview_edit'); + if (preview.operation !== 'preview_edit') assert.fail('expected preview result'); + assert.equal(preview.content, 'header\r\n \tbaz qux\r\ndone\r\n'); + assert.equal(preview.hasUtf8Bom, true); + assert.equal(preview.bytesWritten, Buffer.byteLength(`\uFEFF${preview.content}`)); + assert.equal(isWorkspaceToolResult(request, preview), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const editRequest = { + ...request, + operation: 'edit_file' as const, + expectedBaseSha256: preview.baseSha256, + }; + const edited = await tools.execute(editRequest); + assert.equal(edited.operation, 'edit_file'); + if (edited.operation !== 'edit_file') assert.fail('expected edit result'); + assert.deepEqual(edited.matches, [ + { strategy: 'whitespace-normalized', occurrences: 1 }, + { strategy: 'exact', occurrences: 1 }, + ]); + assert.equal(isWorkspaceToolResult(editRequest, edited), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), `\uFEFF${preview.content}`); +}); + +test('tolerant previews and edits keep source indentation when newText uses a different indent', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-indent-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'header\n foo bar\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const edit = { + protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant' as const, oldText: '\n foo bar', newText: '\n\tbaz qux', + }; + const preview = await tools.execute({ ...edit, operation: 'preview_edit' }); + assert.equal(preview.operation === 'preview_edit' && preview.content, 'header\n baz qux\n'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const result = await tools.execute({ ...edit, operation: 'edit_file' }); + assert.equal(result.operation === 'edit_file' && result.matches?.[0]?.strategy, 'whitespace-normalized'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), 'header\n baz qux\n'); +}); + +test('tolerant previews and edits retain an extra leading blank line across a fenced batch', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-extra-line-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = '\uFEFFheader\r\n foo bar\r\nend\r\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const request = { + protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant' as const, + edits: [ + { oldText: '\n foo bar', newText: '\r\n\t\r\n\tbaz qux' }, + { oldText: 'end', newText: 'done' }, + ], + }; + const previewRequest = { ...request, operation: 'preview_edit' as const }; + const preview = await tools.execute(previewRequest); + if (preview.operation !== 'preview_edit') assert.fail('expected preview result'); + assert.equal(preview.content, 'header\r\n \r\n\tbaz qux\r\ndone\r\n'); + assert.equal(preview.hasUtf8Bom, true); + assert.equal(isWorkspaceToolResult(previewRequest, preview), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const editRequest = { ...request, operation: 'edit_file' as const, + expectedBaseSha256: preview.baseSha256 }; + const edited = await tools.execute(editRequest); + if (edited.operation !== 'edit_file') assert.fail('expected edit result'); + assert.deepEqual(edited.matches, [ + { strategy: 'whitespace-normalized', occurrences: 1 }, + { strategy: 'exact', occurrences: 1 }, + ]); + assert.equal(isWorkspaceToolResult(editRequest, edited), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), `\uFEFF${preview.content}`); +}); + +test('exact preview and edit failures retain whitespace hints without writing the file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-probe-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = '\uFEFFfoo bar\r\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ workspaces: [{ id: 'primary', root, writable: true }] }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', + oldText: 'foo bar\n', newText: '', + }), (error: unknown) => error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT' && + /different whitespace \(whitespace-normalized\)/.test(error.message) && !error.mutationMayHaveCommitted); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + +test('tolerant preview and edit peel newline-only wrappers and preserve source indentation', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-newline-only-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = '\uFEFFheader\r\n foo bar\r\nnext\r\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ workspaces: [{ id: 'primary', root, writable: true }] }); + const request = { protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant' as const, oldText: '\nfoo bar\n', newText: '\n\tbaz qux\t\n' }; + const previewRequest = { ...request, operation: 'preview_edit' as const }; + const preview = await tools.execute(previewRequest); + if (preview.operation !== 'preview_edit') assert.fail('expected preview result'); + assert.equal(preview.content, 'header\r\n baz qux\r\nnext\r\n'); + assert.equal(preview.hasUtf8Bom, true); + assert.equal(isWorkspaceToolResult(previewRequest, preview), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const editRequest = { ...request, operation: 'edit_file' as const, expectedBaseSha256: preview.baseSha256 }; + const edited = await tools.execute(editRequest); + assert.equal(isWorkspaceToolResult(editRequest, edited), true); + assert.deepEqual(edited.operation === 'edit_file' && edited.matches, + [{ strategy: 'whitespace-normalized', occurrences: 1 }]); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), `\uFEFF${preview.content}`); +}); + +test('preview and edit verify a uniquely anchored indentation block larger than the repeat budget', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-large-indent-')); + t.after(() => rm(root, { recursive: true, force: true })); + const oldText = ['anchor', ...Array.from({ length: 99_999 }, () => 'x'), 'end'].join('\n'); + const original = ` ${oldText.replaceAll('\n', '\n ')}\n`; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ workspaces: [{ id: 'primary', root, writable: true }] }); + const request = { protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant' as const, oldText, newText: 'changed' }; + const previewRequest = { ...request, operation: 'preview_edit' as const }; + const preview = await tools.execute(previewRequest); + if (preview.operation !== 'preview_edit') assert.fail('expected preview result'); + assert.equal(preview.content, ' changed\n'); + assert.equal(isWorkspaceToolResult(previewRequest, preview), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const editRequest = { ...request, operation: 'edit_file' as const, expectedBaseSha256: preview.baseSha256 }; + const edited = await tools.execute(editRequest); + assert.equal(isWorkspaceToolResult(editRequest, edited), true); + assert.deepEqual(edited.operation === 'edit_file' && edited.matches, + [{ strategy: 'indentation-flexible', occurrences: 1 }]); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), preview.content); +}); + +test('tolerant previews and edits reject missing source boundary newlines without changing the file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-missing-newline-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'foo bar'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant', oldText: 'foo bar\r\n', newText: 'baz qux\n', + }), (error: unknown) => error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + +test('tolerant previews and edits reject boundary removal outside the token match', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-remove-newline-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'foo bar\r\nnext'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant', oldText: 'foo bar\r\n', newText: 'baz qux', + }), (error: unknown) => error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + +test('tolerant previews and edits refuse partial tokens without changing the file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-boundary-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'prereturn value;\n'; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, operation, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant', oldText: 'return value;', newText: 'return changed;', + }), (error: unknown) => error instanceof WorkspaceToolError && error.code === 'EDIT_CONFLICT'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + +test('large whitespace-only differences work for preview and edit_file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-large-')); + t.after(() => rm(root, { recursive: true, force: true })); + const oldText = `head ${'part '.repeat(16_000)}tail`; + const original = `before ${oldText.replace(/ /g, '\t')} after`; + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const base = { protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + matching: 'tolerant' as const, oldText, newText: 'result' }; + const preview = await tools.execute({ ...base, operation: 'preview_edit' }); + assert.equal(preview.operation === 'preview_edit' && preview.content, 'before result after'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const edited = await tools.execute({ ...base, operation: 'edit_file' }); + assert.equal(edited.operation === 'edit_file' && edited.matches?.[0]?.strategy, 'whitespace-normalized'); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), 'before result after'); +}); + +test('large replaceAll previews and edits return bounded content and accurate match counts', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-stream-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'x'.repeat(256 * 1024); + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const request = { protocolVersion: 1 as const, workspaceId: 'primary', path: 'notes.txt', + edits: [{ oldText: 'x', newText: 'y', replaceAll: true }] }; + const previewRequest = { ...request, operation: 'preview_edit' as const }; + const preview = await tools.execute(previewRequest); + if (preview.operation !== 'preview_edit') assert.fail('expected preview result'); + assert.equal(preview.content, 'y'.repeat(original.length)); + assert.equal(preview.bytesWritten, original.length); + assert.deepEqual(preview.matches, [{ strategy: 'exact', occurrences: original.length }]); + assert.equal(isWorkspaceToolResult(previewRequest, preview), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + const editRequest = { ...request, operation: 'edit_file' as const, + expectedBaseSha256: preview.baseSha256 }; + const edited = await tools.execute(editRequest); + assert.equal(edited.operation === 'edit_file' && edited.bytesWritten, original.length); + assert.deepEqual(edited.operation === 'edit_file' && edited.matches, preview.matches); + assert.equal(isWorkspaceToolResult(editRequest, edited), true); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), preview.content); +}); + +test('oversized replaceAll previews and edits fail before writing the source file', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-edit-limit-')); + t.after(() => rm(root, { recursive: true, force: true })); + const original = 'x'.repeat(10_000); + await writeFile(join(root, 'notes.txt'), original); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + for (const operation of ['preview_edit', 'edit_file'] as const) { + await assert.rejects(tools.execute({ + protocolVersion: 1, + operation, + workspaceId: 'primary', + path: 'notes.txt', + edits: [{ oldText: 'x', newText: 'y'.repeat(100_000), replaceAll: true }], + }), (error: unknown) => error instanceof WorkspaceToolError && + error.code === 'WRITE_LIMIT_EXCEEDED' && !error.mutationMayHaveCommitted); + assert.equal(await readFile(join(root, 'notes.txt'), 'utf8'), original); + } +}); + +test('writes reject symlink targets and symlinked parent directories', async (t) => { const parent = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); t.after(() => rm(parent, { recursive: true, force: true })); const root = join(parent, 'root'); @@ -1366,8 +1754,8 @@ test('writes reject symlink targets and missing parent directories', async (t) = for (const path of [ 'link.txt', - 'missing/notes.txt', 'directory-link/notes.txt', + 'directory-link/missing/notes.txt', ]) { await assert.rejects( tools.execute({ @@ -1382,6 +1770,231 @@ test('writes reject symlink targets and missing parent directories', async (t) = ); } assert.equal(await readFile(join(parent, 'outside.txt'), 'utf8'), 'outside'); + assert.deepEqual(await readdir(join(root, 'real-directory')), []); +}); + +async function heldRootTools(root: string): Promise { + return LocalWorkspaceTools.create({ + workspaces: [ + { id: 'primary', root, identity: await captureWorkspaceRootIdentity(root), writable: true }, + ], + }); +} + +test('writes create missing parent directories inside a held workspace root', async (t) => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-workspace-'))); + t.after(() => rm(root, { recursive: true, force: true })); + await mkdir(join(root, 'src')); + const tools = await heldRootTools(root); + + for (const [path, overwrite] of [ + ['src/feature/deep/new.ts', undefined], + ['fresh/created.ts', false], + ] as const) { + const result = await tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path, + content: 'export {};\n', + ...(overwrite === undefined ? {} : { overwrite }), + }); + assert.deepEqual(result, { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path, + created: true, + bytesWritten: 11, + }); + assert.equal(await readFile(join(root, ...path.split('/')), 'utf8'), 'export {};\n'); + } + assert.equal((await stat(join(root, 'src', 'feature', 'deep'))).isDirectory(), true); +}); + +test('parent creation never follows symlinks, crosses files, or leaves the workspace', async (t) => { + const parent = await realpath( + await mkdtemp(join(tmpdir(), 'librechat-code-workspace-parent-')), + ); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(parent, 'root'); + const outside = join(parent, 'outside'); + await mkdir(join(root, 'src'), { recursive: true }); + await mkdir(outside); + await writeFile(join(root, 'file.txt'), 'file'); + await symlink(outside, join(root, 'linked-outside')); + await symlink(join(root, 'src'), join(root, 'alias')); + await symlink(join(parent, 'missing-target'), join(root, 'dangling')); + const tools = await heldRootTools(root); + + for (const [path, code] of [ + ['linked-outside/new/notes.txt', 'INVALID_PATH'], + ['alias/new/notes.txt', 'INVALID_PATH'], + ['dangling/new/notes.txt', 'INVALID_PATH'], + ['file.txt/new/notes.txt', 'INVALID_PATH'], + ['../outside/new/notes.txt', 'INVALID_REQUEST'], + ['new/../../outside/notes.txt', 'INVALID_REQUEST'], + ] as const) { + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path, + content: 'blocked', + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === code && + !error.mutationMayHaveCommitted, + path, + ); + } + assert.deepEqual(await readdir(outside), []); + assert.deepEqual(await readdir(join(root, 'src')), []); + assert.deepEqual((await readdir(root)).sort(), [ + 'alias', + 'dangling', + 'file.txt', + 'linked-outside', + 'src', + ]); + await assert.rejects(stat(join(parent, 'missing-target'))); +}); + +test('a write finishes once it has created parents, even if cancelled', async (t) => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-workspace-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await heldRootTools(root); + /** Reports cancellation as soon as the parents exist. */ + const signal = { + get aborted() { + return existsSync(join(root, 'fresh', 'nested')); + }, + } as AbortSignal; + + const result = await tools.execute( + { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'fresh/nested/file.txt', + content: 'installed', + }, + signal, + ); + + assert.equal(result.operation === 'write_file' && result.created, true); + assert.equal(await readFile(join(root, 'fresh', 'nested', 'file.txt'), 'utf8'), 'installed'); +}); + +test('a write cancelled before creating parents leaves the workspace unchanged', async (t) => { + const root = await realpath(await mkdtemp(join(tmpdir(), 'librechat-code-workspace-'))); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await heldRootTools(root); + const controller = new AbortController(); + controller.abort(); + + await assert.rejects( + tools.execute( + { + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'fresh/nested/file.txt', + content: 'never installed', + }, + controller.signal, + ), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'EXECUTION_ABORTED' && + !error.mutationMayHaveCommitted, + ); + assert.deepEqual(await readdir(root), []); +}); + +test('a pathname-only workspace root reports a missing parent instead of creating it', async (t) => { + const root = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-')); + t.after(() => rm(root, { recursive: true, force: true })); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + + await assert.rejects( + tools.execute({ + protocolVersion: 1, + operation: 'write_file', + workspaceId: 'primary', + path: 'missing/notes.txt', + content: 'blocked', + }), + (error: unknown) => + error instanceof WorkspaceToolError && + error.code === 'NOT_FOUND' && + error.message === 'Workspace path does not exist', + ); + assert.deepEqual(await readdir(root), []); +}); + +test('a missing workspace path is NOT_FOUND only when reached through the workspace', async (t) => { + const parent = await mkdtemp(join(tmpdir(), 'librechat-code-workspace-parent-')); + t.after(() => rm(parent, { recursive: true, force: true })); + const root = join(parent, 'root'); + const outside = join(parent, 'outside'); + await mkdir(join(root, 'src'), { recursive: true }); + await mkdir(outside); + await writeFile(join(root, 'file.txt'), 'file'); + await symlink(join(root, 'src'), join(root, 'alias')); + await symlink(outside, join(root, 'linked-outside')); + await symlink(join(parent, 'missing-target'), join(root, 'dangling')); + const tools = await LocalWorkspaceTools.create({ + workspaces: [{ id: 'primary', root, writable: true }], + }); + const edits = [{ oldText: 'a', newText: 'b' }]; + const requests = (path: string): WorkspaceToolRequest[] => [ + { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path }, + { protocolVersion: 1, operation: 'edit_file', workspaceId: 'primary', path, edits }, + { protocolVersion: 1, operation: 'preview_edit', workspaceId: 'primary', path, edits }, + { protocolVersion: 1, operation: 'search_text', workspaceId: 'primary', query: 'x', path }, + { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary', path }, + ]; + + for (const [path, code] of [ + ['missing.log', 'NOT_FOUND'], + ['src/missing.ts', 'NOT_FOUND'], + ['not-yet/created/output.log', 'NOT_FOUND'], + ['alias/missing.ts', 'INVALID_PATH'], + ['linked-outside/missing.txt', 'INVALID_PATH'], + ['dangling', 'INVALID_PATH'], + ['file.txt/missing.txt', 'INVALID_PATH'], + ] as const) { + for (const request of requests(path)) { + await assert.rejects( + tools.execute(request), + (error: unknown) => { + assert.ok(error instanceof WorkspaceToolError); + assert.equal(error.code, code, `${request.operation} ${path}`); + assert.equal(error.message.includes(parent), false); + if (code === 'NOT_FOUND') { + assert.equal(error.message, 'Workspace path does not exist'); + } + return true; + }, + ); + } + } + for (const path of ['../outside/missing.txt', 'src/../../outside/missing.txt']) { + for (const request of requests(path)) { + await assert.rejects( + tools.execute(request), + (error: unknown) => + error instanceof WorkspaceToolError && + (error.code === 'INVALID_REQUEST' || error.code === 'INVALID_PATH'), + `${request.operation} ${path}`, + ); + } + } }); test('workspace mutations preserve existing file permissions', async (t) => { @@ -1930,6 +2543,8 @@ test('composes sandboxed commands without exposing them on unconfigured workspac assert.deepEqual(tools.capabilities.editFileModes, ['single', 'batch']); assert.deepEqual(tools.capabilities.editFileFeatures, [ 'expected_base_sha256', + 'tolerant_match', + 'replace_all', ]); assert.deepEqual(tools.capabilities.listFileFeatures, ['after_path']); assert.deepEqual( diff --git a/packages/code/src/workspace.ts b/packages/code/src/workspace.ts index 65757d39..d740d48a 100644 --- a/packages/code/src/workspace.ts +++ b/packages/code/src/workspace.ts @@ -1,6 +1,6 @@ import { createHash, randomBytes } from 'node:crypto'; import { constants } from 'node:fs'; -import { link, lstat, open, realpath, rename, stat, unlink, spawn, withWorkspaceRoot, WorkspaceRootAccessError } from './root-access.js'; +import { deferWorkspaceRootCleanup, holdsWorkspaceRoot, link, lstat, mkdir, open, realpath, rename, stat, unlink, spawn, withWorkspaceRoot, WorkspaceRootAccessError } from './root-access.js'; import { basename, dirname, isAbsolute, relative, resolve, sep } from 'node:path'; import type { FileHandle } from 'node:fs/promises'; @@ -20,10 +20,19 @@ import { isValidBridgeWorkspaceToolCapabilities, isWorkspaceToolRequest, isWorkspaceToolResult, + WORKSPACE_EDIT_FILE_FEATURES, + workspaceEditRequestReportsMatches, } from './protocol.js'; +import { + applyTextEdits, + WorkspaceEditMatchError, + WorkspaceEditOutputLimitError, +} from './edits.js'; +import type { AppliedEdits } from './edits.js'; import type { BridgeWorkspaceDescriptor, + WorkspaceEditMatch, BridgeWorkspaceToolCapabilities, WorkspaceReadFileRequest, WorkspaceReadFileResult, @@ -91,6 +100,8 @@ export interface WorkspaceToolExecutor { execute( request: WorkspaceToolRequest, signal?: AbortSignal, + /** Local epoch ms after which the worker aborts this execution. Advisory work must finish well before it. */ + context?: { deadlineAtMs?: number }, ): Promise; } @@ -124,6 +135,8 @@ const MAX_SEARCH_CANDIDATE_BYTES = 1024 * 1024; const MAX_SEARCH_CANDIDATES = 20_000; const SEARCH_TIMEOUT_MS = 10_000; const LIST_TIMEOUT_MS = 10_000; +const READ_TIMEOUT_MS = 10_000; +const READ_CHUNK_BYTES = 64 * 1024; const READ_OPERATIONS = [ 'read_file', @@ -264,10 +277,18 @@ function resolveWorkspacePath(root: string, requestedPath: string): string { return candidate; } -async function readConfinedFileBuffer( +interface WorkspaceReadControl { + signal?: AbortSignal; + deadline: number; + interrupted: boolean; +} + +async function withConfinedFile( root: string, requestedPath: string, -): Promise { + read: (handle: FileHandle, size: number) => Promise, + control?: WorkspaceReadControl, +): Promise { const candidate = resolveWorkspacePath(root, requestedPath); let handle: FileHandle | undefined; try { @@ -288,7 +309,31 @@ async function readConfinedFileBuffer( ) { throw new Error('Invalid workspace path'); } - if (openedFile.size > BRIDGE_WORKSPACE_READ_MAX_BYTES) { + return await read(handle, openedFile.size); + } catch (error) { + if (error instanceof WorkspaceToolError) throw error; + if (isMissingEntry(error)) { + throw await classifyMissingWorkspacePath(root, candidate); + } + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } finally { + if (handle) { + const closing = handle.close(); + if (control?.interrupted) { + // Node retains the descriptor and buffer until outstanding I/O drains. + deferWorkspaceRootCleanup(); + void closing.catch(() => {}); + } else await closing; + } + } +} + +async function readConfinedFileBuffer( + root: string, + requestedPath: string, +): Promise { + return withConfinedFile(root, requestedPath, async (handle, size) => { + if (size > BRIDGE_WORKSPACE_READ_MAX_BYTES) { throw new WorkspaceToolError( 'Workspace file exceeds read limit', 'READ_LIMIT_EXCEEDED', @@ -313,28 +358,222 @@ async function readConfinedFileBuffer( ); } return buffer.subarray(0, bytesRead); - } catch (error) { - if (error instanceof WorkspaceToolError) throw error; - throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + }); +} + +function interruptRead( + control: WorkspaceReadControl, + message: string, + code: 'EXECUTION_ABORTED' | 'READ_LIMIT_EXCEEDED', +): WorkspaceToolError { + control.interrupted = true; + return new WorkspaceToolError(message, code); +} + +function checkReadDeadline(control: WorkspaceReadControl): void { + if (control.signal?.aborted) { + throw interruptRead( + control, + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ); + } + if (performance.now() >= control.deadline) { + throw interruptRead( + control, + 'Workspace read exceeded its scan time limit; request an earlier startLine or use search_text to locate content', + 'READ_LIMIT_EXCEEDED', + ); + } +} + +async function readWorkspaceChunk( + handle: FileHandle, + buffer: Buffer, + length: number, + position: number, + control: WorkspaceReadControl, +): Promise { + checkReadDeadline(control); + const { signal, deadline } = control; + let timer: ReturnType | undefined; + let abort: (() => void) | undefined; + try { + return await Promise.race([ + handle + .read(buffer, 0, length, position) + .then((result) => result.bytesRead), + new Promise((_, reject) => { + abort = () => + reject( + interruptRead( + control, + 'Workspace tool execution aborted', + 'EXECUTION_ABORTED', + ), + ); + signal?.addEventListener('abort', abort, { once: true }); + if (signal?.aborted) abort(); + timer = setTimeout( + () => + reject( + interruptRead( + control, + 'Workspace read exceeded its scan time limit; retry with an earlier startLine', + 'READ_LIMIT_EXCEEDED', + ), + ), + Math.max(0, deadline - performance.now()), + ); + }), + ]); } finally { - await handle?.close(); + if (timer !== undefined) clearTimeout(timer); + if (abort) signal?.removeEventListener('abort', abort); } } async function readConfinedFile( root: string, - requestedPath: string, -): Promise { - const decoded = decodeWorkspaceText( - await readConfinedFileBuffer(root, requestedPath), - ); - if (Buffer.byteLength(decoded, 'utf8') > BRIDGE_WORKSPACE_READ_MAX_BYTES) { - throw new WorkspaceToolError( - 'Workspace file exceeds read limit', - 'READ_LIMIT_EXCEEDED', + request: WorkspaceReadFileRequest, + signal?: AbortSignal, +): Promise { + const startLine = request.startLine ?? 1; + const maxLines = request.maxLines ?? 200; + const control: WorkspaceReadControl = { + signal, + deadline: performance.now() + READ_TIMEOUT_MS, + interrupted: false, + }; + const read = async ( + handle: FileHandle, + size: number, + ): Promise => { + const buffer = Buffer.allocUnsafe(READ_CHUNK_BYTES); + // Fix the read extent at admission so an appending writer cannot extend the scan. + let position = 0; + const header = Buffer.alloc(3); + let headerBytes = 0; + while (headerBytes < header.length && position < size) { + const bytes = await readWorkspaceChunk( + handle, + buffer, + Math.min(header.length - headerBytes, size - position), + position, + control, + ); + checkReadDeadline(control); + if (bytes === 0) break; + buffer.copy(header, headerBytes, 0, bytes); + headerBytes += bytes; + position += bytes; + } + const utf16le = + headerBytes >= 2 && header[0] === 0xff && header[1] === 0xfe; + const utf16be = + headerBytes >= 2 && header[0] === 0xfe && header[1] === 0xff; + const bomBytes = + utf16le || utf16be + ? 2 + : headerBytes === 3 && + header[0] === 0xef && + header[1] === 0xbb && + header[2] === 0xbf + ? 3 + : 0; + // Preserve ordinary reads' replacement decoding and BOM-marked UTF-16 support. + const decoder = new TextDecoder( + utf16le ? 'utf-16le' : utf16be ? 'utf-16be' : 'utf-8', + { ignoreBOM: true }, ); - } - return decoded; + const lines: string[] = []; + let fragments: string[] = []; + let line = 1; + let lineBytes = 0; + let returnedBytes = 0; + let hasText = false; + let truncated = false; + const consume = (text: string): void => { + let offset = 0; + while (offset < text.length) { + checkReadDeadline(control); + const newline = text.indexOf('\n', offset); + const end = newline < 0 ? text.length : newline; + if (line >= startLine) { + if (lines.length === maxLines) { + truncated = true; + return; + } + const fragment = text.slice(offset, end); + const bytes = Buffer.byteLength(fragment, 'utf8'); + if ( + returnedBytes + (lines.length > 0 ? 1 : 0) + lineBytes + bytes > + BRIDGE_WORKSPACE_READ_MAX_BYTES + ) { + if (lines.length === 0) { + throw new WorkspaceToolError( + `Workspace file exceeds read limit: line ${line} cannot fit within ${BRIDGE_WORKSPACE_READ_MAX_BYTES} UTF-8 bytes; request a later startLine or use search_text`, + 'READ_LIMIT_EXCEEDED', + ); + } + truncated = true; + return; + } + if (fragment.length > 0) fragments.push(fragment); + lineBytes += bytes; + } + hasText ||= end > offset; + if (newline < 0) return; + if (line >= startLine) { + lines.push(fragments.join('')); + returnedBytes += (lines.length > 1 ? 1 : 0) + lineBytes; + fragments = []; + lineBytes = 0; + } + line += 1; + hasText = false; + offset = newline + 1; + } + }; + consume( + decoder.decode(header.subarray(bomBytes, headerBytes), { + stream: true, + }), + ); + while (!truncated && position < size) { + const bytes = await readWorkspaceChunk( + handle, + buffer, + Math.min(buffer.length, size - position), + position, + control, + ); + checkReadDeadline(control); + if (bytes === 0) break; + position += bytes; + consume(decoder.decode(buffer.subarray(0, bytes), { stream: true })); + } + if (!truncated) { + consume(decoder.decode()); + if (!truncated && line >= startLine && (hasText || line === 1)) { + lines.push(fragments.join('')); + } + } + checkReadDeadline(control); + const endLine = startLine + lines.length - 1; + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'read_file', + workspaceId: request.workspaceId, + path: request.path, + content: lines.join('\n'), + startLine, + endLine, + truncated, + ...(truncated ? { nextStartLine: endLine + 1 } : {}), + }; + }; + return withConfinedFile(root, request.path, read, control); } interface WorkspaceRoot { @@ -360,6 +599,147 @@ async function verifyDirectoryPathHasNoSymlinks( return currentIdentity; } +function isMissingEntry(error: unknown): boolean { + return (error as NodeJS.ErrnoException | undefined)?.code === 'ENOENT'; +} + +interface DirectoryIdentity { + path: string; + dev: bigint | number; + ino: bigint | number; +} + +/** Whether each ancestor observed before a miss is still the same real directory. */ +async function ancestorsUnchanged(ancestors: readonly DirectoryIdentity[]): Promise { + for (const ancestor of ancestors) { + try { + const current = await lstat(ancestor.path); + if ( + current.isSymbolicLink() || + !current.isDirectory() || + current.dev !== ancestor.dev || + current.ino !== ancestor.ino + ) { + return false; + } + } catch { + return false; + } + } + return true; +} + +/** + * Reports an absent target as `NOT_FOUND` only when the walk from the root + * reaches the missing entry through real directories, the root included, that + * are unchanged after the miss. A symlink, a file used as a directory, or an ancestor replaced + * during the walk stays a path-safety rejection, so the distinction never + * describes a path beyond the root. + */ +async function classifyMissingWorkspacePath( + root: string, + candidate: string, +): Promise { + const invalid = new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + const segments = relative(root, candidate).split(sep).filter(Boolean); + const ancestors: DirectoryIdentity[] = []; + try { + const rootEntry = await lstat(root); + if (rootEntry.isSymbolicLink() || !rootEntry.isDirectory()) return invalid; + ancestors.push({ path: root, dev: rootEntry.dev, ino: rootEntry.ino }); + } catch { + return invalid; + } + let current = root; + for (const [index, segment] of segments.entries()) { + current = resolve(current, segment); + let entry: Awaited>; + try { + entry = await lstat(current); + } catch (error) { + return isMissingEntry(error) && (await ancestorsUnchanged(ancestors)) + ? new WorkspaceToolError('Workspace path does not exist', 'NOT_FOUND') + : invalid; + } + if (index === segments.length - 1) return invalid; + if (entry.isSymbolicLink() || !entry.isDirectory()) return invalid; + ancestors.push({ path: current, dev: entry.dev, ino: entry.ino }); + } + return invalid; +} + +/** + * Creates each missing ancestor of a write target as a real directory beneath + * the root, one verified level at a time, and syncs each new entry into its + * parent. Creation needs a held root descriptor, so `mkdirat` is anchored to + * the root; a pathname-only root creates nothing and its write reports the + * missing parent. An existing symlink or file stops the walk. + * + * Created directories are a committed side effect, never rolled back: once one + * exists, a later failure of this write is reported as a possible mutation, so + * the worker quarantines rather than claiming an atomic rejection. + */ +async function createMissingParentDirectories( + root: string, + candidate: string, +): Promise { + const created: string[] = []; + if (!holdsWorkspaceRoot()) return created; + const segments = relative(root, dirname(candidate)).split(sep).filter(Boolean); + let current = root; + try { + for (const segment of segments) { + current = resolve(current, segment); + let entry: Awaited> | undefined; + try { + entry = await lstat(current); + } catch (error) { + if (!isMissingEntry(error)) throw error; + } + let createdHere = false; + if (entry == null) { + try { + await mkdir(current, 0o777); + created.push(current); + createdHere = true; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== 'EEXIST') throw error; + } + entry = await lstat(current); + /** Also after a raced `EEXIST`: the write relies on that entry too. */ + await syncWorkspaceDirectory(dirname(current)); + } + if ( + entry.isSymbolicLink() || + !entry.isDirectory() || + (createdHere && !isWithinRoot(root, await realpath(current))) + ) { + throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); + } + } + return created; + } catch (error) { + throw afterCreatedDirectories(created, classifyWritePathValidationError(error)); + } +} + +function afterCreatedDirectories( + created: readonly string[], + error: WorkspaceToolError, +): WorkspaceToolError { + if (created.length === 0 || error.mutationMayHaveCommitted) return error; + return new WorkspaceToolError(error.message, error.code, true); +} + +function assertWithinWriteLimit(content: Buffer): void { + if (content.byteLength > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) { + throw new WorkspaceToolError( + 'Workspace file exceeds write limit', + 'WRITE_LIMIT_EXCEEDED', + ); + } +} + function classifyWritePathValidationError(error: unknown): WorkspaceToolError { if (error instanceof WorkspaceToolError) return error; const code = (error as NodeJS.ErrnoException).code; @@ -472,12 +852,7 @@ async function atomicWriteConfinedFile( allowOverwrite = true, ): Promise<{ created: boolean }> { throwIfAborted(signal); - if (content.byteLength > BRIDGE_WORKSPACE_WRITE_MAX_BYTES) { - throw new WorkspaceToolError( - 'Workspace file exceeds write limit', - 'WRITE_LIMIT_EXCEEDED', - ); - } + assertWithinWriteLimit(content); const candidate = resolveWorkspacePath(root, requestedPath); const parent = dirname(candidate); let canonicalParent: string; @@ -489,6 +864,9 @@ async function atomicWriteConfinedFile( throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); } } catch (error) { + if (isMissingEntry(error)) { + throw await classifyMissingWorkspacePath(root, parent); + } throw classifyWritePathValidationError(error); } @@ -662,14 +1040,26 @@ async function writeWorkspaceFile( signal?: AbortSignal, ): Promise { const content = Buffer.from(request.content, 'utf8'); - const { created } = await atomicWriteConfinedFile( + throwIfAborted(signal); + assertWithinWriteLimit(content); + const directories = await createMissingParentDirectories( root, - request.path, - content, - signal, - undefined, - request.overwrite !== false, + resolveWorkspacePath(root, request.path), ); + let created: boolean; + try { + /** Once parents exist, finish the write rather than abandon them to a cancellation. */ + ({ created } = await atomicWriteConfinedFile( + root, + request.path, + content, + directories.length === 0 ? signal : undefined, + undefined, + request.overwrite !== false, + )); + } catch (error) { + throw afterCreatedDirectories(directories, classifyWritePathValidationError(error)); + } return { protocolVersion: BRIDGE_PROTOCOL_VERSION, operation: 'write_file', @@ -722,7 +1112,10 @@ async function editWorkspaceFile( 'EDIT_CONFLICT', ); } - const { updated, replacements } = applyWorkspaceEdits(original, request); + const { updated, replacements, matches } = applyWorkspaceEdits( + original, + request, + ); await atomicWriteConfinedFile( root, request.path, @@ -741,9 +1134,13 @@ async function editWorkspaceFile( path: request.path, replacements, bytesWritten: updated.byteLength, + ...(matches ? { matches } : {}), }; } catch (error) { if (error instanceof WorkspaceToolError) throw error; + if (isMissingEntry(error)) { + throw await classifyMissingWorkspacePath(root, candidate); + } throw classifyWritePathValidationError(error); } finally { await opened?.close().catch(() => undefined); @@ -753,7 +1150,7 @@ async function editWorkspaceFile( function applyWorkspaceEdits( original: Buffer, request: WorkspaceEditFileRequest | WorkspacePreviewEditRequest, -): { updated: Buffer; replacements: number } { +): { updated: Buffer; replacements: number; matches?: WorkspaceEditMatch[] } { const hasBom = original[0] === 0xef && original[1] === 0xbb && original[2] === 0xbf; const body = hasBom ? original.subarray(3) : original; @@ -767,20 +1164,19 @@ function applyWorkspaceEdits( const edits = request.edits ?? [ { oldText: request.oldText ?? '', newText: request.newText ?? '' }, ]; - let updatedText = text; - for (const edit of edits) { - const first = updatedText.indexOf(edit.oldText); - if (first < 0 || updatedText.indexOf(edit.oldText, first + 1) >= 0) { - throw new WorkspaceToolError( - 'Workspace edit must match exactly once', - 'EDIT_CONFLICT', - ); + let applied: AppliedEdits; + try { + applied = applyTextEdits(text, edits, request.matching ?? 'exact'); + } catch (error) { + if (error instanceof WorkspaceEditMatchError) { + throw new WorkspaceToolError(error.message, 'EDIT_CONFLICT'); + } + if (error instanceof WorkspaceEditOutputLimitError) { + throw new WorkspaceToolError(error.message, 'WRITE_LIMIT_EXCEEDED'); } - updatedText = - updatedText.slice(0, first) + - edit.newText + - updatedText.slice(first + edit.oldText.length); + throw error; } + const updatedText = applied.text; const updatedBody = Buffer.from(updatedText, 'utf8'); const updated = hasBom ? Buffer.concat([Buffer.from([0xef, 0xbb, 0xbf]), updatedBody]) @@ -791,7 +1187,13 @@ function applyWorkspaceEdits( 'WRITE_LIMIT_EXCEEDED', ); } - return { updated, replacements: edits.length }; + return { + updated, + replacements: edits.length, + ...(workspaceEditRequestReportsMatches(request) + ? { matches: applied.matches } + : {}), + }; } async function previewWorkspaceEdit( @@ -806,7 +1208,10 @@ async function previewWorkspaceEdit( 'EXECUTION_ABORTED', ); } - const { updated, replacements } = applyWorkspaceEdits(original, request); + const { updated, replacements, matches } = applyWorkspaceEdits( + original, + request, + ); if (signal?.aborted) { throw new WorkspaceToolError( 'Workspace tool execution aborted', @@ -825,6 +1230,7 @@ async function previewWorkspaceEdit( baseSha256: createHash('sha256').update(original).digest('hex'), replacements, bytesWritten: updated.byteLength, + ...(matches ? { matches } : {}), }; } @@ -987,7 +1393,10 @@ async function searchWorkspace( let canonicalTarget: string; try { canonicalTarget = await realpath(target); - } catch { + } catch (error) { + if (isMissingEntry(error)) { + throw await classifyMissingWorkspacePath(root, target); + } throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); } if (!isWithinRoot(root, canonicalTarget)) @@ -1038,7 +1447,9 @@ async function searchWorkspace( } catch (error) { if ( error instanceof WorkspaceToolError && - (error.code === 'INVALID_PATH' || error.code === 'READ_LIMIT_EXCEEDED') + (error.code === 'INVALID_PATH' || + error.code === 'NOT_FOUND' || + error.code === 'READ_LIMIT_EXCEEDED') ) { continue; } @@ -1145,6 +1556,13 @@ async function listWorkspaceFiles( ); } catch (error) { if (error instanceof WorkspaceToolError) throw error; + if (isMissingEntry(error)) { + throw await withinListDeadline( + classifyMissingWorkspacePath(root, target), + signal, + deadline, + ); + } throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); } if (!isWithinRoot(root, canonicalTarget)) { @@ -1159,6 +1577,13 @@ async function listWorkspaceFiles( ).isDirectory(); } catch (error) { if (error instanceof WorkspaceToolError) throw error; + if (isMissingEntry(error)) { + throw await withinListDeadline( + classifyMissingWorkspacePath(root, target), + signal, + deadline, + ); + } throw new WorkspaceToolError('Invalid workspace path', 'INVALID_PATH'); } const normalizedRequestedResultPath = request.path @@ -1497,7 +1922,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { ...(anyWritable ? { writeFileModes: ['replace', 'create'] } : {}), ...(anyWritable ? { editFileModes: ['single', 'batch'] } : {}), ...(anyWritable - ? { editFileFeatures: ['expected_base_sha256'] } + ? { editFileFeatures: [...WORKSPACE_EDIT_FILE_FEATURES] } : {}), listFileFeatures: ['after_path'], }; @@ -1633,31 +2058,7 @@ export class LocalWorkspaceTools implements WorkspaceToolExecutor { ) { throw new WorkspaceToolError('Invalid workspace read', 'INVALID_REQUEST'); } - const content = await readConfinedFile(root, request.path); - if (signal?.aborted) { - throw new WorkspaceToolError( - 'Workspace tool execution aborted', - 'EXECUTION_ABORTED', - ); - } - const lines = content.endsWith('\n') - ? content.slice(0, -1).split('\n') - : content.split('\n'); - const selected = lines.slice(startLine - 1, startLine - 1 + maxLines); - const endLine = startLine + selected.length - 1; - const truncated = endLine < lines.length; - - return { - protocolVersion: BRIDGE_PROTOCOL_VERSION, - operation: 'read_file', - workspaceId: request.workspaceId, - path: request.path, - content: selected.join('\n'), - startLine, - endLine, - truncated, - ...(truncated ? { nextStartLine: endLine + 1 } : {}), - }; + return readConfinedFile(root, request, signal); } } diff --git a/packages/code/src/worktree-retirement.test.ts b/packages/code/src/worktree-retirement.test.ts new file mode 100644 index 00000000..f10ed8f3 --- /dev/null +++ b/packages/code/src/worktree-retirement.test.ts @@ -0,0 +1,592 @@ +import assert from 'node:assert/strict'; +import { execFile } from 'node:child_process'; +import { mkdir, mkdtemp, readFile, realpath, rename, rm, stat, utimes, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; +import test from 'node:test'; + +import { LinkedWorktreeWorkspaceTools } from './linked-worktrees.js'; +import { LocalWorkspaceTools, WorkspaceToolError } from './workspace.js'; +import { + WorktreeRetirementScheduler, + describeWorktreeRetirement, + retireStaleWorktrees, + worktreeRetirementSettings, +} from './worktree-retirement.js'; + +import type { NativeWorkspaceCommandPool } from './native-pool.js'; +import type { NativeProcessSandboxOptions } from './native-process.js'; +import type { WorktreeRetirementOptions } from './worktree-retirement.js'; + +const execFileAsync = promisify(execFile); +const DAY_MS = 24 * 60 * 60 * 1000; + +async function git(cwd: string, ...args: string[]): Promise { + const { stdout } = await execFileAsync( + 'git', + ['-c', 'user.name=test', '-c', 'user.email=test@example.com', '-c', 'commit.gpgsign=false', ...args], + { cwd }, + ); + return stdout.trim(); +} + +async function exists(path: string): Promise { + try { + await stat(path); + return true; + } catch { + return false; + } +} + +/** A checkout on `main` with an `origin` bare remote and an ignored `node_modules`. */ +async function repository(t: test.TestContext): Promise { + const parent = await realpath(await mkdtemp(join(tmpdir(), 'worktree-retirement-'))); + t.after(() => rm(parent, { recursive: true, force: true })); + const remote = join(parent, 'remote.git'); + await git(parent, 'init', '-q', '--bare', remote); + const root = join(parent, 'repo'); + await mkdir(root); + await git(root, 'init', '-q', '-b', 'main'); + await writeFile(join(root, 'README.md'), 'root\n'); + await writeFile(join(root, '.gitignore'), 'node_modules/\n.worktrees/\n'); + await git(root, 'add', '.'); + await git(root, 'commit', '-q', '-m', 'init'); + await git(root, 'remote', 'add', 'origin', remote); + await git(root, 'push', '-q', 'origin', 'main'); + await mkdir(join(root, '.worktrees')); + return root; +} + +/** A task worktree with one commit of its own and ignored dependencies. */ +async function worktree(root: string, name: string, push = true): Promise { + await git(root, 'worktree', 'add', '-q', '-b', name, `.worktrees/${name}`); + const path = join(root, '.worktrees', name); + await writeFile(join(path, `${name}.txt`), `${name}\n`); + await git(path, 'add', '.'); + await git(path, 'commit', '-q', '-m', name); + if (push) await git(path, 'push', '-q', 'origin', name); + await mkdir(join(path, 'node_modules', 'dependency'), { recursive: true }); + await writeFile(join(path, 'node_modules', 'dependency', 'index.js'), 'module.exports = 1;\n'); + return path; +} + +/** Backdate every on-disk activity signal, as if nobody had touched the worktree for `days`. */ +async function age(root: string, name: string, days = 30): Promise { + const when = new Date(Date.now() - days * DAY_MS); + const metadata = join(root, '.git', 'worktrees', name); + const paths = [ + join(root, '.worktrees', name), + metadata, + ...['HEAD', 'index', join('logs', 'HEAD'), 'ORIG_HEAD', 'FETCH_HEAD'].map((path) => join(metadata, path)), + ]; + for (const path of paths) { + await utimes(path, when, when).catch(() => undefined); + } +} + +function sources(root: string): WorktreeRetirementOptions['sources'] { + return [{ workspaceId: 'repo', root }]; +} + +/** A command pool whose lane commands wait until released. */ +function blockingPool(): { + pool: NativeWorkspaceCommandPool; + started: Promise; + finish: () => void; +} { + let started!: () => void; + let finish!: () => void; + const startedPromise = new Promise((resolve) => { started = resolve; }); + const finished = new Promise((resolve) => { finish = resolve; }); + const pool = { + async registerRoot() {}, + async unregisterRoot() {}, + async execute(request: { workspaceId: string }) { + started(); + await finished; + return { + protocolVersion: 1, + operation: 'execute_command', + workspaceId: request.workspaceId, + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + }; + }, + } as unknown as NativeWorkspaceCommandPool; + return { pool, started: startedPromise, finish }; +} + +async function laneTools(root: string, pool?: NativeWorkspaceCommandPool): Promise { + const delegate = await LocalWorkspaceTools.create({ + repositoryInstructions: false, + workspaces: [{ id: 'repo', root, writable: true }], + }); + return new LinkedWorktreeWorkspaceTools({ + commandPool: pool, + delegate, + sources: new Map([ + ['repo', { + root, + command: { workspaceRoot: root } as NativeProcessSandboxOptions, + repositoryInstructions: false, + writable: true, + }], + ]), + }); +} + +const laneCommand = (worktree: string) => ({ + protocolVersion: 1 as const, + operation: 'execute_command' as const, + workspaceId: 'repo', + worktree, + command: 'true', +}); + +const pause = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms)); + +test('retires a clean, pushed, idle worktree and keeps its branch for recovery', async (t) => { + const root = await repository(t); + const path = await worktree(root, 'done'); + await age(root, 'done'); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, ['repo:done']); + assert.deepEqual(summary.kept, {}); + assert.equal(await exists(path), false, 'the worktree and its ignored files are gone'); + assert.equal(await exists(join(root, '.git', 'worktrees', 'done')), false); + assert.doesNotMatch(await git(root, 'worktree', 'list', '--porcelain'), /\.worktrees\/done/); + const branch = await git(root, 'rev-parse', '--verify', 'refs/heads/done'); + assert.equal(branch, await git(root, 'rev-parse', '--verify', 'refs/remotes/origin/done')); + + await git(root, 'worktree', 'add', '-q', '.worktrees/done', 'done'); + assert.equal(await readFile(join(path, 'done.txt'), 'utf8'), 'done\n'); + assert.equal(await git(root, 'status', '--porcelain'), '', 'the main checkout is untouched'); + assert.equal(await git(root, 'branch', '--show-current'), 'main'); +}); + +test('keeps every worktree that could lose work or is not a linked worktree', async (t) => { + const root = await repository(t); + const worktrees = join(root, '.worktrees'); + + const dirty = await worktree(root, 'dirty'); + await writeFile(join(dirty, 'dirty.txt'), 'changed\n'); + const untracked = await worktree(root, 'untracked'); + await writeFile(join(untracked, 'notes.txt'), 'not yet added\n'); + await worktree(root, 'unpushed', false); + await worktree(root, 'locked'); + await git(root, 'worktree', 'lock', '.worktrees/locked'); + const merging = await worktree(root, 'merging'); + await worktree(root, 'side'); + await git(merging, 'merge', '-q', '--no-commit', '--no-ff', 'side'); + await worktree(root, 'recent'); + const detached = await worktree(root, 'detached'); + await git(detached, 'checkout', '-q', '--detach'); + await writeFile(join(detached, 'local.txt'), 'local\n'); + await git(detached, 'add', '.'); + await git(detached, 'commit', '-q', '-m', 'detached and unpushed'); + await mkdir(join(worktrees, 'forged')); + await writeFile(join(worktrees, 'forged', '.git'), `gitdir: ${join(root, '.git', 'worktrees', 'side')}\n`); + await mkdir(join(worktrees, 'plain')); + for (const name of ['dirty', 'untracked', 'unpushed', 'locked', 'merging', 'side', 'detached']) { + await age(root, name); + } + await age(root, 'forged'); + await age(root, 'plain'); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, ['repo:side']); + assert.deepEqual(summary.kept, { + dirty: 2, + unpushed: 2, + locked: 1, + operation: 1, + recent: 1, + unverified: 2, + }); + for (const name of ['dirty', 'untracked', 'unpushed', 'locked', 'merging', 'recent', 'detached', 'forged', 'plain']) { + assert.ok(await exists(join(worktrees, name)), `${name} is kept`); + } + assert.equal(await readFile(join(dirty, 'dirty.txt'), 'utf8'), 'changed\n'); + assert.equal(await readFile(join(untracked, 'notes.txt'), 'utf8'), 'not yet added\n'); + assert.ok(await exists(join(root, '.git', 'worktrees', 'merging', 'MERGE_HEAD'))); + assert.equal(await git(root, 'status', '--porcelain'), ''); + assert.equal(await readFile(join(root, 'README.md'), 'utf8'), 'root\n'); +}); + +test('a detached HEAD is retired only when a remote-tracking ref contains it', async (t) => { + const root = await repository(t); + const path = await worktree(root, 'review'); + await git(path, 'checkout', '-q', '--detach'); + await age(root, 'review'); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, ['repo:review']); + assert.equal(await exists(path), false); +}); + +test('a squash- or rebase-merged branch whose remote branch was deleted is retired; unmerged work is kept', async (t) => { + const root = await repository(t); + const parent = join(root, '..'); + const tracked = async (name: string): Promise => { + const path = await worktree(root, name); + await git(path, 'branch', '-q', '--set-upstream-to', `origin/${name}`); + return path; + }; + const squashed = await tracked('squashed'); + await writeFile(join(squashed, 'second.txt'), 'second\n'); + await git(squashed, 'add', '.'); + await git(squashed, 'commit', '-q', '-m', 'second'); + await git(squashed, 'push', '-q', 'origin', 'squashed'); + await tracked('rebased'); + await tracked('abandoned'); + const review = 'review-16677-1f35df8'; + await git(root, 'worktree', 'add', '-q', '--detach', `.worktrees/${review}`, await git(squashed, 'rev-parse', 'HEAD')); + await git(root, 'worktree', 'add', '-q', '-b', 'ahead', '.worktrees/ahead'); + const ahead = join(root, '.worktrees', 'ahead'); + await git(ahead, 'push', '-q', '-u', 'origin', 'ahead'); + await writeFile(join(ahead, 'later.txt'), 'later\n'); + await git(ahead, 'add', '.'); + await git(ahead, 'commit', '-q', '-m', 'later, also landed on main but never pushed here'); + + const hosting = join(parent, 'hosting'); + await git(parent, 'clone', '-q', '-b', 'main', join(parent, 'remote.git'), hosting); + await git(hosting, 'merge', '-q', '--squash', 'origin/squashed'); + await git(hosting, 'commit', '-q', '-m', 'squash merge'); + await git(hosting, 'cherry-pick', 'origin/rebased'); + await writeFile(join(hosting, 'later.txt'), 'later\n'); + await git(hosting, 'add', '.'); + await git(hosting, 'commit', '-q', '-m', 'later'); + await git(hosting, 'push', '-q', 'origin', 'main'); + await git(hosting, 'push', '-q', 'origin', '--delete', 'squashed', 'rebased', 'abandoned'); + await git(root, 'fetch', '-q', '--prune', 'origin'); + for (const name of ['squashed', 'rebased', 'abandoned', review, 'ahead']) await age(root, name); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual([...summary.retired].sort(), ['repo:rebased', `repo:${review}`, 'repo:squashed']); + assert.deepEqual(summary.kept, { unpushed: 2 }); + assert.ok(await exists(join(root, '.worktrees', 'abandoned')), 'unmerged work with a deleted remote is kept'); + assert.ok(await exists(ahead), 'commits beyond a live upstream are kept even when main has the same change'); + for (const branch of ['squashed', 'rebased', 'abandoned', 'ahead']) { + await git(root, 'rev-parse', '--verify', `refs/heads/${branch}`); + } +}); + +test('never touches the main checkout or worktrees outside .worktrees', async (t) => { + const root = await repository(t); + const outside = join(root, '..', 'outside'); + await git(root, 'worktree', 'add', '-q', '-b', 'outside', outside); + await git(outside, 'push', '-q', 'origin', 'outside'); + const nested = join(root, 'nested'); + await git(root, 'worktree', 'add', '-q', '-b', 'nested', nested); + await git(nested, 'push', '-q', 'origin', 'nested'); + const old = new Date(Date.now() - 30 * DAY_MS); + for (const path of [root, join(root, '.git'), join(root, '.git', 'HEAD'), join(root, '.git', 'index'), outside, nested]) { + await utimes(path, old, old); + } + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, []); + assert.ok(await exists(join(root, 'README.md'))); + assert.ok(await exists(outside)); + assert.ok(await exists(nested)); + assert.equal((await git(root, 'worktree', 'list', '--porcelain')).match(/^worktree /gm)?.length, 3); +}); + +test('this worker\'s own lane use counts as activity, and an active lane is never retired', async (t) => { + const root = await repository(t); + const path = await worktree(root, 'busy'); + await age(root, 'busy'); + const { pool, started, finish } = blockingPool(); + const tools = await laneTools(root, pool); + + const running = tools.execute(laneCommand('busy')); + await started; + await pause(5); + const during = await retireStaleWorktrees({ sources: sources(root), activity: tools, idleMs: 1 }); + assert.deepEqual(during.kept, { active: 1 }); + assert.ok(await exists(path)); + + finish(); + await running; + const recent = await retireStaleWorktrees({ sources: sources(root), activity: tools }); + assert.deepEqual(recent.kept, { recent: 1 }, 'the default threshold counts this process\'s last use'); + assert.ok(await exists(path)); + + await pause(5); + const after = await retireStaleWorktrees({ sources: sources(root), activity: tools, idleMs: 1 }); + assert.deepEqual(after.retired, ['repo:busy']); + assert.equal(await exists(path), false); + await assert.rejects( + tools.execute(laneCommand('busy')), + (error: unknown) => error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', + ); +}); + +test('a lane request that arrives during retirement waits, then finds the lane gone', async (t) => { + const root = await repository(t); + await worktree(root, 'leaving'); + let executions = 0; + const pool = { + async registerRoot() {}, + async unregisterRoot() {}, + async execute() { + executions += 1; + throw new Error('a retired lane must not run commands'); + }, + } as unknown as NativeWorkspaceCommandPool; + const tools = await laneTools(root, pool); + let release!: () => void; + const gate = new Promise((resolve) => { release = resolve; }); + + const retiring = tools.whileIdle('repo', 'leaving', async () => { + await gate; + await git(root, 'worktree', 'remove', '.worktrees/leaving'); + return 'removed'; + }); + let settled = false; + const request = tools.execute(laneCommand('leaving')); + request.catch(() => undefined).finally(() => { settled = true; }); + let checkoutSettled = false; + const checkout = tools.execute({ protocolVersion: 1, operation: 'read_file', workspaceId: 'repo', path: 'README.md' }); + checkout.finally(() => { checkoutSettled = true; }); + const cancelled = new AbortController(); + const abandoned = tools.execute(laneCommand('leaving'), cancelled.signal); + await pause(20); + assert.equal(settled, false, 'the lane request waits for retirement'); + assert.equal(checkoutSettled, false, 'a checkout request waits too: it can reach the lane being removed'); + assert.deepEqual(await tools.whileIdle('repo', 'leaving', async () => 'again'), { ran: false }); + cancelled.abort(new Error('caller gave up')); + await assert.rejects(abandoned, /caller gave up/, 'cancellation ends the wait without the retirement'); + + release(); + assert.deepEqual(await retiring, { ran: true, value: 'removed' }); + await assert.rejects( + request, + (error: unknown) => error instanceof WorkspaceToolError && error.code === 'INVALID_REQUEST', + ); + const read = await checkout; + assert.equal(read.operation === 'read_file' && read.content.trimEnd(), 'root'); + assert.equal(executions, 0); +}); + +test('retirement never prunes other registered worktrees that are temporarily missing', async (t) => { + const root = await repository(t); + const external = join(root, '..', 'external'); + await git(root, 'worktree', 'add', '-q', '-b', 'external', external); + await rename(external, `${external}.unmounted`); + await worktree(root, 'done'); + await age(root, 'done'); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, ['repo:done']); + assert.ok(await exists(join(root, '.git', 'worktrees', 'external')), 'the unavailable worktree stays registered'); + assert.equal(await exists(join(root, '.git', 'worktrees', 'done')), false, 'removal deletes its own metadata'); + await rename(`${external}.unmounted`, external); + assert.equal(await git(external, 'branch', '--show-current'), 'external'); +}); + +test('a checkout request in flight defers retirement of its lanes', async (t) => { + const root = await repository(t); + const path = await worktree(root, 'quiet'); + await age(root, 'quiet'); + let release!: () => void; + let entered!: () => void; + const inside = new Promise((resolve) => { entered = resolve; }); + const gate = new Promise((resolve) => { release = resolve; }); + const delegate = await LocalWorkspaceTools.create({ + repositoryInstructions: false, + workspaces: [{ id: 'repo', root, writable: true }], + }); + const tools = new LinkedWorktreeWorkspaceTools({ + delegate: { + capabilities: delegate.capabilities, + async execute(request, signal) { + entered(); + await gate; + return await delegate.execute(request, signal); + }, + }, + sources: new Map([['repo', { root, repositoryInstructions: false, writable: true }]]), + }); + + const reading = tools.execute({ protocolVersion: 1, operation: 'read_file', workspaceId: 'repo', path: 'README.md' }); + await inside; + const during = await retireStaleWorktrees({ sources: sources(root), activity: tools }); + assert.deepEqual(during.kept, { active: 1 }); + release(); + await reading; + assert.ok(await exists(path)); + + const after = await retireStaleWorktrees({ sources: sources(root), activity: tools }); + assert.deepEqual(after.retired, ['repo:quiet']); +}); + +test('a checkout request that runs during inspection defers retirement', async (t) => { + const root = await repository(t); + await worktree(root, 'touched'); + await age(root, 'touched'); + const tools = await laneTools(root); + const before = tools.checkoutActivity('repo'); + await tools.execute({ protocolVersion: 1, operation: 'read_file', workspaceId: 'repo', path: 'README.md' }); + assert.equal(tools.checkoutActivity('repo'), before + 2, 'a checkout request counts at start and finish'); + + let reads = 0; + const duringInspection = { + lastUsed: (workspaceId: string, name: string) => tools.lastUsed(workspaceId, name), + // The second read, under the reservation, sees a checkout request that ran during inspection. + checkoutActivity: () => reads++, + whileIdle: (workspaceId: string, name: string, task: () => Promise) => tools.whileIdle(workspaceId, name, task), + }; + const deferred = await retireStaleWorktrees({ sources: sources(root), activity: duringInspection }); + assert.deepEqual(deferred.kept, { changed: 1 }); + assert.ok(await exists(join(root, '.worktrees', 'touched'))); + + const quiet = await retireStaleWorktrees({ sources: sources(root), activity: tools }); + assert.deepEqual(quiet.retired, ['repo:touched']); +}); + +test('quarantined lanes and checkouts are left for the operator', async (t) => { + const root = await repository(t); + await worktree(root, 'held'); + await worktree(root, 'free'); + await age(root, 'held'); + await age(root, 'free'); + + const checkoutHeld = await retireStaleWorktrees({ + sources: sources(root), + isQuarantined: async (_workspaceId, worktree) => worktree == null, + }); + assert.deepEqual(checkoutHeld, { retired: [], kept: {}, freedBytes: 0, truncated: false }); + + const laneHeld = await retireStaleWorktrees({ + sources: sources(root), + isQuarantined: async (_workspaceId, worktree) => worktree === 'held', + }); + assert.deepEqual(laneHeld.retired, ['repo:free']); + assert.deepEqual(laneHeld.kept, { quarantined: 1 }); + assert.ok(await exists(join(root, '.worktrees', 'held'))); +}); + +test('a quarantine that appears after inspection is caught under the lane reservation', async (t) => { + const root = await repository(t); + await worktree(root, 'late'); + await age(root, 'late'); + for (const quarantinedLater of [undefined, 'late']) { + const asked: string[] = []; + const summary = await retireStaleWorktrees({ + sources: sources(root), + activity: await laneTools(root), + isQuarantined: async (_workspaceId, worktree) => { + asked.push(worktree ?? ''); + // Clear for the scan and inspection; quarantined once the lane is reserved. + return asked.length > 2 && worktree === quarantinedLater; + }, + }); + assert.deepEqual(summary.kept, { quarantined: 1 }); + assert.deepEqual(asked, ['', 'late', '', ...(quarantinedLater ? ['late'] : [])]); + assert.ok(await exists(join(root, '.worktrees', 'late'))); + } +}); + +test('one worktree failing does not stop the pass', async (t) => { + const root = await repository(t); + await worktree(root, 'broken'); + await worktree(root, 'fine'); + await rm(join(root, '.git', 'worktrees', 'broken', 'HEAD')); + await age(root, 'broken'); + await age(root, 'fine'); + + const summary = await retireStaleWorktrees({ sources: sources(root) }); + + assert.deepEqual(summary.retired, ['repo:fine']); + assert.equal(Object.values(summary.kept).reduce((total, count) => total + (count ?? 0), 0), 1); + assert.ok(await exists(join(root, '.worktrees', 'broken'))); +}); + +test('worktrees kept for lasting reasons cannot starve the rest of a backlog', async (t) => { + const root = await repository(t); + for (const [index, name] of ['first', 'second', 'third'].entries()) { + await worktree(root, name, false); + await age(root, name, 40 - index); + } + await worktree(root, 'newest'); + await age(root, 'newest', 10); + const rotation = new Set(); + const options = { sources: sources(root), rotation, limits: { inspect: 2 } }; + + const first = await retireStaleWorktrees(options); + assert.deepEqual(first.retired, []); + assert.deepEqual(first.kept, { unpushed: 2, deferred: 2 }); + assert.equal(first.truncated, true, 'uninspected worktrees remain in this rotation'); + + const second = await retireStaleWorktrees(options); + assert.deepEqual(second.retired, ['repo:newest']); + assert.deepEqual(second.kept, { unpushed: 1, deferred: 2 }); + assert.equal(second.truncated, false, 'the rotation is complete'); + + const third = await retireStaleWorktrees(options); + assert.deepEqual(third.kept, { unpushed: 2, deferred: 1 }, 'a new rotation starts from the oldest'); + assert.equal(third.truncated, true); +}); + +test('retirement is on by default and can be disabled or retuned', () => { + assert.deepEqual(worktreeRetirementSettings({ optOut: false }), { enabled: true, idleMs: 7 * DAY_MS }); + assert.equal(worktreeRetirementSettings({ optOut: true }).enabled, false); + assert.equal(worktreeRetirementSettings({ optOut: false, enabled: 'FALSE' }).enabled, false); + assert.equal(worktreeRetirementSettings({ optOut: true, enabled: 'true' }).enabled, false); + assert.equal(worktreeRetirementSettings({ optOut: false, enabled: ' ' }).enabled, true); + assert.equal(worktreeRetirementSettings({ optOut: false, idleDays: '14' }).idleMs, 14 * DAY_MS); + assert.throws(() => worktreeRetirementSettings({ optOut: false, enabled: 'off' }), /must be true or false/); + for (const idleDays of ['0', '-1', '1.5', 'week', '3651']) { + assert.throws(() => worktreeRetirementSettings({ optOut: false, idleDays }), /IDLE_DAYS/); + } +}); + +test('scheduled passes never overlap and stop cleanly', async (t) => { + const root = await repository(t); + await worktree(root, 'scheduled'); + await age(root, 'scheduled'); + const messages: string[] = []; + const scheduler = new WorktreeRetirementScheduler({ + sources: sources(root), + startDelayMs: 60_000, + log: (level, message) => { + if (level === 'info') messages.push(message); + }, + }); + scheduler.start(); + + const first = scheduler.runNow(); + assert.equal(scheduler.runNow(), first, 'a pass requested while one runs joins it'); + const summary = await first; + assert.deepEqual(summary?.retired, ['repo:scheduled']); + assert.equal(messages.length, 1); + assert.match(messages[0]!, /^worktree retirement: retired 1, kept 0(, freed about .+)?$/); + + await scheduler.stop(); + assert.equal(await scheduler.runNow(), undefined, 'a stopped scheduler runs nothing'); +}); + +test('summaries count reasons and note a remaining backlog', () => { + assert.equal( + describeWorktreeRetirement({ + retired: ['repo:a', 'repo:b'], + kept: { unpushed: 2, dirty: 1, recent: 4 }, + freedBytes: 3 * 1024 ** 3, + truncated: true, + }), + 'worktree retirement: retired 2, kept 7 (dirty 1, recent 4, unpushed 2), freed about 3.0 GiB, more next pass', + ); +}); diff --git a/packages/code/src/worktree-retirement.ts b/packages/code/src/worktree-retirement.ts new file mode 100644 index 00000000..2ca8b78f --- /dev/null +++ b/packages/code/src/worktree-retirement.ts @@ -0,0 +1,714 @@ +import { execFile } from 'node:child_process'; +import { lstat, opendir, statfs } from 'node:fs/promises'; +import { join } from 'node:path'; +import { promisify } from 'node:util'; + +import { LINKED_WORKTREE_DIRECTORY, verifyLinkedWorktree } from './linked-worktrees.js'; +import { isValidLinkedWorktreeName } from './protocol.js'; + +import type { LinkedWorktreeActivity, VerifiedLinkedWorktree } from './linked-worktrees.js'; +import type { WorkspaceRootIdentity } from './root-identity.js'; + +const execFileAsync = promisify(execFile); + +const DAY_MS = 24 * 60 * 60 * 1000; +export const DEFAULT_WORKTREE_IDLE_DAYS = 7; +const MAX_WORKTREE_IDLE_DAYS = 3650; +export const WORKTREE_RETIREMENT_INTERVAL_MS = 6 * 60 * 60 * 1000; +const WORKTREE_RETIREMENT_START_DELAY_MS = 2 * 60 * 1000; +/** A pass that hit a bound continues soon instead of waiting a full interval. */ +const WORKTREE_RETIREMENT_CONTINUATION_MS = 5 * 60 * 1000; +/** + * Directory entries read beneath one checkout's `.worktrees` per pass: a + * memory bound far above any real checkout, so every pass sees every entry. + * Git inspection is what is bounded per pass, and rotation keeps it fair. + */ +const SCAN_LIMIT = 65_536; +/** Idle worktrees whose Git state is inspected per pass, oldest first. */ +const INSPECT_LIMIT = 128; +/** Worktrees removed per pass. */ +const RETIRE_LIMIT = 32; +const GIT_TIMEOUT_MS = 30_000; +/** + * Removal deletes ignored build output too, which can take a while. It has no + * timeout and ignores cancellation: a half-deleted worktree loses its `.git` + * file and can no longer be verified, retired or recognized. + */ +const GIT_REMOVE_NO_TIMEOUT = 0; +const GIT_OUTPUT_LIMIT = 64 * 1024; +/** Path and commit lists between a stale branch and its default branch can be long. */ +const GIT_LIST_LIMIT = 16 * 1024 * 1024; +/** Per-worktree Git state that marks a merge, rebase, cherry-pick, revert or bisect in progress. */ +const OPERATION_STATE = [ + 'MERGE_HEAD', + 'CHERRY_PICK_HEAD', + 'REVERT_HEAD', + 'BISECT_LOG', + 'BISECT_START', + 'rebase-merge', + 'rebase-apply', + 'sequencer', +]; +/** Per-worktree Git metadata that Git rewrites whenever the worktree is used. */ +const ACTIVITY_STATE = ['HEAD', 'index', join('logs', 'HEAD'), 'ORIG_HEAD', 'FETCH_HEAD']; + +export type WorktreeKeptReason = + | 'unverified' + | 'recent' + | 'quarantined' + | 'locked' + | 'operation' + | 'no-commit' + | 'dirty' + | 'unpushed' + | 'active' + | 'changed' + | 'failed' + | 'deferred'; + +export interface WorktreeRetirementSource { + workspaceId: string; + root: string; + identity?: WorkspaceRootIdentity; +} + +export interface WorktreeRetirementOptions { + sources: readonly WorktreeRetirementSource[]; + /** Lane use in this worker; without it, only on-disk activity counts. */ + activity?: LinkedWorktreeActivity; + idleMs?: number; + /** A quarantined checkout or lane awaits an operator and is never retired. */ + isQuarantined?: (workspaceId: string, worktree?: string) => Promise; + log?: (level: 'debug' | 'info', message: string) => void; + /** + * Idle worktrees already inspected in the current rotation. A pass inspects + * the others first and records what it inspects, so worktrees kept for a + * lasting reason cannot starve the rest of a backlog larger than one pass. + */ + rotation?: Set; + /** Per-pass bounds; the defaults suit production. */ + limits?: { inspect?: number; retire?: number }; +} + +export interface WorktreeRetirementSummary { + /** `:` for each retired worktree. */ + retired: string[]; + kept: Partial>; + /** Free-space gain on the checkouts' filesystems across the pass; other writers make it approximate. */ + freedBytes: number; + /** Idle worktrees remain that this rotation has not inspected yet. */ + truncated: boolean; +} + +export interface WorktreeRetirementSettings { + enabled: boolean; + idleMs: number; +} + +interface Candidate { + source: WorktreeRetirementSource; + name: string; + lane: VerifiedLinkedWorktree; + metadata: string; + /** Newest on-disk activity, re-read under the reservation to detect use during inspection. */ + diskActiveAt: number; + /** This worker's own last use of the lane, re-read under the reservation. */ + usedAt?: number; + /** Checkout request activity when inspection began, re-read under the reservation. */ + checkoutAt?: number; + /** Newest of on-disk activity and this worker's own lane use. */ + activeAt: number; +} + +type Kept = { kept: WorktreeKeptReason; detail?: string }; +type Verdict = Kept | { head: string; basis: string }; + +/** + * Resolve the operator's choice. Retirement is on unless explicitly disabled; + * malformed values fail startup rather than silently choosing either way. + */ +export function worktreeRetirementSettings(input: { + optOut: boolean; + enabled?: string; + idleDays?: string; +}): WorktreeRetirementSettings { + const enabled = input.enabled?.trim().toLowerCase() ?? ''; + if (enabled !== '' && enabled !== 'true' && enabled !== 'false') { + throw new Error('LIBRECHAT_CODE_WORKTREE_RETIREMENT must be true or false'); + } + const idleDays = input.idleDays?.trim() ?? ''; + const days = idleDays === '' ? DEFAULT_WORKTREE_IDLE_DAYS : Number(idleDays); + if (!Number.isSafeInteger(days) || days < 1 || days > MAX_WORKTREE_IDLE_DAYS) { + throw new Error( + `LIBRECHAT_CODE_WORKTREE_IDLE_DAYS must be an integer between 1 and ${MAX_WORKTREE_IDLE_DAYS}`, + ); + } + return { enabled: !input.optOut && enabled !== 'false', idleMs: days * DAY_MS }; +} + +function gitEnvironment(): NodeJS.ProcessEnv { + return { + PATH: process.env.PATH, + SYSTEMROOT: process.env.SYSTEMROOT, + GIT_CONFIG_NOSYSTEM: '1', + GIT_CONFIG_GLOBAL: '/dev/null', + GIT_TERMINAL_PROMPT: '0', + GIT_OPTIONAL_LOCKS: '0', + LC_ALL: 'C', + }; +} + +export async function git( + cwd: string, + args: string[], + signal?: AbortSignal, + timeout = GIT_TIMEOUT_MS, + maxBuffer = GIT_OUTPUT_LIMIT, +): Promise { + const { stdout } = await execFileAsync( + 'git', + ['--no-optional-locks', '-C', cwd, '-c', 'core.fsmonitor=false', ...args], + { encoding: 'utf8', env: gitEnvironment(), maxBuffer, signal, timeout }, + ); + return stdout; +} + +/** Same invocation as `git`, but the raw stdout bytes, for callers that must not accept lossy decoding. */ +export async function gitBytes( + cwd: string, + args: string[], + signal?: AbortSignal, + timeout = GIT_TIMEOUT_MS, + maxBuffer = GIT_OUTPUT_LIMIT, + extraEnvironment: Record = {}, +): Promise { + const { stdout } = await execFileAsync( + 'git', + ['--no-optional-locks', '-C', cwd, '-c', 'core.fsmonitor=false', ...args], + { encoding: 'buffer', env: { ...gitEnvironment(), ...extraEnvironment }, maxBuffer, signal, timeout }, + ); + return stdout; +} + +/** The commit a ref names, or undefined when it does not resolve. */ +async function resolveCommit(cwd: string, ref: string, signal?: AbortSignal): Promise { + try { + const commit = (await git(cwd, ['rev-parse', '--verify', '--quiet', `${ref}^{commit}`], signal)).trim(); + return /^[0-9a-f]{40,64}$/.test(commit) ? commit : undefined; + } catch { + signal?.throwIfAborted(); + return undefined; + } +} + +function pathList(output: string): string[] { + return output.split('\0').filter(Boolean); +} + +/** The remote's default branch as last fetched: its `HEAD` symref, else `main` or `master`. */ +async function remoteDefaultBranch( + checkout: string, + remote: string, + signal?: AbortSignal, +): Promise<{ ref: string; commit: string } | undefined> { + let symbolic: string | undefined; + try { + symbolic = (await git(checkout, ['symbolic-ref', '-q', `refs/remotes/${remote}/HEAD`], signal)).trim(); + } catch { + signal?.throwIfAborted(); + } + const refs = [ + ...(symbolic?.startsWith(`refs/remotes/${remote}/`) ? [symbolic] : []), + `refs/remotes/${remote}/main`, + `refs/remotes/${remote}/master`, + ]; + for (const ref of refs) { + const commit = await resolveCommit(checkout, ref, signal); + if (commit) return { ref: ref.replace(/^refs\/remotes\//, ''), commit }; + } + return undefined; +} + +/** + * Whether a HEAD that no remote-tracking ref contains is nonetheless already + * in its remote's default branch by content, as after a squash or rebase merge + * whose remote branch was then deleted. Only a HEAD without a live upstream + * qualifies: detached, never pushed, or whose upstream ref is gone. A live + * upstream that lacks HEAD means unpushed commits. Content counts when every + * commit beyond the default branch is patch-equivalent to one in it (and none + * is a merge, which could carry its own changes), or when the default branch + * has identical content at every path the branch changed since their merge + * base. Local refs only; nothing is fetched. + */ +async function mergedByContent( + lane: VerifiedLinkedWorktree, + head: string, + signal?: AbortSignal, +): Promise { + const checkout = lane.checkoutRoot; + let branch: string | undefined; + try { + branch = (await git(lane.root, ['symbolic-ref', '-q', 'HEAD'], signal)).trim() || undefined; + } catch { + signal?.throwIfAborted(); + } + let remote = 'origin'; + if (branch?.startsWith('refs/heads/')) { + const [upstream = '', upstreamRemote = ''] = ( + await git(checkout, ['for-each-ref', '--format=%(upstream)%00%(upstream:remotename)', branch], signal) + ) + .trim() + .split('\0'); + if (upstream) { + if (await resolveCommit(checkout, upstream, signal)) return undefined; + if (/^[A-Za-z0-9._-]+$/.test(upstreamRemote)) remote = upstreamRemote; + } + } + const target = await remoteDefaultBranch(checkout, remote, signal); + if (!target) return undefined; + const merges = (await git(checkout, ['rev-list', '--count', '--merges', `${target.commit}..${head}`], signal)).trim(); + if (merges === '0') { + const cherry = (await git(checkout, ['cherry', target.commit, head], signal, GIT_TIMEOUT_MS, GIT_LIST_LIMIT)) + .split('\n') + .filter(Boolean); + if (cherry.every((line) => line.startsWith('- '))) return `patch-equivalent to ${target.ref}`; + } + let base: string; + try { + base = (await git(checkout, ['merge-base', target.commit, head], signal)).trim(); + } catch { + signal?.throwIfAborted(); + return undefined; + } + const diff = (from: string): Promise => + git( + checkout, + ['diff', '--name-only', '-z', '--no-renames', '--no-ext-diff', '--no-textconv', from, head], + signal, + GIT_TIMEOUT_MS, + GIT_LIST_LIMIT, + ); + const touched = pathList(await diff(base)); + const differing = new Set(pathList(await diff(target.commit))); + return touched.every((path) => !differing.has(path)) ? `same content as ${target.ref}` : undefined; +} + +function errorDetail(error: unknown): string { + const stderr = (error as { stderr?: unknown }).stderr; + const text = typeof stderr === 'string' && stderr.trim() ? stderr : error instanceof Error ? error.message : String(error); + return text.trim().split('\n')[0]!.slice(0, 200); +} + +/** Absent is the only answer that clears a check; anything unreadable counts as present. */ +async function present(path: string): Promise { + try { + await lstat(path); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code !== 'ENOENT'; + } +} + +async function modifiedAt(path: string): Promise { + try { + return (await lstat(path)).mtimeMs; + } catch { + return 0; + } +} + +/** The newest on-disk sign of use: the worktree directory itself and its Git metadata. */ +async function lastActivity(root: string, metadata: string): Promise { + const times = await Promise.all( + [root, metadata, ...ACTIVITY_STATE.map((path) => join(metadata, path))].map(modifiedAt), + ); + return Math.max(...times); +} + +async function worktreeNames(checkout: string): Promise<{ names: string[]; truncated: boolean }> { + let directory; + try { + directory = await opendir(join(checkout, LINKED_WORKTREE_DIRECTORY)); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') return { names: [], truncated: false }; + throw error; + } + const names: string[] = []; + try { + for await (const entry of directory) { + if (names.length >= SCAN_LIMIT) return { names, truncated: true }; + if (isValidLinkedWorktreeName(entry.name)) names.push(entry.name); + } + } finally { + await directory.close().catch(() => undefined); + } + return { names, truncated: false }; +} + +async function freeSpace(paths: readonly string[]): Promise> { + const space = new Map(); + for (const path of paths) { + try { + const device = String((await lstat(path)).dev); + if (space.has(device)) continue; + const status = await statfs(path, { bigint: true }); + space.set(device, Number(status.bavail * status.bsize)); + } catch { + // Space reporting is informational; it never decides a retirement. + } + } + return space; +} + +function describeChanges(status: string): string { + const entries = status.split('\0').filter(Boolean); + const untracked = entries.filter((entry) => entry.startsWith('?? ')).length; + return untracked === entries.length + ? 'untracked files' + : untracked > 0 + ? 'tracked changes and untracked files' + : 'tracked changes'; +} + +function formatBytes(bytes: number): string { + const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB']; + let value = bytes; + let unit = 0; + while (value >= 1024 && unit < units.length - 1) { + value /= 1024; + unit += 1; + } + return `${unit === 0 ? value : value.toFixed(1)} ${units[unit]}`; +} + +export function describeWorktreeRetirement(summary: WorktreeRetirementSummary): string { + const kept = Object.entries(summary.kept) + .sort(([a], [b]) => a.localeCompare(b)) + .map(([reason, count]) => `${reason} ${count}`); + const keptTotal = Object.values(summary.kept).reduce((total, count) => total + (count ?? 0), 0); + return [ + `worktree retirement: retired ${summary.retired.length}`, + `kept ${keptTotal}${kept.length ? ` (${kept.join(', ')})` : ''}`, + ...(summary.freedBytes > 0 ? [`freed about ${formatBytes(summary.freedBytes)}`] : []), + ...(summary.truncated ? ['more next pass'] : []), + ].join(', '); +} + +/** Everything short of the worker's own activity that could make removal lose work. */ +async function inspect( + candidate: Candidate, + options: WorktreeRetirementOptions, + signal?: AbortSignal, +): Promise { + const { lane, metadata, name, source } = candidate; + try { + if (await options.isQuarantined?.(source.workspaceId, name)) return { kept: 'quarantined' }; + } catch { + return { kept: 'quarantined', detail: 'quarantine state is unreadable' }; + } + if (await present(join(metadata, 'locked'))) return { kept: 'locked' }; + for (const state of OPERATION_STATE) { + if (await present(join(metadata, state))) return { kept: 'operation', detail: state }; + } + let head: string; + try { + head = (await git(lane.root, ['rev-parse', '--verify', '--quiet', 'HEAD^{commit}'], signal)).trim(); + } catch { + signal?.throwIfAborted(); + return { kept: 'no-commit' }; + } + if (!/^[0-9a-f]{40,64}$/.test(head)) return { kept: 'no-commit' }; + let status: string; + try { + status = await git( + lane.root, + ['status', '--porcelain', '-z', '--untracked-files=normal', '--ignore-submodules=none'], + signal, + ); + } catch (error) { + signal?.throwIfAborted(); + if ((error as NodeJS.ErrnoException).code === 'ERR_CHILD_PROCESS_STDIO_MAXBUFFER') { + return { kept: 'dirty', detail: 'too many changes to list' }; + } + return { kept: 'failed', detail: `status: ${errorDetail(error)}` }; + } + if (status.length > 0) return { kept: 'dirty', detail: describeChanges(status) }; + let remote: string; + try { + remote = ( + await git( + lane.checkoutRoot, + ['for-each-ref', '--count=1', '--contains', head, '--format=%(refname)', 'refs/remotes/'], + signal, + ) + ).trim(); + } catch (error) { + signal?.throwIfAborted(); + return { kept: 'failed', detail: `remote containment: ${errorDetail(error)}` }; + } + if (remote) return { head, basis: `contained in ${remote.replace(/^refs\/remotes\//, '')}` }; + let merged: string | undefined; + try { + merged = await mergedByContent(lane, head, signal); + } catch (error) { + signal?.throwIfAborted(); + return { kept: 'failed', detail: `merged content: ${errorDetail(error)}` }; + } + return merged + ? { head, basis: merged } + : { kept: 'unpushed', detail: 'not on a remote-tracking ref or merged into the default branch' }; +} + +/** Re-checked under the lane reservation: anything that moved since inspection defers to the next pass. */ +async function remove( + candidate: Candidate, + head: string, + options: WorktreeRetirementOptions, + signal?: AbortSignal, +): Promise { + const { lane, metadata, name, source } = candidate; + if (options.activity?.lastUsed(source.workspaceId, name) !== candidate.usedAt) { + return { kept: 'changed', detail: 'used during inspection' }; + } + // A checkout request can reach `.worktrees/*`, including ignored files that `git status` never reports. + if (options.activity?.checkoutActivity(source.workspaceId) !== candidate.checkoutAt) { + return { kept: 'changed', detail: 'checkout used during inspection' }; + } + // A request that finished between inspection and this reservation may have quarantined either. + try { + if ( + (await options.isQuarantined?.(source.workspaceId)) || + (await options.isQuarantined?.(source.workspaceId, name)) + ) { + return { kept: 'quarantined' }; + } + } catch { + return { kept: 'quarantined', detail: 'quarantine state is unreadable' }; + } + let current: VerifiedLinkedWorktree; + try { + current = await verifyLinkedWorktree(source.root, name, source.identity); + } catch { + return { kept: 'changed', detail: 'no longer a verified linked worktree' }; + } + if (current.identity.dev !== lane.identity.dev || current.identity.ino !== lane.identity.ino) { + return { kept: 'changed', detail: 'worktree directory was replaced' }; + } + if ((await lastActivity(lane.root, metadata)) !== candidate.diskActiveAt) { + return { kept: 'changed', detail: 'used during inspection' }; + } + if (await present(join(metadata, 'locked'))) return { kept: 'locked' }; + try { + const now = (await git(lane.root, ['rev-parse', '--verify', '--quiet', 'HEAD^{commit}'], signal)).trim(); + if (now !== head) return { kept: 'changed', detail: 'HEAD moved during inspection' }; + } catch { + signal?.throwIfAborted(); + return { kept: 'changed', detail: 'HEAD is unreadable' }; + } + signal?.throwIfAborted(); + try { + // Never --force: Git re-checks for changes and untracked files itself and refuses a dirty or locked worktree. + await git(lane.checkoutRoot, ['worktree', 'remove', lane.root], undefined, GIT_REMOVE_NO_TIMEOUT); + } catch (error) { + return { kept: 'failed', detail: `remove: ${errorDetail(error)}` }; + } + return 'retired'; +} + +/** + * Retire linked worktrees beneath each checkout's `.worktrees` that are + * verified, idle, clean, free of in-progress operations and locks, and whose + * HEAD is contained in a remote-tracking ref or already merged into the + * remote's default branch by content. Removal is `git worktree remove` + * without `--force`, which also deletes that worktree's own metadata; no + * repository-wide `git worktree prune` runs, since it would also expire other + * registered worktrees that are only temporarily unavailable. Branches are + * kept, so `git worktree add` restores any of them. Each worktree is judged + * independently; one failure never ends the pass. + */ +export async function retireStaleWorktrees( + options: WorktreeRetirementOptions, + signal?: AbortSignal, +): Promise { + const now = Date.now(); + const idleMs = options.idleMs ?? DEFAULT_WORKTREE_IDLE_DAYS * DAY_MS; + const summary: WorktreeRetirementSummary = { retired: [], kept: {}, freedBytes: 0, truncated: false }; + const keep = (source: WorktreeRetirementSource, name: string, reason: WorktreeKeptReason, detail?: string): void => { + summary.kept[reason] = (summary.kept[reason] ?? 0) + 1; + options.log?.('debug', `worktree retirement kept ${source.workspaceId}:${name}: ${reason}${detail ? ` (${detail})` : ''}`); + }; + const candidates: Candidate[] = []; + for (const source of options.sources) { + signal?.throwIfAborted(); + try { + if (await options.isQuarantined?.(source.workspaceId)) { + options.log?.('debug', `worktree retirement skipped ${source.workspaceId}: checkout is quarantined`); + continue; + } + const { names, truncated } = await worktreeNames(source.root); + if (truncated) { + options.log?.('debug', `worktree retirement read only the first ${SCAN_LIMIT} entries of ${source.workspaceId}`); + } + for (const name of names) { + signal?.throwIfAborted(); + let lane: VerifiedLinkedWorktree; + try { + lane = await verifyLinkedWorktree(source.root, name, source.identity); + } catch (error) { + keep(source, name, 'unverified', errorDetail(error)); + continue; + } + const metadata = join(lane.commonGitDir, 'worktrees', name); + const diskActiveAt = await lastActivity(lane.root, metadata); + const usedAt = options.activity?.lastUsed(source.workspaceId, name); + const activeAt = Math.max(diskActiveAt, usedAt ?? 0); + if (now - activeAt < idleMs) { + keep(source, name, 'recent'); + continue; + } + candidates.push({ source, name, lane, metadata, diskActiveAt, usedAt, activeAt }); + } + } catch (error) { + signal?.throwIfAborted(); + options.log?.('debug', `worktree retirement skipped ${source.workspaceId}: ${errorDetail(error)}`); + } + } + if (candidates.length === 0) { + options.rotation?.clear(); + return summary; + } + const { rotation } = options; + const key = (candidate: Candidate): string => `${candidate.source.workspaceId}\0${candidate.name}`; + if (rotation) { + const current = new Set(candidates.map(key)); + for (const entry of rotation) { + if (!current.has(entry)) rotation.delete(entry); + } + if (rotation.size === current.size) rotation.clear(); + } + const visited = (candidate: Candidate): number => (rotation?.has(key(candidate)) ? 1 : 0); + candidates.sort((a, b) => visited(a) - visited(b) || a.activeAt - b.activeAt); + const checkouts = [...new Set(candidates.map((candidate) => candidate.lane.checkoutRoot))]; + const before = await freeSpace(checkouts); + const retiredCheckouts = new Set(); + let inspected = 0; + for (const candidate of candidates) { + signal?.throwIfAborted(); + const { source, name } = candidate; + if ( + inspected >= (options.limits?.inspect ?? INSPECT_LIMIT) || + summary.retired.length >= (options.limits?.retire ?? RETIRE_LIMIT) + ) { + summary.truncated ||= visited(candidate) === 0; + keep(source, name, 'deferred'); + continue; + } + inspected += 1; + rotation?.add(key(candidate)); + try { + candidate.checkoutAt = options.activity?.checkoutActivity(source.workspaceId); + const verdict = await inspect(candidate, options, signal); + if ('kept' in verdict) { + keep(source, name, verdict.kept, verdict.detail); + continue; + } + const retire = (): Promise => remove(candidate, verdict.head, options, signal); + const reserved = options.activity + ? await options.activity.whileIdle(source.workspaceId, name, retire) + : { ran: true as const, value: await retire() }; + if (!reserved.ran) { + keep(source, name, 'active'); + continue; + } + if (reserved.value !== 'retired') { + keep(source, name, reserved.value.kept, reserved.value.detail); + continue; + } + summary.retired.push(`${source.workspaceId}:${name}`); + retiredCheckouts.add(candidate.lane.checkoutRoot); + options.log?.('debug', `worktree retirement retired ${source.workspaceId}:${name} (${verdict.basis}); its branch is kept`); + } catch (error) { + signal?.throwIfAborted(); + keep(source, name, 'failed', errorDetail(error)); + } + } + if (retiredCheckouts.size > 0) { + const after = await freeSpace(checkouts); + for (const [device, free] of after) { + summary.freedBytes += Math.max(0, free - (before.get(device) ?? free)); + } + } + return summary; +} + +export interface WorktreeRetirementSchedulerOptions extends WorktreeRetirementOptions { + intervalMs?: number; + startDelayMs?: number; + continuationMs?: number; +} + +/** + * Runs retirement passes in the background: once shortly after start, then + * every interval, sooner while a backlog remains. Passes never overlap; a pass + * requested while one runs joins it. + */ +export class WorktreeRetirementScheduler { + private readonly controller = new AbortController(); + private readonly rotation = new Set(); + private running?: Promise; + private timer?: NodeJS.Timeout; + private started = false; + + constructor(private readonly options: WorktreeRetirementSchedulerOptions) {} + + start(): void { + if (this.started || this.controller.signal.aborted) return; + this.started = true; + if (this.running == null) this.schedule(this.options.startDelayMs ?? WORKTREE_RETIREMENT_START_DELAY_MS); + } + + /** Run a pass now, or join the one already running. */ + runNow(): Promise { + if (this.controller.signal.aborted) return Promise.resolve(undefined); + this.running ??= this.pass().finally(() => { + this.running = undefined; + }); + return this.running; + } + + /** Stop scheduling; a removal already under way finishes first. */ + async stop(): Promise { + this.controller.abort(); + clearTimeout(this.timer); + await this.running; + } + + private schedule(delayMs: number): void { + clearTimeout(this.timer); + if (!this.started || this.controller.signal.aborted) return; + this.timer = setTimeout(() => void this.runNow(), delayMs); + this.timer.unref(); + } + + private async pass(): Promise { + clearTimeout(this.timer); + let summary: WorktreeRetirementSummary | undefined; + try { + summary = await retireStaleWorktrees( + { ...this.options, rotation: this.rotation }, + this.controller.signal, + ); + this.options.log?.('info', describeWorktreeRetirement(summary)); + } catch (error) { + if (!this.controller.signal.aborted) { + this.options.log?.('info', `worktree retirement pass failed: ${errorDetail(error)}`); + } + } + this.schedule( + summary?.truncated + ? (this.options.continuationMs ?? WORKTREE_RETIREMENT_CONTINUATION_MS) + : (this.options.intervalMs ?? WORKTREE_RETIREMENT_INTERVAL_MS), + ); + return summary; + } +} diff --git a/scripts/openshell/probe-policy.yaml b/scripts/openshell/probe-policy.yaml new file mode 100644 index 00000000..bef0a892 --- /dev/null +++ b/scripts/openshell/probe-policy.yaml @@ -0,0 +1,11 @@ +version: 1 +filesystem_policy: + include_workdir: false + read_only: [/bin, /usr, /lib, /lib64, /etc] + read_write: [/tmp, /dev/null] +landlock: + compatibility: hard_requirement +process: + run_as_user: '1000' + run_as_group: '1000' +network_policies: {} diff --git a/scripts/openshell/probe.mjs b/scripts/openshell/probe.mjs new file mode 100644 index 00000000..116beb56 --- /dev/null +++ b/scripts/openshell/probe.mjs @@ -0,0 +1,217 @@ +import { execFile } from 'node:child_process'; +import { randomUUID } from 'node:crypto'; +import { parseArgs } from 'node:util'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +export const OPENSHELL_VERSION = '0.1.2'; +export const OPENSHELL_COMMIT = '6648bd0c290efbc41ba131ee9831ee45cd431f94'; +const POLICY = fileURLToPath(new URL('./probe-policy.yaml', import.meta.url)); +const MARKER = 'codeapi-openshell-probe-ok'; + +export class ProbeError extends Error { + constructor(stage) { + super(`OpenShell probe failed at ${stage}`); + this.stage = stage; + } +} + +export function commandRunner(binary = 'openshell') { + const env = { ...process.env, NO_COLOR: '1' }; + // Named gateway selection must not inherit an endpoint or insecure override. + delete env.OPENSHELL_GATEWAY_ENDPOINT; + delete env.OPENSHELL_GATEWAY_INSECURE; + return (args, { stage, timeoutMs, signal }) => + new Promise((resolve, reject) => { + const child = execFile( + binary, + args, + { + env, + timeout: timeoutMs, + killSignal: 'SIGKILL', + maxBuffer: 65536, + signal, + }, + (error, stdout) => { + // CLI errors can contain credentials, policy contents, or provider details. + if (error) reject(new ProbeError(stage)); + else resolve(stdout); + } + ); + child.stdin.end(); + }); +} + +export async function runProbe(options, run = commandRunner()) { + const { + gateway, + image, + workspace = 'default', + timeoutMs = 120000, + signal, + } = options; + if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/.test(gateway ?? '')) { + throw new ProbeError('gateway validation'); + } + if (!/^[a-zA-Z0-9][a-zA-Z0-9_.-]{0,127}$/.test(workspace)) { + throw new ProbeError('workspace validation'); + } + if ( + !/^\S+@sha256:[a-f0-9]{64}$/.test(image ?? '') || + image.startsWith('-') + ) { + throw new ProbeError('image validation'); + } + if ( + !Number.isSafeInteger(timeoutMs) || + timeoutMs < 100 || + timeoutMs > 300000 + ) { + throw new ProbeError('timeout validation'); + } + const invoke = (args, stage, useSignal = true) => + run(args, { stage, timeoutMs, signal: useSignal ? signal : undefined }); + const version = await invoke(['--version'], 'version'); + if (version.trim() !== `openshell ${OPENSHELL_VERSION}`) { + throw new ProbeError('version mismatch'); + } + if (signal?.aborted) throw new ProbeError('cancelled'); + + const name = `codeapi-probe-${randomUUID()}`; + const scope = ['--gateway', gateway, '--workspace', workspace]; + const timings = {}; + const timed = async (stage, args, useSignal = true) => { + const start = performance.now(); + try { + return await invoke([...scope, ...args], stage, useSignal); + } finally { + timings[stage] = Math.round(performance.now() - start); + } + }; + let failure; + let cleanup = 'unconfirmed'; + try { + await timed('create', [ + 'sandbox', + 'create', + '--name', + name, + '--from', + image, + '--policy', + POLICY, + '--no-auto-providers', + '--approval-mode', + 'manual', + '--cpu', + '1', + '--memory', + '256Mi', + '--keep', + '--detach', + '--no-tty', + '--output', + 'json', + '--', + '/bin/sh', + '-c', + 'sleep 300', + ]); + const output = await timed('exec', [ + 'sandbox', + 'exec', + '--name', + name, + '--timeout', + '10', + '--no-tty', + '--no-login-shell', + '--', + '/bin/sh', + '-c', + `printf '%s' '${MARKER}' > /tmp/codeapi-probe && cat /tmp/codeapi-probe`, + ]); + if (output.trim() !== MARKER) throw new ProbeError('exec output'); + } catch (error) { + failure = error instanceof ProbeError ? error.stage : 'execution'; + } finally { + try { + const output = await timed( + 'delete', + ['sandbox', 'delete', name], + false + ); + const text = output.replace(/\x1b\[[0-9;]*m/g, '').trim(); + // Exit zero also covers accepted/pending deletion. Only terminal outcomes pass. + if ( + text === `✓ Deleted sandbox ${name}` || + text === `✓ Sandbox ${name} already deleted` + ) { + cleanup = 'confirmed'; + } + } catch { + // Keep the execution failure and report cleanup independently. + } + } + if (signal?.aborted && failure === undefined) failure = 'cancelled'; + return { + ok: failure === undefined && cleanup === 'confirmed', + version: OPENSHELL_VERSION, + upstreamCommit: OPENSHELL_COMMIT, + sandbox: name, + workspace, + cleanup, + ...(failure ? { failedStage: failure } : {}), + timingsMs: timings, + }; +} + +async function main() { + const { values } = parseArgs({ + options: { + gateway: { type: 'string' }, + workspace: { type: 'string', default: 'default' }, + image: { type: 'string' }, + 'timeout-ms': { type: 'string', default: '120000' }, + help: { type: 'boolean' }, + }, + }); + if (values.help) { + console.log( + 'node scripts/openshell/probe.mjs --gateway NAME --image REGISTRY/IMAGE@sha256:DIGEST [--workspace NAME] [--timeout-ms 120000]' + ); + return; + } + const controller = new AbortController(); + const abort = () => controller.abort(); + process.on('SIGINT', abort); + process.on('SIGTERM', abort); + try { + const result = await runProbe({ + gateway: values.gateway, + workspace: values.workspace, + image: values.image, + timeoutMs: Number(values['timeout-ms']), + signal: controller.signal, + }); + console.log(JSON.stringify(result, null, 2)); + process.exitCode = result.ok ? 0 : 1; + } finally { + process.off('SIGINT', abort); + process.off('SIGTERM', abort); + } +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + main().catch(error => { + console.error( + error instanceof ProbeError + ? error.message + : 'Invalid OpenShell probe invocation' + ); + process.exitCode = 1; + }); +} diff --git a/scripts/openshell/probe.test.mjs b/scripts/openshell/probe.test.mjs new file mode 100644 index 00000000..2ec95997 --- /dev/null +++ b/scripts/openshell/probe.test.mjs @@ -0,0 +1,273 @@ +import assert from 'node:assert/strict'; +import { test } from 'node:test'; +import { + commandRunner, + OPENSHELL_VERSION, + ProbeError, + runProbe, +} from './probe.mjs'; + +const options = { + gateway: 'evaluation', + workspace: 'codeapi-test', + image: `registry.test/shell@sha256:${'a'.repeat(64)}`, +}; +function fixture(overrides = {}) { + const calls = []; + const run = async (args, call) => { + calls.push({ args, ...call }); + if (overrides[call.stage]) return overrides[call.stage](args, call); + if (call.stage === 'version') return `openshell ${OPENSHELL_VERSION}\n`; + if (call.stage === 'exec') return 'codeapi-openshell-probe-ok'; + if (call.stage === 'delete') + return `✓ Deleted sandbox ${args.at(-1)}\n`; + return '{}'; + }; + return { run, calls }; +} + +test('scopes every operation and creates a bounded, credential-free sandbox', async () => { + const { run, calls } = fixture(); + const result = await runProbe(options, run); + assert.equal(result.ok, true); + assert.equal(result.cleanup, 'confirmed'); + assert.match(result.sandbox, /^codeapi-probe-[a-f0-9-]{36}$/); + assert.deepEqual( + calls.map(call => call.stage), + ['version', 'create', 'exec', 'delete'] + ); + for (const call of calls.slice(1)) { + assert.deepEqual(call.args.slice(0, 4), [ + '--gateway', + 'evaluation', + '--workspace', + 'codeapi-test', + ]); + assert.equal(call.timeoutMs, 120000); + assert.ok(call.args.includes(result.sandbox)); + } + const create = calls[1].args; + for (const flag of [ + '--policy', + '--no-auto-providers', + '--keep', + '--detach', + '--no-tty', + ]) + assert.ok(create.includes(flag)); + assert.equal(create.includes('--provider'), false); + assert.equal(create.includes('--env'), false); + assert.equal(create[create.indexOf('--from') + 1], options.image); + assert.equal(create[create.indexOf('--cpu') + 1], '1'); + assert.equal(create[create.indexOf('--memory') + 1], '256Mi'); + assert.equal(create[create.indexOf('--approval-mode') + 1], 'manual'); +}); + +test('rejects mutable images, ambiguous scope, and invalid timeouts before invoking the CLI', async () => { + for (const invalid of [ + { image: 'shell:latest' }, + { image: `-shell@sha256:${'a'.repeat(64)}` }, + { gateway: undefined }, + { workspace: '../default' }, + { timeoutMs: 0 }, + { timeoutMs: 300001 }, + ]) { + const { run, calls } = fixture(); + await assert.rejects( + runProbe({ ...options, ...invalid }, run), + ProbeError + ); + assert.equal(calls.length, 0); + } +}); + +test('rejects an unpinned CLI without creating or deleting a sandbox', async () => { + const { run, calls } = fixture({ version: () => 'openshell 0.1.3' }); + await assert.rejects(runProbe(options, run), /version mismatch/); + assert.equal(calls.length, 1); +}); + +for (const stage of ['create', 'exec']) { + test(`attempts cleanup after ${stage} failure without exposing upstream details`, async () => { + const { run, calls } = fixture({ + [stage]: () => { + throw new Error('secret provider credential'); + }, + }); + const result = await runProbe(options, run); + assert.equal(result.ok, false); + assert.equal(result.failedStage, 'execution'); + assert.equal(result.cleanup, 'confirmed'); + assert.equal(calls.at(-1).stage, 'delete'); + assert.equal(JSON.stringify(result).includes('secret'), false); + }); +} + +test('does not accept unexpected command output as success', async () => { + const { run } = fixture({ exec: () => 'unexpected' }); + const result = await runProbe(options, run); + assert.equal(result.ok, false); + assert.equal(result.failedStage, 'exec output'); + assert.equal(result.cleanup, 'confirmed'); +}); + +for (const output of [ + 'deletion accepted; cleanup is pending', + 'unsupported outcome', + '', + '✓ Deleted sandbox someone-else', +]) { + test(`does not report terminal cleanup for ${JSON.stringify( + output + )}`, async () => { + const { run } = fixture({ delete: () => output }); + const result = await runProbe(options, run); + assert.equal(result.ok, false); + assert.equal(result.cleanup, 'unconfirmed'); + }); +} + +test('accepts an already-absent sandbox as terminal cleanup', async () => { + const { run } = fixture({ + delete: args => `✓ Sandbox ${args.at(-1)} already deleted\n`, + }); + assert.equal((await runProbe(options, run)).cleanup, 'confirmed'); +}); + +test('preserves execution failure when cleanup also fails', async () => { + const { run } = fixture({ + exec: () => { + throw new ProbeError('exec'); + }, + delete: () => { + throw new Error('secret'); + }, + }); + const result = await runProbe(options, run); + assert.equal(result.failedStage, 'exec'); + assert.equal(result.cleanup, 'unconfirmed'); + assert.equal(result.ok, false); +}); + +test('cancellation before create does not launch work', async () => { + const controller = new AbortController(); + controller.abort(); + const { run, calls } = fixture(); + await assert.rejects( + runProbe({ ...options, signal: controller.signal }, run), + /cancelled/ + ); + assert.equal(calls.length, 1); +}); + +test('cancellation during create still uses an independent cleanup attempt', async () => { + const controller = new AbortController(); + const { run, calls } = fixture({ + create: () => { + controller.abort(); + throw new ProbeError('create'); + }, + }); + const result = await runProbe( + { ...options, signal: controller.signal }, + run + ); + assert.equal(result.ok, false); + assert.equal(calls[1].signal, controller.signal); + assert.equal(calls.at(-1).signal, undefined); + assert.equal(calls.at(-1).stage, 'delete'); +}); + +test('command runner redacts stdout and stderr on failure', async () => { + await assert.rejects( + commandRunner(process.execPath)( + [ + '-e', + 'console.log("secret"); console.error("private-key"); process.exit(2)', + ], + { stage: 'exec', timeoutMs: 1000 } + ), + error => { + assert.equal(error.message, 'OpenShell probe failed at exec'); + assert.equal(error.cause, undefined); + assert.equal(error.stdout, undefined); + assert.equal(error.stderr, undefined); + return true; + } + ); +}); + +test('command runner kills a stalled CLI within its timeout', async () => { + await assert.rejects( + commandRunner(process.execPath)(['-e', 'setInterval(() => {}, 1000)'], { + stage: 'create', + timeoutMs: 100, + }), + /failed at create/ + ); +}); + +test('command runner removes inherited endpoint and insecure overrides', async () => { + const savedEndpoint = process.env.OPENSHELL_GATEWAY_ENDPOINT; + const savedInsecure = process.env.OPENSHELL_GATEWAY_INSECURE; + try { + process.env.OPENSHELL_GATEWAY_ENDPOINT = 'http://wrong-gateway'; + process.env.OPENSHELL_GATEWAY_INSECURE = 'true'; + const output = await commandRunner(process.execPath)( + [ + '-e', + 'console.log(JSON.stringify([process.env.OPENSHELL_GATEWAY_ENDPOINT, process.env.OPENSHELL_GATEWAY_INSECURE]))', + ], + { stage: 'version', timeoutMs: 1000 } + ); + assert.deepEqual(JSON.parse(output), [null, null]); + } finally { + if (savedEndpoint === undefined) + delete process.env.OPENSHELL_GATEWAY_ENDPOINT; + else process.env.OPENSHELL_GATEWAY_ENDPOINT = savedEndpoint; + if (savedInsecure === undefined) + delete process.env.OPENSHELL_GATEWAY_INSECURE; + else process.env.OPENSHELL_GATEWAY_INSECURE = savedInsecure; + } +}); + +test('late cancellation cannot report success while cleanup completes', async () => { + const controller = new AbortController(); + const { run } = fixture({ + delete: args => { + controller.abort(); + return `✓ Deleted sandbox ${args.at(-1)}\n`; + }, + }); + const result = await runProbe( + { ...options, signal: controller.signal }, + run + ); + assert.equal(result.ok, false); + assert.equal(result.failedStage, 'cancelled'); + assert.equal(result.cleanup, 'confirmed'); +}); + +test('command runner rejects oversized output without exposing it', async () => { + await assert.rejects( + commandRunner(process.execPath)( + ['-e', 'process.stdout.write("x".repeat(100000))'], + { stage: 'exec', timeoutMs: 1000 } + ), + error => { + assert.equal(error.message, 'OpenShell probe failed at exec'); + assert.equal(error.stdout, undefined); + return true; + } + ); +}); + +test('command runner handles a missing CLI without exposing process details', async () => { + await assert.rejects( + commandRunner('codeapi-nonexistent-probe-binary')([], { + stage: 'version', + timeoutMs: 1000, + }), + /failed at version/ + ); +}); diff --git a/service/bun.lock b/service/bun.lock index 0a0dea4a..9aacef7e 100644 --- a/service/bun.lock +++ b/service/bun.lock @@ -12,14 +12,14 @@ "@opentelemetry/exporter-trace-otlp-proto": "0.219.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0", - "axios": "^1.8.4", + "axios": "^1.20.0", "bullmq": "^5.13.1", "busboy": "^1.6.0", "dotenv": "^17.4.2", - "express": "^4.19.2", + "express": "^4.22.3", "express-rate-limit": "^7.4.1", "ioredis": "^5.4.1", - "minio": "^8.0.5", + "minio": "^8.0.7", "nanoid": "^5.1.16", "prom-client": "^15.1.3", "rate-limit-redis": "^4.2.0", @@ -52,6 +52,8 @@ }, "overrides": { "decode-uri-component": "0.5.0", + "qs": "6.16.0", + "uuid": "11.1.1", }, "packages": { "@aws-sdk/checksums": ["@aws-sdk/checksums@3.1000.19", "", { "dependencies": { "@aws-sdk/core": "^3.976.0", "@aws-sdk/types": "^3.974.2", "@smithy/core": "^3.29.4", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-Hc4N100RdkuWshKBnhPzmpdftfi9mCLz+OHFELHM1QIgMH4QRUUWyWgfiebta/YX2Bd62wTcm3EqAP8TeXv0gA=="], @@ -142,6 +144,8 @@ "@napi-rs/wasm-runtime": ["@napi-rs/wasm-runtime@0.2.12", "", { "dependencies": { "@emnapi/core": "^1.4.3", "@emnapi/runtime": "^1.4.3", "@tybys/wasm-util": "^0.10.0" } }, "sha512-ZVWUcfwY4E/yPitQJl481FjFo3K22D6qF0DuFH6Y/nbnE11GY5uguDxZMGXPQ8WQ0128MXQD7TnfHyK4oWoIJQ=="], + "@nodable/entities": ["@nodable/entities@3.1.0", "", {}, "sha512-LsS/DjHr+uDM647Gru/cA8+J3a3HfhttwCKLyuoyN7yXTFCxKBtSgMQBvrW9yNPa2/zDRUNuGPaH5QUFoa4arQ=="], + "@nodelib/fs.scandir": ["@nodelib/fs.scandir@2.1.5", "", { "dependencies": { "@nodelib/fs.stat": "2.0.5", "run-parallel": "^1.1.9" } }, "sha512-vq24Bq3ym5HEQm2NKCr3yXDwjc7vTsEThRDnkp2DK9p1uqLR+DHurm/NOTo0KG7HYHU7eppKZj3MyqYuMBf62g=="], "@nodelib/fs.stat": ["@nodelib/fs.stat@2.0.5", "", {}, "sha512-RkhPPp2zrqDAQA/2jNhnztcPAlv64XdhIp7a7454A5ovI7Bukxgt7MX7udwAu3zg1DcpPU0rz3VV1SeaqvY4+A=="], @@ -312,8 +316,6 @@ "@unrs/resolver-binding-win32-x64-msvc": ["@unrs/resolver-binding-win32-x64-msvc@1.11.1", "", { "os": "win32", "cpu": "x64" }, "sha512-lrW200hZdbfRtztbygyaq/6jP6AKE8qQN2KvPcJ+x7wiD038YtnYtZ82IMNJ69GJibV7bwL3y9FgK+5w/pYt6g=="], - "@zxing/text-encoding": ["@zxing/text-encoding@0.9.0", "", {}, "sha512-U/4aVJ2mxI0aDNI8Uq0wEhMgY+u4CNtEb0om3+y3+niDAsoTCOB33UF0sxpzqzdqXLqmvc+vZyAt4O8pPdfkwA=="], - "accepts": ["accepts@1.3.8", "", { "dependencies": { "mime-types": "~2.1.34", "negotiator": "0.6.3" } }, "sha512-PYAthTa2m2VKxuvSD3DPC/Gy+U+sOA1LAuT8mkmRuvw+NACSaeXEQ+NHcVF7rONl6qcaxV3Uuemwawk+7+SJLw=="], "acorn": ["acorn@8.15.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg=="], @@ -322,12 +324,16 @@ "acorn-walk": ["acorn-walk@8.3.4", "", { "dependencies": { "acorn": "^8.11.0" } }, "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g=="], - "ajv": ["ajv@6.12.6", "", { "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g=="], + "agent-base": ["agent-base@6.0.2", "", { "dependencies": { "debug": "4" } }, "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ=="], + + "ajv": ["ajv@6.14.0", "", { "dependencies": { "fast-deep-equal": "^3.1.1", "fast-json-stable-stringify": "^2.0.0", "json-schema-traverse": "^0.4.1", "uri-js": "^4.2.2" } }, "sha512-IWrosm/yrn43eiKqkfkHis7QioDleaXQHdDVPKg0FSwwd/DuvyX79TZnFOnYpB7dcsFAMmtFztZuXPDvSePkFw=="], "ansi-regex": ["ansi-regex@5.0.1", "", {}, "sha512-quJQXlTSUGL2LH9SUXo8VwsY4soanhgo6LNSm84E1LBcE8s3O0wpdiRzyR9z/ZZJMlMWv37qOOb9pdJlMUEKFQ=="], "ansi-styles": ["ansi-styles@4.3.0", "", { "dependencies": { "color-convert": "^2.0.1" } }, "sha512-zbB9rCJAT1rbjiVDb2hqKFHNYLxgtk8NURxZ3IZwD3F6NtxbXZQCnnSi1Lkx+IDohdPlFp222wVALIheZJQSEg=="], + "anynum": ["anynum@1.0.1", "", {}, "sha512-N6//FLET/tXYNM/F6ABca1oH6fWB+KlTt909Le28WMDBk8oaT4vY17DCrwg2MvmuqUKt3Ni4N5dGJ/EoBgcO6A=="], + "arg": ["arg@4.1.3", "", {}, "sha512-58S9QDqG0Xx27YwPSt9fJxivjYl432YCwfDMfZ+71RAqUrZef7LrKQZ3LHLOwCS4FLNBplP533Zx895SeOCHvA=="], "argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], @@ -358,7 +364,7 @@ "available-typed-arrays": ["available-typed-arrays@1.0.7", "", { "dependencies": { "possible-typed-array-names": "^1.0.0" } }, "sha512-wvUjBtSGN7+7SjNpq/9M2Tg350UZD3q62IFZLbRAR1bSMlCo1ZaeW+BJ+D090e4hIIZLBcTDWe4Mh4jvUDajzQ=="], - "axios": ["axios@1.13.2", "", { "dependencies": { "follow-redirects": "^1.15.6", "form-data": "^4.0.4", "proxy-from-env": "^1.1.0" } }, "sha512-VPk9ebNqPcy5lRGuSlKx752IlDatOjT9paPlm8A7yOuW2Fbvp4X3JznJtT4f0GzGLLiWE9W8onz51SqLYwzGaA=="], + "axios": ["axios@1.20.0", "", { "dependencies": { "follow-redirects": "^1.16.0", "form-data": "^4.0.6", "https-proxy-agent": "^5.0.1", "proxy-from-env": "^2.1.0" } }, "sha512-r8aOh8j9cGKpgQAqpzrUHnSIc6a59Y3Xf/cv8sy1DrHCkZHzQGEuoq1tARk6qSyDdtQGSDgpb9kFlruzPvrgwg=="], "balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], @@ -366,11 +372,11 @@ "block-stream2": ["block-stream2@2.1.0", "", { "dependencies": { "readable-stream": "^3.4.0" } }, "sha512-suhjmLI57Ewpmq00qaygS8UgEq2ly2PCItenIyhMqVjo4t4pGzqMvfgJuX8iWTeSDdfSSqS6j38fL4ToNL7Pfg=="], - "body-parser": ["body-parser@1.20.4", "", { "dependencies": { "bytes": "~3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "~1.2.0", "http-errors": "~2.0.1", "iconv-lite": "~0.4.24", "on-finished": "~2.4.1", "qs": "~6.14.0", "raw-body": "~2.5.3", "type-is": "~1.6.18", "unpipe": "~1.0.0" } }, "sha512-ZTgYYLMOXY9qKU/57FAo8F+HA2dGX7bqGc71txDRC1rS4frdFI5R7NhluHxH6M0YItAP0sHB4uqAOcYKxO6uGA=="], + "body-parser": ["body-parser@1.20.6", "", { "dependencies": { "bytes": "~3.1.2", "content-type": "~1.0.5", "debug": "2.6.9", "depd": "2.0.0", "destroy": "~1.2.0", "http-errors": "~2.0.1", "iconv-lite": "~0.4.24", "on-finished": "~2.4.1", "qs": "~6.15.1", "raw-body": "~2.5.3", "type-is": "~1.6.18", "unpipe": "~1.0.0" } }, "sha512-p5tAzS57i5MV9fZFDj9LeIiTZEufbSe2eDozP+ElheSUq1m74CRq1jI4mYNDdVs9vQztXFLuk/Gd6BWTdwRJ5g=="], "bowser": ["bowser@2.14.1", "", {}, "sha512-tzPjzCxygAKWFOJP011oxFHs57HzIhOEracIgAePE4pqB3LikALKnSzUyU4MGs9/iCEUuHlAJTjTc5M+u7YEGg=="], - "brace-expansion": ["brace-expansion@1.1.12", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-9T9UjW3r0UW5c1Q7GTwllptXwhvYmEzFhzMfZ9H7FQWt+uZePjZPjBP/W1ZEyZ1twGWom5/56TF4lPcqjnDHcg=="], + "brace-expansion": ["brace-expansion@1.1.21", "", { "dependencies": { "balanced-match": "^1.0.0", "concat-map": "0.0.1" } }, "sha512-9zeA+KLZNNzglF2TPKRQEDyx6Yby7daAkuy8MiPzpXPsYDWi/DRM8jmwUDxokQjYqBpv5DgPiwD4h4ZZSy1Ujw=="], "braces": ["braces@3.0.3", "", { "dependencies": { "fill-range": "^7.1.1" } }, "sha512-yQbXgO/OSZVD2IsiLlro+7Hf6Q18EJrKSEsdoMzKePKXct3gvD8oLcOQdIzGupr5Fj+EDe8gO/lxc1BzfMpxvA=="], @@ -456,7 +462,7 @@ "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], - "diff": ["diff@4.0.2", "", {}, "sha512-58lmxKSA4BNyLz+HHMUzlOEpg09FV+ev6ZMe3vJihgdxzgcwZ8VoEEPmALCZG9LmqfVoNMMKpttIYTVG6uDY7A=="], + "diff": ["diff@4.0.4", "", {}, "sha512-X07nttJQkwkfKfvTPG/KSnE2OMdcUCao6+eXF3wmnIQRn2aPAHH3VxDbDOdegkd6JbPsXqShpvEOHfAT+nCNwQ=="], "dir-glob": ["dir-glob@3.0.1", "", { "dependencies": { "path-type": "^4.0.0" } }, "sha512-WkrWp9GR4KXfKGYzOLmTuGVi1UWFfws377n9cc55/tb6DuqyF6pcQ5AbiHEshaDpY9v6oaSr2XCDidGmMwdzIA=="], @@ -520,7 +526,7 @@ "eventemitter3": ["eventemitter3@5.0.1", "", {}, "sha512-GWkBvjiSZK87ELrYOSESUYeVIc9mvLLf/nXalMOS5dYrgZq9o5OVkbZAVM06CVxYsCwH9BDZFPlQTlPA1j4ahA=="], - "express": ["express@4.22.1", "", { "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", "body-parser": "~1.20.3", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", "cookie-signature": "~1.0.6", "debug": "2.6.9", "depd": "2.0.0", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", "finalhandler": "~1.3.1", "fresh": "~0.5.2", "http-errors": "~2.0.0", "merge-descriptors": "1.0.3", "methods": "~1.1.2", "on-finished": "~2.4.1", "parseurl": "~1.3.3", "path-to-regexp": "~0.1.12", "proxy-addr": "~2.0.7", "qs": "~6.14.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", "serve-static": "~1.16.2", "setprototypeof": "1.2.0", "statuses": "~2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" } }, "sha512-F2X8g9P1X7uCPZMA3MVf9wcTqlyNp7IhH5qPCI0izhaOIYXaW9L535tGA3qmjRzpH+bZczqq7hVKxTR4NWnu+g=="], + "express": ["express@4.22.3", "", { "dependencies": { "accepts": "~1.3.8", "array-flatten": "1.1.1", "body-parser": "~1.20.5", "content-disposition": "~0.5.4", "content-type": "~1.0.4", "cookie": "~0.7.1", "cookie-signature": "~1.0.6", "debug": "2.6.9", "depd": "2.0.0", "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "etag": "~1.8.1", "finalhandler": "~1.3.1", "fresh": "~0.5.2", "http-errors": "~2.0.0", "merge-descriptors": "1.0.3", "methods": "~1.1.2", "on-finished": "~2.4.1", "parseurl": "~1.3.3", "path-to-regexp": "~0.1.13", "proxy-addr": "~2.0.7", "qs": "~6.16.0", "range-parser": "~1.2.1", "safe-buffer": "5.2.1", "send": "~0.19.0", "serve-static": "~1.16.2", "setprototypeof": "1.2.0", "statuses": "~2.0.1", "type-is": "~1.6.18", "utils-merge": "1.0.1", "vary": "~1.1.2" } }, "sha512-Bdcs4+3qlpVlx2NRn6fgX2Ue2/gGRaPeawebgclM0ERSCqDpA+owF1fdPwjJUTAJWMTuAaxjDf+hzb0/4eKvvw=="], "express-rate-limit": ["express-rate-limit@7.5.1", "", { "peerDependencies": { "express": ">= 4.11" } }, "sha512-7iN8iPMDzOMHPUYllBEsQdWVB6fPDMPqwjBaFrgr4Jgr/+okjvzAy+UHlYYL/Vs0OsOrMkwS6PJDkFlJwoxUnw=="], @@ -532,7 +538,9 @@ "fast-levenshtein": ["fast-levenshtein@2.0.6", "", {}, "sha512-DCXu6Ifhqcks7TZKY3Hxp3y6qphY5SJZmrWMDrKcERSOXWQdMhU9Ig/PYrzyw/ul9jOIyh0N4M0tbC5hodg8dw=="], - "fast-xml-parser": ["fast-xml-parser@4.5.3", "", { "dependencies": { "strnum": "^1.1.1" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-RKihhV+SHsIUGXObeVy9AXiBbFwkVk7Syp8XgwN5U3JV416+Gwp/GO9i0JYKmikykgz/UHRrrV4ROuZEo/T0ig=="], + "fast-xml-builder": ["fast-xml-builder@1.3.1", "", { "dependencies": { "path-expression-matcher": "^1.6.2", "xml-naming": "^0.3.0" } }, "sha512-pIM/1n3ntFXKYrUZwW7QCK0gAW7XY+wzj1YMIV3tLDvPj/V+zTGJK5e3/4WJfwj0qWw2ElNXiTixda/R+3YSug=="], + + "fast-xml-parser": ["fast-xml-parser@5.11.2", "", { "dependencies": { "@nodable/entities": "^3.0.1", "fast-xml-builder": "^1.2.0", "is-unsafe": "^2.0.0", "path-expression-matcher": "^1.6.2", "strnum": "^2.4.2", "xml-naming": "^0.3.0" }, "bin": { "fxparser": "src/cli/cli.js" } }, "sha512-R9iDuNrQYeQut46cn2r2wHKn4HYzVDvDm5J1wW+koZewykv0yuO2HChTYeZtrULyHIDM9cj9TUXloCsueCQUog=="], "fastq": ["fastq@1.19.1", "", { "dependencies": { "reusify": "^1.0.4" } }, "sha512-GwLTyxkCXjXbxqIhTsMI2Nui8huMPtnxg7krajPJAjnEG/iiOS7i+zCtWGZR9G0NBKbXKh6X9m9UIsYX/N6vvQ=="], @@ -556,15 +564,15 @@ "flat-cache": ["flat-cache@3.2.0", "", { "dependencies": { "flatted": "^3.2.9", "keyv": "^4.5.3", "rimraf": "^3.0.2" } }, "sha512-CYcENa+FtcUKLmhhqyctpclsq7QF38pKjZHsGNiSQF5r4FtoKDWabFDl3hzaEQMvT1LHEysw5twgLvpYYb4vbw=="], - "flatted": ["flatted@3.3.3", "", {}, "sha512-GX+ysw4PBCz0PzosHDepZGANEuFCMLrnRTiEy9McGjmkCQYwRq4A/X786G/fjM/+OjsWSU1ZrY5qyARZmO/uwg=="], + "flatted": ["flatted@3.4.2", "", {}, "sha512-PjDse7RzhcPkIJwy5t7KPWQSZ9cAbzQXcafsetQoD7sOJRQlGikNbx7yZp2OotDnJyrDcbyRq3Ttb18iYOqkxA=="], "fn.name": ["fn.name@1.1.0", "", {}, "sha512-GRnmB5gPyJpAhTQdSZTSp9uaPSvl09KoYcMQtsB9rQoOmzs9dH6ffeccH+Z+cv6P68Hu5bC6JjRh4Ah/mHSNRw=="], - "follow-redirects": ["follow-redirects@1.15.11", "", {}, "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ=="], + "follow-redirects": ["follow-redirects@1.16.0", "", { "peerDependencies": { "debug": "*" }, "optionalPeers": ["debug"] }, "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw=="], "for-each": ["for-each@0.3.5", "", { "dependencies": { "is-callable": "^1.2.7" } }, "sha512-dKx12eRCVIzqCxFGplyFKJMPvLEWgmNtUrpTiJIR5u97zEhRG8ySrtboPHZXx7daLxQVrl643cTzbab2tkQjxg=="], - "form-data": ["form-data@4.0.5", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.2", "mime-types": "^2.1.12" } }, "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w=="], + "form-data": ["form-data@4.0.6", "", { "dependencies": { "asynckit": "^0.4.0", "combined-stream": "^1.0.8", "es-set-tostringtag": "^2.1.0", "hasown": "^2.0.4", "mime-types": "^2.1.35" } }, "sha512-vKatAh4SlVfgbv+YtmhiRjhEMJsYpsG1Y2rMQtR+SVSbytsSD1YGzDIcrAJmdFec88u/+VoGmxnl+80gL1tRCQ=="], "forwarded": ["forwarded@0.2.0", "", {}, "sha512-buRG0fpBtRHSTCOASe6hD258tEubFoRLb4ZNA6NxMVHNw2gOcwHo9wyablzMzOA5z9xA9L1KNjk/Nt6MT9aYow=="], @@ -624,6 +632,8 @@ "http-errors": ["http-errors@2.0.1", "", { "dependencies": { "depd": "~2.0.0", "inherits": "~2.0.4", "setprototypeof": "~1.2.0", "statuses": "~2.0.2", "toidentifier": "~1.0.1" } }, "sha512-4FbRdAX+bSdmo4AUFuS0WNiPz8NgFt+r8ThgNWmlrjQjt1Q7ZR9+zTlce2859x4KSXrwIsaeTqDoKQmtP8pLmQ=="], + "https-proxy-agent": ["https-proxy-agent@5.0.1", "", { "dependencies": { "agent-base": "6", "debug": "4" } }, "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA=="], + "iconv-lite": ["iconv-lite@0.4.24", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3" } }, "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA=="], "ignore": ["ignore@7.0.5", "", {}, "sha512-Hs59xBNfUIunMFgWAbGX5cq6893IbWg4KnrjbYwX3tx0ztorVgTDA6B2sxf8ejHJ4wz8BqGUMYlnzNBer5NvGg=="], @@ -644,8 +654,6 @@ "ipaddr.js": ["ipaddr.js@2.3.0", "", {}, "sha512-Zv/pA+ciVFbCSBBjGfaKUya/CcGmUHzTydLMaTwrUUEM2DIEO3iZvueGxmacvmN50fGpGVKeTXpb2LcYQxeVdg=="], - "is-arguments": ["is-arguments@1.2.0", "", { "dependencies": { "call-bound": "^1.0.2", "has-tostringtag": "^1.0.2" } }, "sha512-7bVbi0huj/wrIAOzb8U1aszg9kdi3KN/CyU19CTI7tAoZYEZoL9yCDXpbXN+uPsuWnP02cyug1gleqq+TU+YCA=="], - "is-array-buffer": ["is-array-buffer@3.0.5", "", { "dependencies": { "call-bind": "^1.0.8", "call-bound": "^1.0.3", "get-intrinsic": "^1.2.6" } }, "sha512-DDfANUiiG2wC1qawP66qlTugJeL5HyzMpfr8lLK+jMQirGzNod0B12cFB/9q838Ru27sBwfw78/rdoU7RERz6A=="], "is-async-function": ["is-async-function@2.1.1", "", { "dependencies": { "async-function": "^1.0.0", "call-bound": "^1.0.3", "get-proto": "^1.0.1", "has-tostringtag": "^1.0.2", "safe-regex-test": "^1.1.0" } }, "sha512-9dgM/cZBnNvjzaMYHVoxxfPj2QXt22Ev7SuuPrs+xav0ukGB0S6d4ydZdEiM48kLx5kDV+QBPrpVnFyefL8kkQ=="], @@ -702,6 +710,8 @@ "is-typed-array": ["is-typed-array@1.1.15", "", { "dependencies": { "which-typed-array": "^1.1.16" } }, "sha512-p3EcsicXjit7SaskXHs1hA91QxgTw46Fv6EFKKGS5DRFLD8yKnohjF3hxoju94b/OcMZoQukzpPpBE9uLVKzgQ=="], + "is-unsafe": ["is-unsafe@2.0.2", "", {}, "sha512-HgbIHPBH0KHHCcjLfGsCvhtPTVxjaAZlXjwdz7/GQC40SjSe4sfQsar8J5VFo8JOSbarkpV0OLG95bbaNd9aAQ=="], + "is-weakmap": ["is-weakmap@2.0.2", "", {}, "sha512-K5pXYOm9wqY1RgjpL3YTkF39tni1XajUIkawTLUo9EZEVUFga5gSQJF8nNS7ZwJQ02y+1YCNYcMh+HIf1ZqE+w=="], "is-weakref": ["is-weakref@1.1.1", "", { "dependencies": { "call-bound": "^1.0.3" } }, "sha512-6i9mGWSlqzNMEqpCp93KwRS1uUOodk2OJ6b+sq7ZPDSy2WuI5NFIxp/254TytR8ftefexkWn5xNiHUNpPOfSew=="], @@ -712,7 +722,7 @@ "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], - "js-yaml": ["js-yaml@4.1.1", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-qQKT4zQxXl8lLwBtHMWwaTcGfFOZviOJet3Oy/xmGk2gZH677CJM9EvtfdSkgWcATZhj/55JZ0rmy3myCT5lsA=="], + "js-yaml": ["js-yaml@4.3.2", "", { "dependencies": { "argparse": "^2.0.1" }, "bin": { "js-yaml": "bin/js-yaml.js" } }, "sha512-SFNOvSJ+Dgf/9An904Yx+CgSlIPCkIpao4qo51lpee25TIRejdH3rhR4EZMGoNx3/TP3O+wzWuiTFl4sqbltzA=="], "json-buffer": ["json-buffer@3.0.1", "", {}, "sha512-4bV5BfR2mqfQTJm+V5tPPdf+ZpuhiIvTuAB5g8kcrXOZpTT/QwwVRWBywX1ozr6lEuPdbHxwaJlm9G6mI2sfSQ=="], @@ -732,7 +742,7 @@ "locate-path": ["locate-path@6.0.0", "", { "dependencies": { "p-locate": "^5.0.0" } }, "sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw=="], - "lodash": ["lodash@4.17.21", "", {}, "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg=="], + "lodash": ["lodash@4.18.0", "", {}, "sha512-l1mfj2atMqndAHI3ls7XqPxEjV2J9ZkcNyHpoZA3r2T1LLwDB69jgkMWh71YKwhBbK0G2f4WSn05ahmQXVxupA=="], "lodash.defaults": ["lodash.defaults@4.2.0", "", {}, "sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ=="], @@ -766,11 +776,11 @@ "mime-types": ["mime-types@2.1.35", "", { "dependencies": { "mime-db": "1.52.0" } }, "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw=="], - "minimatch": ["minimatch@3.1.2", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-J7p63hRiAjw1NDEww1W7i37+ByIrOWO5XQQAzZ3VOcL0PNybwpfmV/N05zFAzwQ9USyEcX6t3UO+K5aqBQOIHw=="], + "minimatch": ["minimatch@3.1.4", "", { "dependencies": { "brace-expansion": "^1.1.7" } }, "sha512-twmL+S8+7yIsE9wsqgzU3E8/LumN3M3QELrBZ20OdmQ9jB2JvW5oZtBEmft84k/Gs5CG9mqtWc6Y9vW+JEzGxw=="], "minimist": ["minimist@1.2.8", "", {}, "sha512-2yyAR8qBkN3YuheJanUpWC5U3bb5osDywNB8RzDVlDwDHbocAJveqqj1u8+SVD7jkWT4yvsHCpWqqWqAxb0zCA=="], - "minio": ["minio@8.0.6", "", { "dependencies": { "async": "^3.2.4", "block-stream2": "^2.1.0", "browser-or-node": "^2.1.1", "buffer-crc32": "^1.0.0", "eventemitter3": "^5.0.1", "fast-xml-parser": "^4.4.1", "ipaddr.js": "^2.0.1", "lodash": "^4.17.21", "mime-types": "^2.1.35", "query-string": "^7.1.3", "stream-json": "^1.8.0", "through2": "^4.0.2", "web-encoding": "^1.1.5", "xml2js": "^0.5.0 || ^0.6.2" } }, "sha512-sOeh2/b/XprRmEtYsnNRFtOqNRTPDvYtMWh+spWlfsuCV/+IdxNeKVUMKLqI7b5Dr07ZqCPuaRGU/rB9pZYVdQ=="], + "minio": ["minio@8.0.7", "", { "dependencies": { "async": "^3.2.4", "block-stream2": "^2.1.0", "browser-or-node": "^2.1.1", "buffer-crc32": "^1.0.0", "eventemitter3": "^5.0.1", "fast-xml-parser": "^5.3.4", "ipaddr.js": "^2.0.1", "lodash": "^4.17.21", "mime-types": "^2.1.35", "query-string": "^7.1.3", "stream-json": "^1.8.0", "through2": "^4.0.2", "xml2js": "^0.5.0 || ^0.6.2" } }, "sha512-E737MgufW8CeQAsTAtnEMrxZ9scMSf29kkhZoXzDTKj/Jszzo2SfeZUH9wbDQH2Rsq6TCtl/yQL0+XdVKZansQ=="], "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], @@ -822,17 +832,19 @@ "path-exists": ["path-exists@4.0.0", "", {}, "sha512-ak9Qy5Q7jYb2Wwcey5Fpvg2KoAc/ZIhLSLOSBmRmygPsGwkVVt0fZa0qrtMz+m6tJTAHfZQ8FnmB4MG4LWy7/w=="], + "path-expression-matcher": ["path-expression-matcher@1.6.2", "", {}, "sha512-enSlaiat05iasnzmgNxRj8reFdj3puY2QpNgP1aPIaVfT6nn9ICuPoFlKHk8EN22HcwewshO+mN2DGbkCEOtqQ=="], + "path-is-absolute": ["path-is-absolute@1.0.1", "", {}, "sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg=="], "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], "path-parse": ["path-parse@1.0.7", "", {}, "sha512-LDJzPVEEEPR+y48z93A0Ed0yXb8pAByGWo/k5YYdYgpY2/2EsOsksJrq7lOHxryrVOn1ejG6oAp8ahvOIQD8sw=="], - "path-to-regexp": ["path-to-regexp@0.1.12", "", {}, "sha512-RA1GjUVMnvYFxuqovrEqZoxxW5NUZqbwKtYz/Tt7nXerk0LbLblQmrsgdeOxV5SFHf0UDggjS/bSeOZwt1pmEQ=="], + "path-to-regexp": ["path-to-regexp@0.1.13", "", {}, "sha512-A/AGNMFN3c8bOlvV9RreMdrv7jsmF9XIfDeCd87+I8RNg6s78BhJxMu69NEMHBSJFxKidViTEdruRwEk/WIKqA=="], "path-type": ["path-type@4.0.0", "", {}, "sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw=="], - "picomatch": ["picomatch@4.0.3", "", {}, "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q=="], + "picomatch": ["picomatch@4.0.4", "", {}, "sha512-QP88BAKvMam/3NxH6vj2o21R6MjxZUAd6nlwAS/pnGvN9IVLocLHxGYIzFhg6fUQ+5th6P4dv4eW9jX3DSIj7A=="], "possible-typed-array-names": ["possible-typed-array-names@1.1.0", "", {}, "sha512-/+5VFTchJDoVj3bhoqi6UeymcD00DAwb1nJwamzPvHEszJ4FpF6SNNbUbOS8yI56qHzdV8eK0qEfOSiodkTdxg=="], @@ -842,11 +854,11 @@ "proxy-addr": ["proxy-addr@2.0.7", "", { "dependencies": { "forwarded": "0.2.0", "ipaddr.js": "1.9.1" } }, "sha512-llQsMLSUDUPT44jdrU/O37qlnifitDP+ZwrmmZcoSKyLKvtZxpyV0n2/bD/N4tBAAZ/gJEdZU7KMraoK1+XYAg=="], - "proxy-from-env": ["proxy-from-env@1.1.0", "", {}, "sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg=="], + "proxy-from-env": ["proxy-from-env@2.1.0", "", {}, "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA=="], "punycode": ["punycode@2.3.1", "", {}, "sha512-vYt7UD1U9Wg6138shLtLOvdAu+8DsC/ilFtEVHcH+wydcSpNE20AfSOduf6MkRFahL5FY7X1oU7nKVZFtfq8Fg=="], - "qs": ["qs@6.14.0", "", { "dependencies": { "side-channel": "^1.1.0" } }, "sha512-YWWTjgABSKcvs/nWBi9PycY/JiPJqOD4JA6o9Sej2AtvSGarXxKC3OQSk4pAarbdQlKAh5D4FCQkJNkW+GAn3w=="], + "qs": ["qs@6.16.0", "", { "dependencies": { "es-define-property": "^1.0.1", "side-channel": "^1.1.1" } }, "sha512-h6fhOIaRrID2CbEY2fqs+7t+UXZo+MLAnU5gRIq85uFtdiUPCdsApMlHhXogKVM4HM2DVbIjGNTTYH2OcmP1vA=="], "query-string": ["query-string@7.1.3", "", { "dependencies": { "decode-uri-component": "^0.2.2", "filter-obj": "^1.1.0", "split-on-first": "^1.0.0", "strict-uri-encode": "^2.0.0" } }, "sha512-hh2WYhq4fi8+b+/2Kg9CEge4fDPvHS534aOOvOZeQ3+Vf2mCFsaFBYj0i+iXcAq6I9Vzp5fjMFBlONvayDC1qg=="], @@ -880,7 +892,7 @@ "rimraf": ["rimraf@3.0.2", "", { "dependencies": { "glob": "^7.1.3" }, "bin": { "rimraf": "bin.js" } }, "sha512-JZkJMZkAGFFPP2YqXZXPbMlMBgsxzE8ILs4lMIX/2o0L9UBw9O/Y3o6wFw/i9YLapcUJWwqbi3kdxIPdC62TIA=="], - "rollup": ["rollup@2.79.2", "", { "optionalDependencies": { "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-fS6iqSPZDs3dr/y7Od6y5nha8dW1YnbgtsyotCVvoFGKbERG++CVRFv1meyGDE1SNItQA8BrnCw7ScdAhRJ3XQ=="], + "rollup": ["rollup@2.80.0", "", { "optionalDependencies": { "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-cIFJOD1DESzpjOBl763Kp1AH7UE/0fcdHe6rZXUdQ9c50uvgigvW97u3IcSeBwOkgqL/PXPBktBCh0KEu5L8XQ=="], "rollup-plugin-copy": ["rollup-plugin-copy@3.5.0", "", { "dependencies": { "@types/fs-extra": "^8.0.1", "colorette": "^1.1.0", "fs-extra": "^8.1.0", "globby": "10.0.1", "is-plain-object": "^3.0.0" } }, "sha512-wI8D5dvYovRMx/YYKtUNt3Yxaw4ORC9xo6Gt9t22kveWz1enG9QrhVlagzwrxSC455xD1dHMKhIJkbsQ7d48BA=="], @@ -920,9 +932,9 @@ "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], - "side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="], + "side-channel": ["side-channel@1.1.1", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.4", "side-channel-list": "^1.0.1", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-6x6dK6zJdpTzF4sQeNYxwtvBzf6Eg4GtlesS94HOvTudUeyK2WXAaIfmDgsyslYrRBeFIlsi54AYsFGUuhmvrQ=="], - "side-channel-list": ["side-channel-list@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3" } }, "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA=="], + "side-channel-list": ["side-channel-list@1.0.1", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.4" } }, "sha512-mjn/0bi/oUURjc5Xl7IaWi/OJJJumuoJFQJfDDyO46+hBWsfaVM65TBHq2eoZBhzl9EchxOijpkbRC8SVBQU0w=="], "side-channel-map": ["side-channel-map@1.0.1", "", { "dependencies": { "call-bound": "^1.0.2", "es-errors": "^1.3.0", "get-intrinsic": "^1.2.5", "object-inspect": "^1.13.3" } }, "sha512-VCjCNfgMsby3tTdo02nbjtM/ewra6jPHmpThenkTYh8pG9ucZ/1P8So4u4FGBek/BjpOVsDCMoLA/iuBKIFXRA=="], @@ -968,7 +980,7 @@ "strip-json-comments": ["strip-json-comments@3.1.1", "", {}, "sha512-6fPc+R4ihwqP6N/aIv2f1gMH8lOVtWQHoqC4yK6oSDVVocumAsfCqjkXnqiYMhmMwS/mEHLp7Vehlt3ql6lEig=="], - "strnum": ["strnum@1.1.2", "", {}, "sha512-vrN+B7DBIoTTZjnPNewwhx6cBA/H+IS7rfW68n7XxC1y7uoiGQBxaKzqucGUgavX15dJgiGztLJ8vxuEzwqBdA=="], + "strnum": ["strnum@2.4.2", "", { "dependencies": { "anynum": "^1.0.1" } }, "sha512-rDG3Ah4TV0k1hWvLSzkZtMmLN9+eS+h3knq4MP6A42Y3Yh5qGNnOUs1jJkoSr8FG5dsL28c7KgkIBzSEykqtuw=="], "supports-color": ["supports-color@7.2.0", "", { "dependencies": { "has-flag": "^4.0.0" } }, "sha512-qpCAvRl9stuOHveKsn7HncJRvv501qIacKzQlO/+Lwxc9+0q2wLyv4Dfvt80/DPn2pqOBsJdDiogXGR9+OvwRw=="], @@ -984,7 +996,7 @@ "tinyglobby": ["tinyglobby@0.2.15", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.3" } }, "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ=="], - "tmp": ["tmp@0.2.5", "", {}, "sha512-voyz6MApa1rQGUxT3E+BK7/ROe8itEx7vD8/HEvt4xwXucvQ5G5oeEiHkmHZJuBO21RpOf+YYm9MOivj709jow=="], + "tmp": ["tmp@0.2.7", "", {}, "sha512-e0votIpp4Uo2AJYSzVHV6xCcawuiez3DzqDAbrTc3YxBkplN6e+dM13ZeIcZnDg/QpSuU2zfZ3rzwY8ukEnaXw=="], "to-regex-range": ["to-regex-range@5.0.1", "", { "dependencies": { "is-number": "^7.0.0" } }, "sha512-65P7iz6X5yEr1cwcgvQxbbIw7Uk3gOy5dIdtZ4rDveLqhrdJP+Li/Hx6tyK0NEb+2GCyneCMJiGqrADCSNk8sQ=="], @@ -1028,20 +1040,16 @@ "uri-js": ["uri-js@4.4.1", "", { "dependencies": { "punycode": "^2.1.0" } }, "sha512-7rKUyy33Q1yc98pQ1DAmLtwX109F7TIfWlW1Ydo8Wl1ii1SeHieeh0HHfPeL2fMXK6z0s8ecKs9frCuLJvndBg=="], - "util": ["util@0.12.5", "", { "dependencies": { "inherits": "^2.0.3", "is-arguments": "^1.0.4", "is-generator-function": "^1.0.7", "is-typed-array": "^1.1.3", "which-typed-array": "^1.1.2" } }, "sha512-kZf/K6hEIrWHI6XqOFUiiMa+79wE/D8Q+NCNAWclkyg3b4d2k7s0QGepNjiABc+aR3N1PAyHL7p6UcLY6LmrnA=="], - "util-deprecate": ["util-deprecate@1.0.2", "", {}, "sha512-EPD5q1uXyFxJpCrLnCc1nHnq3gOa6DZBocAIiI2TaSCA7VCJ1UJDMagCzIkXNsUYfD1daK//LTEQ8xiIbrHtcw=="], "utils-merge": ["utils-merge@1.0.1", "", {}, "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA=="], - "uuid": ["uuid@11.1.0", "", { "bin": { "uuid": "dist/esm/bin/uuid" } }, "sha512-0/A9rDy9P7cJ+8w1c9WD9V//9Wj15Ce2MPz8Ri6032usz+NfePxx5AcN3bN+r6ZL6jEo066/yNYB3tn4pQEx+A=="], + "uuid": ["uuid@11.1.1", "", { "bin": { "uuid": "dist/esm/bin/uuid" } }, "sha512-vIYxrBCC/N/K+Js3qSN88go7kIfNPssr/hHCesKCQNAjmgvYS2oqr69kIufEG+O4+PfezOH4EbIeHCfFov8ZgQ=="], "v8-compile-cache-lib": ["v8-compile-cache-lib@3.0.1", "", {}, "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg=="], "vary": ["vary@1.1.2", "", {}, "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg=="], - "web-encoding": ["web-encoding@1.1.5", "", { "dependencies": { "util": "^0.12.3" }, "optionalDependencies": { "@zxing/text-encoding": "0.9.0" } }, "sha512-HYLeVCdJ0+lBYV2FvNZmv3HJ2Nt0QYXqZojk3d9FJOLkwnuhzM9tmamh8d7HPM8QqjKH8DeHkFTx+CFlWpZZDA=="], - "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], "which-boxed-primitive": ["which-boxed-primitive@1.1.1", "", { "dependencies": { "is-bigint": "^1.1.0", "is-boolean-object": "^1.2.1", "is-number-object": "^1.1.1", "is-string": "^1.1.1", "is-symbol": "^1.1.1" } }, "sha512-TbX3mj8n0odCBFVlY8AxkqcHASw3L60jIuF8jFP78az3C2YhmGvqbHBpAjTRH2/xqYunrJ9g1jSyjCjpoWzIAA=="], @@ -1060,6 +1068,8 @@ "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], + "xml-naming": ["xml-naming@0.3.0", "", {}, "sha512-ghig2TBE/H11aOVgmahA3MhimvkBr6JIYknH/Dhdk10nXwdbIqBJsbfMxpvFPG8bAw77gN29aQWvKpmVoPlvPQ=="], + "xml2js": ["xml2js@0.6.2", "", { "dependencies": { "sax": ">=0.6.0", "xmlbuilder": "~11.0.0" } }, "sha512-T4rieHaC1EXcES0Kxxj4JWgaUQHDk+qwHcYOCFHfiwKz7tOVPLq7Hjq9dM1WCMhylqMEfP7hMcOIChvotiZegA=="], "xmlbuilder": ["xmlbuilder@11.0.1", "", {}, "sha512-fDlsI/kFEx7gLvbecc0/ohLG50fugQp8ryHzMTuW9vSa1GJ0XYWKnhsUx7oie3G98+r56aTQIUB4kht42R3JvA=="], @@ -1088,11 +1098,11 @@ "@eslint/eslintrc/ignore": ["ignore@5.3.2", "", {}, "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g=="], - "@types/minimatch/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="], + "@types/minimatch/minimatch": ["minimatch@9.0.7", "", { "dependencies": { "brace-expansion": "^5.0.2" } }, "sha512-MOwgjc8tfrpn5QQEvjijjmDVtMw2oL88ugTevzxQnzRLm6l3fVEF2gzU0kYeYYKD8C66+IdGX6peJ4MyUlUnPg=="], "@types/serve-static/@types/send": ["@types/send@0.17.6", "", { "dependencies": { "@types/mime": "^1", "@types/node": "*" } }, "sha512-Uqt8rPBE8SY0RK8JB1EzVOIZ32uqy8HwdxCnoCOsYrvnswqmFZ/k+9Ikidlk/ImhsdvBsloHbAlewb2IEBV/Og=="], - "@typescript-eslint/typescript-estree/minimatch": ["minimatch@9.0.5", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-G6T0ZX48xgozx7587koeX9Ys2NYy6Gmv//P89sEte9V9whIapMNF4idKxnW2QtCcLiTWlb/wfCabAtAFWhhBow=="], + "@typescript-eslint/typescript-estree/minimatch": ["minimatch@9.0.7", "", { "dependencies": { "brace-expansion": "^5.0.2" } }, "sha512-MOwgjc8tfrpn5QQEvjijjmDVtMw2oL88ugTevzxQnzRLm6l3fVEF2gzU0kYeYYKD8C66+IdGX6peJ4MyUlUnPg=="], "@typescript-eslint/typescript-estree/semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], @@ -1122,13 +1132,17 @@ "finalhandler/debug": ["debug@2.6.9", "", { "dependencies": { "ms": "2.0.0" } }, "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA=="], + "form-data/hasown": ["hasown@2.0.4", "", { "dependencies": { "function-bind": "^1.1.2" } }, "sha512-T2UbfbBEF32wiepXIsMlTW9+dDYC6wMh/t/vYA4tuOMKqWz/n3vr1NFSxQiyP+zk2mXsoMA/i/7qV6LKut1t1A=="], + "globby/ignore": ["ignore@5.3.2", "", {}, "sha512-hsBTNUqQTDwkWtcdYI2i06Y/nUBEsNEDJKjWdigLvegy8kDuJAS8uRlpkkcQpyEXL0Z/pjDy5HBmMjRCJ2gq+g=="], + "internal-slot/side-channel": ["side-channel@1.1.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3", "side-channel-list": "^1.0.0", "side-channel-map": "^1.0.1", "side-channel-weakmap": "^1.0.2" } }, "sha512-ZX99e6tRweoUXqR+VBrslhda51Nh5MTQwou5tnUDgbtyM0dBgmhEDtWGP/xbKn6hqfPRHujUNwz5fy/wbbhnpw=="], + "ioredis-mock/semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], "is-bun-module/semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], - "micromatch/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + "micromatch/picomatch": ["picomatch@2.3.2", "", {}, "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA=="], "proxy-addr/ipaddr.js": ["ipaddr.js@1.9.1", "", {}, "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g=="], @@ -1146,9 +1160,9 @@ "@aws-sdk/middleware-sdk-s3/@aws-sdk/signature-v4-multi-region/@smithy/signature-v4": ["@smithy/signature-v4@5.6.9", "", { "dependencies": { "@smithy/core": "^3.29.8", "@smithy/types": "^4.16.1", "tslib": "^2.6.2" } }, "sha512-g5rnEii/mkT0mjVJmlsaOfyNBtHNTecD9Lo4NP8D5HzMUEnZNpz7/FbvBCjNcV4vteHFAxOGiLUYNxPkDZZAPw=="], - "@types/minimatch/minimatch/brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="], + "@types/minimatch/minimatch/brace-expansion": ["brace-expansion@5.0.12", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ=="], - "@typescript-eslint/typescript-estree/minimatch/brace-expansion": ["brace-expansion@2.0.2", "", { "dependencies": { "balanced-match": "^1.0.0" } }, "sha512-Jt0vHyM+jmUBqojB7E1NIYadt0vI0Qxjxd2TErW94wDz+E2LAm5vKMXXwg6ZZBTHPuUlDgQHKXvjGBdfcF1ZDQ=="], + "@typescript-eslint/typescript-estree/minimatch/brace-expansion": ["brace-expansion@5.0.12", "", { "dependencies": { "balanced-match": "^4.0.2" } }, "sha512-YovQ3rzhaLMIrDjNDMkNS01tea93qhEhG5xy8f6+R0l+dw3Ki+5sCoIoI942iuLZTHWogWktgwVDhU09iNEimQ=="], "body-parser/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], @@ -1158,12 +1172,18 @@ "finalhandler/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], + "internal-slot/side-channel/side-channel-list": ["side-channel-list@1.0.0", "", { "dependencies": { "es-errors": "^1.3.0", "object-inspect": "^1.13.3" } }, "sha512-FCLHtRD/gnpCiCHEiJLOwdmFP+wzCmDEkc9y7NsYxeF4u7Btsn1ZuwgwJGxImImHicJArLP4R0yX4c2KCrMrTA=="], + "rollup-plugin-sourcemaps/@rollup/pluginutils/@types/estree": ["@types/estree@0.0.39", "", {}, "sha512-EYNwp3bU+98cpU4lAWYYL7Zz+2gryWH1qbdDTidVd6hkiR6weksdbMadyXKXNPEkQFhXM+hVO9ZygomHXp+AIw=="], "rollup-plugin-sourcemaps/@rollup/pluginutils/estree-walker": ["estree-walker@1.0.1", "", {}, "sha512-1fMXF3YP4pZZVozF8j/ZLfvnR8NSIljt56UhbZ5PeeDmmGHpgpdwQt7ITlGvYaQukCvuBRMLEiKiYC+oeIg4cg=="], - "rollup-plugin-sourcemaps/@rollup/pluginutils/picomatch": ["picomatch@2.3.1", "", {}, "sha512-JU3teHTNjmE2VCGFzuY8EXzCDVwEqB2a8fsIvwaStHhAWJEeVd1o1QD80CU6+ZdEXXSLbSsuLwJjkCBWqRQUVA=="], + "rollup-plugin-sourcemaps/@rollup/pluginutils/picomatch": ["picomatch@2.3.2", "", {}, "sha512-V7+vQEJ06Z+c5tSye8S+nHUfI51xoXIXjHQ99cQtKUkQqqO1kO/KCJUfZXuB47h/YBlDhah2H3hdUGXn8ie0oA=="], "send/debug/ms": ["ms@2.0.0", "", {}, "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A=="], + + "@types/minimatch/minimatch/brace-expansion/balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], + + "@typescript-eslint/typescript-estree/minimatch/brace-expansion/balanced-match": ["balanced-match@4.0.4", "", {}, "sha512-BLrgEcRTwX2o6gGxGOCNyMvGSp35YofuYzw9h1IMTRmKqttAZZVU67bdb9Pr2vUHA8+j3i2tJfjO6C6+4myGTA=="], } } diff --git a/service/openapi.yml b/service/openapi.yml index a257fd18..07d2d7e3 100644 --- a/service/openapi.yml +++ b/service/openapi.yml @@ -64,6 +64,40 @@ components: - $ref: '#/components/schemas/ExecutionProfileError' schemas: + WorkspaceToolRequest: + type: object + description: Workspace tool protocol v1 body; operation-specific fields follow the worker protocol. + required: [protocolVersion, operation, workspaceId] + properties: + protocolVersion: { type: integer, enum: [1] } + operation: + type: string + enum: [list_files, search_text, read_file, write_file, preview_edit, edit_file, execute_command] + workspaceId: { type: string } + additionalProperties: true + + WorkspaceRequestStatus: + type: object + required: [requestId, state, workerId, workspaceId, queueWaitMs, cancelRequested] + properties: + requestId: { type: string } + state: { type: string, enum: [queued, admitted, completed, failed, cancelled] } + workerId: { type: string } + workspaceId: { type: string } + workspaceInstanceId: { type: string } + worktree: { type: string } + queuePosition: { type: integer, minimum: 1 } + queueWaitMs: { type: integer, minimum: 0 } + executionDeadlineAt: { type: string, format: date-time } + cancelRequested: { type: boolean } + result: { type: object, additionalProperties: true } + error: + type: object + required: [code, message] + properties: + code: { type: string } + message: { type: string } + FileRef: type: object properties: @@ -312,6 +346,77 @@ components: type: string paths: + /workspace-tools/capabilities: + get: + summary: Discover opt-in durable workspace request support + responses: + '200': + description: All replicas must support version 1 before a client enables durable submission. + content: + application/json: + schema: + type: object + properties: + durableWorkspaceRequests: { type: integer, enum: [0, 1] } + + /workspace-tools/requests: + post: + summary: Submit an idempotent durable workspace request + description: >- + Disconnect does not cancel accepted work. IDs and results are retained for 24 hours + from acceptance. Never resubmit after that retention window or fall back to synchronous + execution after durable acceptance. The execution budget begins after admission. + parameters: + - $ref: '#/components/parameters/ExpectedExecutionProfile' + - in: header + name: X-LibreChat-Workspace-Request-Id + required: true + schema: { type: string, pattern: '^[A-Za-z0-9_-]{16,128}$' } + - in: header + name: X-LibreChat-Workspace-Queue-Wait-Ms + schema: { type: integer, minimum: 1, maximum: 300000, default: 30000 } + requestBody: + required: true + content: + application/json: + schema: { $ref: '#/components/schemas/WorkspaceToolRequest' } + responses: + '202': + description: Accepted, or an identical request with this principal-scoped ID already exists. + content: + application/json: + schema: { $ref: '#/components/schemas/WorkspaceRequestStatus' } + '400': { $ref: '#/components/responses/BadRequest' } + '409': { $ref: '#/components/responses/Conflict' } + '429': { description: Admission capacity or execution rate limit reached } + '503': { description: Code environment unavailable } + + /workspace-tools/requests/{requestId}: + parameters: + - in: path + name: requestId + required: true + schema: { type: string, pattern: '^[A-Za-z0-9_-]{16,128}$' } + - $ref: '#/components/parameters/ExpectedExecutionProfile' + get: + summary: Read a retained workspace request and result without consuming it + responses: + '200': + description: Status scoped to the authenticated tenant and user. ASSIGNMENT_EXPIRED is not proof of non-execution. + content: + application/json: + schema: { $ref: '#/components/schemas/WorkspaceRequestStatus' } + '404': { description: Unknown, expired, or another principal's request } + delete: + summary: Request cancellation of the same logical call + responses: + '200': + description: Cancellation recorded, or terminal status returned. A committed result can win the race. + content: + application/json: + schema: { $ref: '#/components/schemas/WorkspaceRequestStatus' } + '404': { description: Unknown, expired, or another principal's request } + /hosted-apps: post: summary: Start or reassert a resident hosted app diff --git a/service/package.json b/service/package.json index c5a49a82..a3679c8f 100644 --- a/service/package.json +++ b/service/package.json @@ -33,14 +33,14 @@ "@opentelemetry/exporter-trace-otlp-proto": "0.219.0", "@opentelemetry/resources": "2.8.0", "@opentelemetry/sdk-trace-base": "2.8.0", - "axios": "^1.8.4", + "axios": "^1.20.0", "bullmq": "^5.13.1", "busboy": "^1.6.0", "dotenv": "^17.4.2", - "express": "^4.19.2", + "express": "^4.22.3", "express-rate-limit": "^7.4.1", "ioredis": "^5.4.1", - "minio": "^8.0.5", + "minio": "^8.0.7", "nanoid": "^5.1.16", "prom-client": "^15.1.3", "rate-limit-redis": "^4.2.0", @@ -74,6 +74,8 @@ "typescript": "^5.5.4" }, "overrides": { - "decode-uri-component": "0.5.0" + "decode-uri-component": "0.5.0", + "uuid": "11.1.1", + "qs": "6.16.0" } } diff --git a/service/scripts/dump-pipe-preamble.ts b/service/scripts/dump-pipe-preamble.ts new file mode 100644 index 00000000..86ec11fa --- /dev/null +++ b/service/scripts/dump-pipe-preamble.ts @@ -0,0 +1,6 @@ +process.env.DOTENV_CONFIG_QUIET = 'true'; +const { generatePreamble } = await import('../src/preamble'); +process.stdout.write(generatePreamble({ + callbackUrl: 'http://unused.invalid', callbackToken: 'pipe-test-token', + executionId: 'pipe-test-execution', tools: [], +})); diff --git a/service/src/bridge/admission.ts b/service/src/bridge/admission.ts index 4002ed6d..18ab1f49 100644 --- a/service/src/bridge/admission.ts +++ b/service/src/bridge/admission.ts @@ -1,5 +1,22 @@ import type Redis from 'ioredis'; +/** Redis time and claim ownership gate every durable admission mutation. */ +export function durableAdmissionFence( + requestKey: string, + claimKey: string, + token: string, + rejected: number, +): string[] { + return [ + `if redis.call('GET', ${claimKey}) ~= ${token} then return ${rejected} end`, + `if redis.call('HGET', ${requestKey}, 'state') ~= 'queued' then return ${rejected} end`, + `if redis.call('HGET', ${requestKey}, 'cancelRequested') == '1' then return ${rejected} end`, + 'local admissionTime = redis.call(\'TIME\')', + 'local admissionNowMs = tonumber(admissionTime[1]) * 1000.0 + math.floor(tonumber(admissionTime[2]) / 1000)', + `if tonumber(redis.call('HGET', ${requestKey}, 'queueDeadlineAtMs')) <= admissionNowMs then return ${rejected} end`, + ]; +} + /** Bounded FIFO admission shared by API replicas. Entries expire after caller deadlines. */ export class BridgeAdmissionQueue { constructor( @@ -57,6 +74,44 @@ export class BridgeAdmissionQueue { ); } + async submit(args: { + workerId: string; id: string; deadlineAtMs: number; workspaceId?: string; + key: string; activeKey: string; fingerprint: string; record: string; retentionMs: number; + }): Promise<'accepted' | 'existing' | 'conflict' | 'full'> { + const result = Number(await this.redis.eval([ + 'local fingerprint = redis.call(\'HGET\', KEYS[5], \'fingerprint\')', + 'if fingerprint then', + ' if fingerprint == ARGV[6] then return 2 end', + ' return -1', + 'end', + 'local expired = redis.call(\'ZRANGEBYSCORE\', KEYS[2], \'-inf\', ARGV[2])', + 'for _, id in ipairs(expired) do', + ' redis.call(\'ZREM\', KEYS[1], id); redis.call(\'ZREM\', KEYS[2], id); redis.call(\'HDEL\', KEYS[4], id)', + 'end', + 'if redis.call(\'ZCARD\', KEYS[1]) >= tonumber(ARGV[4]) then return 0 end', + 'local sequence = redis.call(\'INCR\', KEYS[3])', + 'redis.call(\'ZADD\', KEYS[1], sequence, ARGV[1])', + 'redis.call(\'ZADD\', KEYS[2], ARGV[3], ARGV[1])', + 'if ARGV[5] ~= \'\' then redis.call(\'HSET\', KEYS[4], ARGV[1], ARGV[5]) end', + 'local latest = redis.call(\'ZREVRANGE\', KEYS[2], 0, 0, \'WITHSCORES\')', + 'for i = 1, 4 do redis.call(\'PEXPIREAT\', KEYS[i], tonumber(latest[2]) + 30000) end', + 'redis.call(\'HSET\', KEYS[5], \'record\', ARGV[7], \'fingerprint\', ARGV[6], \'state\', \'queued\', \'queueDeadlineAtMs\', ARGV[3])', + 'redis.call(\'PEXPIRE\', KEYS[5], ARGV[8])', + 'redis.call(\'ZADD\', KEYS[6], ARGV[2], KEYS[5])', + 'return 1', + ].join('\n'), 6, ...this.keys(args.workerId), args.key, args.activeKey, + args.id, Date.now(), args.deadlineAtMs, this.capacity, args.workspaceId ?? '', + args.fingerprint, args.record, args.retentionMs)); + if (result === 2) return 'existing'; + if (result === -1) return 'conflict'; + return result === 1 ? 'accepted' : 'full'; + } + + async position(workerId: string, id: string): Promise { + const rank = await this.redis.zrank(this.keys(workerId)[0], id); + return rank == null ? undefined : rank + 1; + } + async isHead(workerId: string, id: string): Promise { const [order, deadlines, , workspaces] = this.keys(workerId); return ( diff --git a/service/src/bridge/concurrent-store.test.ts b/service/src/bridge/concurrent-store.test.ts index 33f3d082..6bdfde88 100644 --- a/service/src/bridge/concurrent-store.test.ts +++ b/service/src/bridge/concurrent-store.test.ts @@ -357,6 +357,51 @@ test('same-root work waits while another root progresses', async () => { await Promise.all([nextA, b]); }); +test('a long same-root queue allowance does not extend slot or assignment TTLs', async () => { + await register(); + const active = dispatch('a'); + const activeAssignment = await store.lease( + workerId, incarnationId, 1000, undefined, undefined, 0, + ); + const controller = new AbortController(); + const queued = store.dispatchWorkspaceTool({ + workerId, + signal: controller.signal, + deadlineAtMs: Date.now() + 300_000, + executionTimeoutMs: 305_000, + request: { protocolVersion: 1, operation: 'read_file', workspaceId: 'a', path: 'second.txt' }, + }); + void queued.catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 150)); + await settle(activeAssignment!); + await active; + + const assignment = await store.lease( + workerId, incarnationId, 1000, undefined, undefined, 0, + ); + if (assignment == null) { + controller.abort(); + await queued.catch(() => undefined); + throw new Error('Queued request was not leased'); + } + expect(assignment.request).toMatchObject({ path: 'second.txt' }); + try { + const expiresAtMs = Date.parse(assignment.expiresAt); + const slotExpiresAtMs = Number(await redis.hget( + `codeapi:bridge:v1:worker:${workerId}:workspace-slots`, 'e:0', + )); + expect(slotExpiresAtMs).toBeGreaterThan(expiresAtMs + 28_000); + expect(slotExpiresAtMs).toBeLessThan(expiresAtMs + 32_000); + const assignmentTtlMs = await redis.pttl(`codeapi:bridge:v1:assignment:${assignment.assignmentId}`); + const remainingMs = expiresAtMs - Date.now(); + expect(assignmentTtlMs).toBeGreaterThan(remainingMs + 28_000); + expect(assignmentTtlMs).toBeLessThan(remainingMs + 32_000); + } finally { + await settle(assignment); + await queued; + } +}); + test('conversation worktrees on one source use independent capacity lanes', async () => { await register(); const firstId = 'a'.repeat(64); diff --git a/service/src/bridge/index.ts b/service/src/bridge/index.ts index 16857fad..f7ba03b4 100644 --- a/service/src/bridge/index.ts +++ b/service/src/bridge/index.ts @@ -10,8 +10,25 @@ export const bridgeStore = new RedisBridgeStore( undefined, undefined, env.BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, + env.BRIDGE_LANE_GIT, +); +export const bridgePairings = new RedisBridgePairingStore( + connection, + undefined, + undefined, + undefined, + undefined, + env.BRIDGE_RECOVERY_SERVER_ID + ? { + serverId: env.BRIDGE_RECOVERY_SERVER_ID, + enrollmentTtlSeconds: env.BRIDGE_ENROLLMENT_TTL_SECONDS, + challengeTtlSeconds: env.BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS, + maxChallengesPerMinute: env.BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE, + maxAttemptsPerMinute: env.BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE, + maxUntrustedRequestsPerMinute: env.BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE, + } + : undefined, ); -export const bridgePairings = new RedisBridgePairingStore(connection); export default createBridgeRouter({ enabled: isBridgeEnabled(), diff --git a/service/src/bridge/lane-git.test.ts b/service/src/bridge/lane-git.test.ts new file mode 100644 index 00000000..604f2070 --- /dev/null +++ b/service/src/bridge/lane-git.test.ts @@ -0,0 +1,251 @@ +import { afterEach, describe, expect, spyOn, test } from 'bun:test'; +import RedisMock from 'ioredis-mock'; + +import type Redis from 'ioredis'; + +import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { laneGitPolicy } from './lane-git'; +import { RedisBridgeStore } from './store'; + +const redis = new RedisMock() as unknown as Redis; +const incarnationId = 'incarnation-00000001'; +const head = 'a'.repeat(40); + +afterEach(async () => { + await redis.flushall(); +}); + +function capabilities(advertise: boolean) { + return { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file' as const, 'execute_command' as const], + workspaces: [{ id: 'primary' }], + ...(advertise ? { commandResultFeatures: ['lane_git' as const] } : {}), + }, + }; +} + +const request = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'execute_command' as const, + workspaceId: 'primary', + command: 'git checkout -b agent', +}; + +function commandResult(extra: Record = {}) { + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'execute_command' as const, + workspaceId: 'primary', + exitCode: 0, + stdout: '', + stderr: '', + truncated: false, + timedOut: false, + ...extra, + }; +} + +/** Registers a worker, runs one command through the store, and returns what the caller receives. */ +async function run(opts: { enabled: boolean; advertise: boolean; workerResult: Record }) { + const store = new RedisBridgeStore(redis, undefined, undefined, 1, opts.enabled); + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'lane-worker', + incarnationId, + capabilities: capabilities(opts.advertise), + }); + const completion = store.dispatchWorkspaceTool({ + workerId: 'lane-worker', + tenantId: 'tenant-1', + request, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + const assignment = await store.lease('lane-worker', incarnationId, 1_000); + await store.settle('lane-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: opts.workerResult as never, + }); + // Read the durable copy before the dispatcher cleans it up; it must exist, or the + // "not stored" assertions below would pass vacuously. + const stored = await redis.get(`codeapi:bridge:v1:assignment:${assignment?.assignmentId}:settlement`); + expect(stored).not.toBeNull(); + const settlement = await completion; + return { settlement, stored: stored as string }; +} + +describe('laneGit on command settlements', () => { + test('a valid laneGit from an advertising worker reaches the caller unchanged', async () => { + const laneGit = { branch: 'librechat/conversation-abcd1234-0123', head }; + const { settlement } = await run({ enabled: true, advertise: true, workerResult: commandResult({ laneGit }) }); + expect(settlement.status).toBe('fulfilled'); + expect((settlement as { result: { laneGit?: unknown } }).result.laneGit).toEqual(laneGit); + }); + + test('null branch and head (detached, no commit) are valid and preserved', async () => { + const laneGit = { branch: null, head: null }; + const { settlement } = await run({ enabled: true, advertise: true, workerResult: commandResult({ laneGit }) }); + expect((settlement as { result: { laneGit?: unknown } }).result.laneGit).toEqual(laneGit); + }); + + test('a 64 character head is accepted', async () => { + const laneGit = { branch: 'main', head: 'b'.repeat(64) }; + const { settlement } = await run({ enabled: true, advertise: true, workerResult: commandResult({ laneGit }) }); + expect((settlement as { result: { laneGit?: unknown } }).result.laneGit).toEqual(laneGit); + }); + + const invalid: Array<[string, unknown]> = [ + ['extra keys', { branch: 'main', head, path: '/srv/repo' }], + ['an over-long branch', { branch: 'x'.repeat(257), head }], + ['an empty branch', { branch: '', head }], + ['a control character in the branch', { branch: 'bad\nname', head }], + ['a bidi override in the branch', { branch: 'feature/\u202eabc', head }], + ['a zero-width character in the branch', { branch: 'fix\u200bme', head }], + ['uppercase hex', { branch: 'main', head: 'A'.repeat(40) }], + ['a short head', { branch: 'main', head: 'a'.repeat(39) }], + ['non-hex head', { branch: 'main', head: 'g'.repeat(40) }], + ['a missing head', { branch: 'main' }], + ['a non-object', 'main'], + ['an array', [null, null]], + ]; + for (const [name, laneGit] of invalid) { + test(`laneGit with ${name} is dropped and the command still completes`, async () => { + const { settlement, stored } = await run({ enabled: true, advertise: true, workerResult: commandResult({ laneGit }) }); + expect(settlement.status).toBe('fulfilled'); + const result = (settlement as unknown as { result: Record }).result; + expect('laneGit' in result).toBe(false); + expect(result.exitCode).toBe(0); + expect(stored).not.toContain('laneGit'); + expect(stored).not.toContain('/srv/repo'); + }); + } + + test('laneGit from a worker that did not advertise lane_git is stripped', async () => { + const { settlement, stored } = await run({ + enabled: true, + advertise: false, + workerResult: commandResult({ laneGit: { branch: 'main', head } }), + }); + expect(settlement.status).toBe('fulfilled'); + expect('laneGit' in (settlement as { result: object }).result).toBe(false); + expect(stored).not.toContain('laneGit'); + }); + + test('laneGit is stripped when the deployment setting is off, even from an advertising worker', async () => { + const { settlement, stored } = await run({ + enabled: false, + advertise: true, + workerResult: commandResult({ laneGit: { branch: 'main', head } }), + }); + expect(settlement.status).toBe('fulfilled'); + expect('laneGit' in (settlement as { result: object }).result).toBe(false); + expect(stored).not.toContain('laneGit'); + }); + + test('an old worker without the field still registers and runs commands, with the setting on or off', async () => { + for (const enabled of [false, true]) { + const { settlement } = await run({ enabled, advertise: false, workerResult: commandResult() }); + expect(settlement.status).toBe('fulfilled'); + expect((settlement as { result: { exitCode: number } }).result.exitCode).toBe(0); + await redis.flushall(); + } + }); + + test('the policy never reports the value, only a fixed reason', () => { + const result = commandResult({ laneGit: { branch: 'secret-branch', head, path: '/srv/repo' } }); + const outcome = laneGitPolicy({ result, enabled: true, advertised: true }); + expect(outcome.dropped).toBe('invalid'); + expect(JSON.stringify(outcome.dropped)).not.toContain('secret-branch'); + }); +}); + +describe('retried settlements across replicas that disagree on the setting', () => { + // A replica commits a settlement and its response is lost; the worker retries the same body + // against another replica mid-rollout. The retry must be recognized as the same settlement. + for (const [first, second] of [ + [true, false], + [false, true], + ] as const) { + test(`commit with the setting ${first ? 'on' : 'off'}, retry on a replica with it ${second ? 'on' : 'off'}`, async () => { + const a = new RedisBridgeStore(redis, undefined, undefined, 1, first); + const b = new RedisBridgeStore(redis, undefined, undefined, 1, second); + await a.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'lane-worker', + incarnationId, + capabilities: capabilities(true), + }); + const completion = a.dispatchWorkspaceTool({ + workerId: 'lane-worker', + tenantId: 'tenant-1', + request, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + completion.catch(() => undefined); + const assignment = await a.lease('lane-worker', incarnationId, 1_000); + const settlement = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled' as const, + result: commandResult({ laneGit: { branch: 'main', head } }) as never, + }; + await a.settle('lane-worker', assignment?.assignmentId ?? '', settlement); + await expect(b.settle('lane-worker', assignment?.assignmentId ?? '', settlement)).resolves.toBeUndefined(); + await completion; + }); + } +}); + +describe('Redis reads while settling', () => { + async function readsDuring(withLaneGit: boolean): Promise { + const store = new RedisBridgeStore(redis, undefined, undefined, 1, true); + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'lane-worker', + incarnationId, + capabilities: capabilities(true), + }); + const completion = store.dispatchWorkspaceTool({ + workerId: 'lane-worker', + tenantId: 'tenant-1', + request, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + completion.catch(() => undefined); + const assignment = await store.lease('lane-worker', incarnationId, 1_000); + const spy = spyOn(redis, 'get'); + await store.settle('lane-worker', assignment?.assignmentId ?? '', { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment?.generation ?? 0, + leaseToken: assignment?.leaseToken ?? '', + incarnationId, + status: 'fulfilled', + result: commandResult(withLaneGit ? { laneGit: { branch: 'main', head } } : {}) as never, + }); + const calls = spy.mock.calls.length; + spy.mockRestore(); + await completion; + return calls; + } + + test('a settlement with laneGit costs no more Redis reads than one without', async () => { + const without = await readsDuring(false); + await redis.flushall(); + const withIt = await readsDuring(true); + expect(without).toBeGreaterThan(0); + expect(withIt).toBe(without); + }); +}); diff --git a/service/src/bridge/lane-git.ts b/service/src/bridge/lane-git.ts new file mode 100644 index 00000000..69ddf7b9 --- /dev/null +++ b/service/src/bridge/lane-git.ts @@ -0,0 +1,36 @@ +import { isWorkspaceLaneGit } from '../../../packages/code/src/protocol'; + +export type LaneGitDropReason = 'disabled' | 'not_advertised' | 'invalid'; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +/** Whether a fulfilled command result carries a `laneGit` key at all. */ +export function hasLaneGit(result: unknown): boolean { + return isRecord(result) && 'laneGit' in result; +} + +/** + * Decide what happens to the `laneGit` on an execute_command result. It is kept, + * unchanged, only when the deployment enables the feature, the worker advertised + * `lane_git`, and the value is exactly `{ branch, head }` within bounds. Anything + * else is dropped; the command result itself is never failed over it. The reason + * is a fixed label, never the value. + */ +export function laneGitPolicy(args: { + result: Record; + enabled: boolean; + advertised: boolean; +}): { result: Record; dropped?: LaneGitDropReason } { + const reason: LaneGitDropReason | undefined = !args.enabled + ? 'disabled' + : !args.advertised + ? 'not_advertised' + : isWorkspaceLaneGit(args.result.laneGit) + ? undefined + : 'invalid'; + if (reason === undefined) return { result: args.result }; + const { laneGit: _laneGit, ...rest } = args.result; + return { result: rest, dropped: reason }; +} diff --git a/service/src/bridge/linked-worktree.test.ts b/service/src/bridge/linked-worktree.test.ts new file mode 100644 index 00000000..5a286726 --- /dev/null +++ b/service/src/bridge/linked-worktree.test.ts @@ -0,0 +1,197 @@ +import { createHash, randomUUID } from 'node:crypto'; +import { expect, test } from 'bun:test'; +import Redis from 'ioredis'; +import { RedisBridgeStore } from './store'; +import type { CodeBridgeAssignment } from './store'; +import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; + +const redisUrl = process.env.BRIDGE_TEST_REDIS_URL; +const incarnationId = 'lane-incarnation'; + +function fenceKey(workerId: string, fence: string): string { + const hash = createHash('sha256').update(fence).digest('hex'); + return `codeapi:bridge:v1:worker:${encodeURIComponent(workerId)}:workspace:${hash}:quarantined`; +} + +async function withLaneWorker( + run: (context: { + store: RedisBridgeStore; + redis: Redis; + workerId: string; + dispatch: ( + request: Partial, + budgetMs?: number, + ) => Promise; + lease: (slot: number) => Promise; + settle: (assignment: CodeBridgeAssignment) => Promise; + }) => Promise, + scopes = true, +): Promise { + const redis = new Redis(redisUrl!); + const store = new RedisBridgeStore(redis, 60, 1000, 3); + const workerId = `lanes-${randomUUID()}`; + const controller = new AbortController(); + const pending: Promise[] = []; + try { + const generation = await store.register({ + protocolVersion: 1, + workerId, + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceLeaseSlots: 3, + requiresReadyConfirmation: true, + workspaceTools: { + protocolVersion: 1, + operations: ['read_file'], + workspaces: [ + { + id: 'repo', + ...(scopes ? { workspaceScopes: ['git_linked_worktree' as const] } : {}), + }, + ], + }, + }, + }); + await store.confirmReady(workerId, incarnationId, generation); + await run({ + store, + redis, + workerId, + dispatch(request, budgetMs = 3000) { + const result = store.dispatchWorkspaceTool({ + workerId, + signal: controller.signal, + deadlineAtMs: Date.now() + budgetMs, + executionTimeoutMs: 5000, + request: { + protocolVersion: 1, + operation: 'read_file', + workspaceId: 'repo', + path: 'probe', + ...request, + } as WorkspaceToolRequest, + }); + void result.catch(() => undefined); + pending.push(result); + return result; + }, + lease: (slot) => store.lease(workerId, incarnationId, 1000, undefined, undefined, slot), + async settle(assignment) { + await store.acknowledgeLease( + workerId, + incarnationId, + assignment.assignmentId, + assignment.generation, + assignment.leaseToken, + ); + await store.settle(workerId, assignment.assignmentId, { + protocolVersion: 1, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected', + error: 'probe complete', + }); + await store.confirmWorkspaceCleanup(workerId, assignment.assignmentId, { + protocolVersion: 1, + incarnationId, + generation: assignment.generation, + leaseToken: assignment.leaseToken, + status: 'rejected', + error: 'local cleanup confirmed', + }); + }, + }); + } finally { + controller.abort(); + await Promise.allSettled(pending); + await redis.quit(); + } +} + +test.skipIf(!redisUrl)( + 'sibling linked-worktree lanes run together while their checkout waits for both', + async () => { + await withLaneWorker(async ({ dispatch, lease, settle }) => { + const laneA = dispatch({ worktree: 'task-a' }); + const laneB = dispatch({ worktree: 'task-b' }); + const first = await lease(0); + const second = await lease(1); + expect( + [first?.request, second?.request].map( + (request) => (request as WorkspaceToolRequest).worktree, + ).sort(), + ).toEqual(['task-a', 'task-b']); + + const checkout = dispatch({}, 400); + expect(await lease(2)).toBeUndefined(); + await expect(checkout).rejects.toMatchObject({ code: 'WORKSPACE_QUEUE_TIMEOUT' }); + + await settle(first!); + await settle(second!); + await Promise.allSettled([laneA, laneB]); + }); + }, +); + +test.skipIf(!redisUrl)('a busy checkout holds every lane beneath it', async () => { + await withLaneWorker(async ({ dispatch, lease, settle }) => { + const checkout = dispatch({}); + const held = await lease(0); + expect((held?.request as WorkspaceToolRequest).worktree).toBeUndefined(); + + const lane = dispatch({ worktree: 'task-a' }); + expect(await lease(1)).toBeUndefined(); + + await settle(held!); + await Promise.allSettled([checkout]); + const admitted = await lease(0); + expect((admitted?.request as WorkspaceToolRequest).worktree).toBe('task-a'); + await settle(admitted!); + await Promise.allSettled([lane]); + }); +}); + +test.skipIf(!redisUrl)('a lane may not start while its checkout is quarantined', async () => { + await withLaneWorker(async ({ redis, workerId, dispatch }) => { + await redis.set(fenceKey(workerId, 'native-workspace:repo'), 'quarantined:earlier'); + await expect(dispatch({ worktree: 'task-a' })).rejects.toMatchObject({ + code: 'WORKSPACE_QUARANTINED', + }); + expect(await redis.exists(fenceKey(workerId, 'native-workspace:\0linked-worktree\0repo\0task-a'))).toBe(0); + }); +}); + +test.skipIf(!redisUrl)('a checkout is refused while a lane beneath it keeps a stuck fence', async () => { + await withLaneWorker(async ({ redis, workerId, dispatch, lease, settle }) => { + const settleNext = async (request: Partial) => { + const pending = dispatch(request); + const assignment = await lease(0); + expect((assignment?.request as WorkspaceToolRequest).worktree).toBe(request.worktree); + await settle(assignment!); + await Promise.allSettled([pending]); + }; + await settleNext({ worktree: 'task-a' }); + await settleNext({}); + + // A lane whose cleanup was never acknowledged keeps its fence after its slot is released. + await settleNext({ worktree: 'task-a' }); + await redis.set( + fenceKey(workerId, 'native-workspace:\0linked-worktree\0repo\0task-a'), + 'quarantined:cleanup-unacknowledged', + ); + + await expect(dispatch({})).rejects.toMatchObject({ code: 'WORKSPACE_QUARANTINED' }); + }); +}); + +test.skipIf(!redisUrl)('a lane is refused unless the worker advertises linked-worktree scopes', async () => { + await withLaneWorker(async ({ dispatch }) => { + await expect(dispatch({ worktree: 'task-a' })).rejects.toMatchObject({ + code: 'WORKER_MISMATCH', + }); + }, false); +}); diff --git a/service/src/bridge/pairing.ts b/service/src/bridge/pairing.ts index 9eb6d76c..b8a32162 100644 --- a/service/src/bridge/pairing.ts +++ b/service/src/bridge/pairing.ts @@ -6,13 +6,26 @@ import { import type Redis from 'ioredis'; -import { verifyBridgeRequest } from '../../../packages/code/src/identity'; +import { + verifyBridgeRecovery, + verifyBridgeRecoveryStart, + verifyBridgeRequest, +} from '../../../packages/code/src/identity'; +import type { + BridgeRecoveryProofInput, + BridgeRecoveryStartProofInput, +} from '../../../packages/code/src/identity'; const PREFIX = 'codeapi:bridge:v1'; const DEFAULT_PAIRING_TTL_SECONDS = 10 * 60; const DEFAULT_CREDENTIAL_TTL_SECONDS = 15 * 60; const PROOF_NONCE_TTL_SECONDS = 2 * 60; const PROOF_CLOCK_SKEW_MS = 60_000; +const DEFAULT_CHALLENGE_TTL_SECONDS = 60; +const DEFAULT_CHALLENGES_PER_MINUTE = 12; +const DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE = 30; +const DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE = 240; +const RECOVERY_START_NONCE_TTL_SECONDS = 3 * 60; const LEGACY_SCAN_CLAIM_TTL_MS = 5_000; const LEGACY_SCAN_POLL_INTERVAL_MS = 25; const LEGACY_SCAN_PENDING = 'pending'; @@ -41,6 +54,7 @@ elseif (generation or '0') ~= ARGV[2] then redis.call('DEL', KEYS[1]) return 0 end +if ARGV[7] ~= '' and (generation or '0') ~= ARGV[9] then return 0 end redis.call('DEL', KEYS[1]) if redis.call('GET', KEYS[5]) == KEYS[1] then redis.call('DEL', KEYS[5]) @@ -48,9 +62,26 @@ end redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[4]) redis.call('SET', KEYS[4], ARGV[5], 'EX', ARGV[4]) redis.call('SET', KEYS[6], ARGV[6], 'EX', ARGV[4]) +if ARGV[7] ~= '' then + if tonumber(ARGV[8]) > 0 then + redis.call('SET', KEYS[7], ARGV[7], 'EX', ARGV[8]) + else + redis.call('SET', KEYS[7], ARGV[7]) + end + redis.call('SET', KEYS[8], ARGV[10]) +else + redis.call('DEL', KEYS[7], KEYS[8]) +end return 1 `; const ROTATE_CREDENTIAL_SCRIPT = ` +if ARGV[6] ~= '' then + if redis.call('GET', KEYS[5]) ~= ARGV[6] or (redis.call('GET', KEYS[6]) or '0') ~= ARGV[7] or redis.call('GET', KEYS[7]) ~= ARGV[8] then + return 0 + end +elseif redis.call('EXISTS', KEYS[7]) == 1 then + return 0 +end local activeDigest = redis.call('GET', KEYS[1]) local previous = redis.call('GET', KEYS[2]) if not activeDigest or not previous then @@ -82,12 +113,45 @@ if credential then redis.call('DEL', KEYS[3]) redis.call('DEL', ARGV[1] .. credential) end -redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7]) +redis.call('DEL', KEYS[1], KEYS[3], KEYS[4], KEYS[5], KEYS[6], KEYS[7], KEYS[8]) if activeIncarnation then redis.call('SET', ARGV[2] .. activeIncarnation .. ':fenced', '1') end return 1 `; +const CREATE_RECOVERY_CHALLENGE_SCRIPT = ` +if redis.call('GET', KEYS[1]) ~= ARGV[1] or (redis.call('GET', KEYS[4]) or '0') ~= ARGV[4] or redis.call('GET', KEYS[5]) ~= ARGV[6] then + return 0 +end +if redis.call('EXISTS', KEYS[6]) == 1 then return -2 end +local count = redis.call('INCR', KEYS[2]) +if count == 1 then redis.call('EXPIRE', KEYS[2], 60) end +if count > tonumber(ARGV[2]) then return -1 end +if redis.call('SET', KEYS[3], ARGV[3], 'EX', ARGV[5], 'NX') ~= 'OK' then return 0 end +redis.call('SET', KEYS[6], '1', 'EX', ARGV[7]) +return 1 +`; +const LIMIT_RECOVERY_ATTEMPTS_SCRIPT = ` +local count = redis.call('INCR', KEYS[1]) +if count == 1 then redis.call('EXPIRE', KEYS[1], 60) end +if count > tonumber(ARGV[1]) then return 0 end +return 1 +`; +const COMPLETE_RECOVERY_CHALLENGE_SCRIPT = ` +if redis.call('GET', KEYS[1]) ~= ARGV[1] or redis.call('GET', KEYS[2]) ~= ARGV[2] or (redis.call('GET', KEYS[6]) or '0') ~= ARGV[3] or redis.call('GET', KEYS[7]) ~= ARGV[8] then + return 0 +end +local stableIdentity = redis.call('GET', KEYS[5]) +if stableIdentity and stableIdentity ~= ARGV[4] then return 0 end +local count = redis.call('INCR', KEYS[8]) +if count == 1 then redis.call('EXPIRE', KEYS[8], 60) end +if count > tonumber(ARGV[9]) then return -1 end +redis.call('DEL', KEYS[2]) +redis.call('SET', KEYS[3], ARGV[5], 'EX', ARGV[6]) +redis.call('SET', KEYS[4], ARGV[7], 'EX', ARGV[6]) +redis.call('SET', KEYS[5], ARGV[4], 'EX', ARGV[6]) +return 1 +`; const RELEASE_LEGACY_SCAN_CLAIM_SCRIPT = ` if redis.call('GET', KEYS[1]) == ARGV[1] then return redis.call('DEL', KEYS[1]) @@ -147,8 +211,34 @@ interface StoredCredential { publicKey: string; expiresAt: string; binding?: BridgeWorkerBinding; + /** Present only for credentials backed by durable machine enrollment. */ + enrollmentGeneration?: string; +} + +interface StoredEnrollment { + workerId: string; + serverId: string; + identityId: string; + generation: string; + pairingGeneration: number; + publicKey: string; + binding?: BridgeWorkerBinding; +} + +export interface BridgeRecoveryOptions { + /** Stable HTTPS origin of this Code API deployment, identical on every replica. */ + serverId: string; + /** Zero (the default) keeps enrollment until explicit revocation. */ + enrollmentTtlSeconds?: number; + challengeTtlSeconds?: number; + maxChallengesPerMinute?: number; + maxAttemptsPerMinute?: number; + /** Shared per-connection-peer cap, separate from machine-signed budgets. */ + maxUntrustedRequestsPerMinute?: number; } +export type BridgeRecoveryChallenge = BridgeRecoveryProofInput; + export interface BridgePairing { workerId: string; code: string; @@ -168,7 +258,10 @@ export class BridgePairingError extends Error { | 'PUBLIC_KEY_INVALID' | 'CREDENTIAL_INVALID' | 'PROOF_INVALID' - | 'PROOF_REPLAYED', + | 'PROOF_REPLAYED' + | 'ENROLLMENT_INVALID' + | 'CHALLENGE_INVALID' + | 'RECOVERY_RATE_LIMITED', message: string, ) { super(message); @@ -196,6 +289,39 @@ function workerStableIdentityKey(workerId: string): string { return `${PREFIX}:stable-identity:${workerId}`; } +function workerEnrollmentKey(workerId: string): string { + return `${PREFIX}:enrollment:${workerId}`; +} + +function workerEnrollmentRequiredKey(workerId: string): string { + return `${PREFIX}:enrollment-required:${workerId}`; +} + +function recoveryChallengeKey(challenge: string): string { + return `${PREFIX}:recovery:challenge:${digest(challenge)}`; +} + +function recoveryStartNonceKey(workerId: string, nonce: string): string { + return `${PREFIX}:recovery:nonce:${workerId}:${digest(nonce)}`; +} + +function recoveryChallengeAttemptKey(challenge: string): string { + return `${PREFIX}:recovery:attempt:${digest(challenge)}`; +} + +function untrustedRecoveryRateKey(peer: string, operation: 'challenge' | 'recover'): string { + // Never partition by a caller-supplied worker ID or an untrusted forwarded IP. + return `${PREFIX}:recovery:untrusted:${operation}:${digest(peer)}`; +} + +function recoveryRateKey( + workerId: string, + enrollmentGeneration: string, + operation: 'start' | 'complete', +): string { + return `${PREFIX}:recovery:rate:${operation}:${workerId}:${enrollmentGeneration}`; +} + function workerPairingGenerationKey(workerId: string): string { return `${PREFIX}:pairing-generation:${workerId}`; } @@ -236,8 +362,120 @@ export class RedisBridgePairingStore { private readonly credentialTtlSeconds = DEFAULT_CREDENTIAL_TTL_SECONDS, private readonly legacyScanClaimTtlMs = LEGACY_SCAN_CLAIM_TTL_MS, private readonly rollbackEpoch = - process.env.CODEAPI_BRIDGE_PAIRING_ROLLBACK_EPOCH?.trim() ?? '', - ) {} + process.env.CODEAPI_BRIDGE_PAIRING_ROLLBACK_EPOCH?.trim() ?? '', + private readonly recovery?: BridgeRecoveryOptions, + ) { + if (recovery == null) return; + let server: URL; + try { + server = new URL(recovery.serverId); + } catch { + throw new Error('Bridge recovery requires a stable HTTPS server origin'); + } + if ( + server.protocol !== 'https:' || + server.username !== '' || + server.password !== '' || + server.pathname !== '/' || + server.search !== '' || + server.hash !== '' || + server.origin !== recovery.serverId + ) { + throw new Error('Bridge recovery requires a stable HTTPS server origin'); + } + for (const [value, max] of [ + [recovery.enrollmentTtlSeconds ?? 0, 10 * 365 * 24 * 3600], + [recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS, 300], + [recovery.maxChallengesPerMinute ?? DEFAULT_CHALLENGES_PER_MINUTE, 120], + [recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE, 120], + [recovery.maxUntrustedRequestsPerMinute ?? DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE, 1200], + ]) { + if (!Number.isSafeInteger(value) || value < 0 || value > max) { + throw new RangeError('Invalid bridge recovery lifetime or rate limit'); + } + } + if ( + (recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS) === 0 || + (recovery.maxChallengesPerMinute ?? DEFAULT_CHALLENGES_PER_MINUTE) === 0 || + (recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE) === 0 || + (recovery.maxUntrustedRequestsPerMinute ?? DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE) === 0 + ) { + throw new RangeError('Bridge recovery challenge lifetime and rate limits must be positive'); + } + } + + get recoveryEnabled(): boolean { + return this.recovery != null; + } + + async limitUntrustedRecovery( + peer: string, + operation: 'challenge' | 'recover', + ): Promise { + if (this.recovery == null) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine recovery is disabled'); + } + const allowed = await this.redis.eval( + LIMIT_RECOVERY_ATTEMPTS_SCRIPT, + 1, + untrustedRecoveryRateKey(peer, operation), + String(this.recovery.maxUntrustedRequestsPerMinute ?? DEFAULT_UNTRUSTED_RECOVERY_REQUESTS_PER_MINUTE), + ); + if (allowed !== 1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many recovery requests from this peer'); + } + } + + private checkedEnrollment( + workerId: string, + raw: string | null, + pairingGeneration: string | null, + requiredGeneration: string | null, + ): StoredEnrollment { + let parsed: unknown; + try { + parsed = raw == null ? undefined : JSON.parse(raw); + } catch { + parsed = undefined; + } + const enrollment = ( + typeof parsed === 'object' && parsed != null && !Array.isArray(parsed) + ? parsed + : {} + ) as Partial; + if ( + this.recovery == null || + enrollment.workerId !== workerId || + enrollment.serverId !== this.recovery.serverId || + typeof enrollment.identityId !== 'string' || + !/^[A-Za-z0-9_-]{24}$/.test(enrollment.identityId) || + typeof enrollment.generation !== 'string' || + !/^[A-Za-z0-9_-]{24}$/.test(enrollment.generation) || + enrollment.generation !== requiredGeneration || + typeof enrollment.publicKey !== 'string' || + !validEd25519PublicKey(enrollment.publicKey) || + !Number.isSafeInteger(enrollment.pairingGeneration) || + String(enrollment.pairingGeneration) !== (pairingGeneration ?? '0') + ) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine enrollment is unavailable or revoked'); + } + return enrollment as StoredEnrollment; + } + + private checkEnrolledCredential( + credential: StoredCredential, + enrollment: StoredEnrollment, + ): void { + if ( + credential.workerId !== enrollment.workerId || + credential.identityId !== enrollment.identityId || + credential.publicKey !== enrollment.publicKey || + credential.enrollmentGeneration !== enrollment.generation || + JSON.stringify(credential.binding ?? null) !== JSON.stringify(enrollment.binding ?? null) + ) { + throw new BridgePairingError('CREDENTIAL_INVALID', 'Worker credential does not match its machine enrollment'); + } + } async issue( workerId: string, @@ -314,28 +552,49 @@ export class RedisBridgePairingStore { Date.now() + this.credentialTtlSeconds * 1000, ).toISOString(); const identityId = randomBytes(18).toString('base64url'); + const pairingGeneration = pairing.generation ?? Number( + (await this.redis.get(workerPairingGenerationKey(args.workerId))) ?? '0', + ); + const enrollment: StoredEnrollment | undefined = this.recovery == null + ? undefined + : { + workerId: args.workerId, + serverId: this.recovery.serverId, + identityId, + generation: randomBytes(18).toString('base64url'), + pairingGeneration, + publicKey: args.publicKey, + binding: pairing.binding, + }; const stored: StoredCredential = { workerId: args.workerId, identityId, publicKey: args.publicKey, expiresAt, binding: pairing.binding, + ...(enrollment == null ? {} : { enrollmentGeneration: enrollment.generation }), }; const accepted = await this.redis.eval( REDEEM_PAIRING_SCRIPT, - 6, + 8, codeKey, workerPairingGenerationKey(pairing.workerId), credentialDigestKey(credentialDigest), workerIdentityKey(args.workerId), workerPairingIndexKey(args.workerId), workerStableIdentityKey(args.workerId), + workerEnrollmentKey(args.workerId), + workerEnrollmentRequiredKey(args.workerId), raw, pairing.generation == null ? '' : String(pairing.generation), JSON.stringify(stored), String(this.credentialTtlSeconds), credentialDigest, identityId, + enrollment == null ? '' : JSON.stringify(enrollment), + String(this.recovery?.enrollmentTtlSeconds ?? 0), + String(pairingGeneration), + enrollment?.generation ?? '', ); if (accepted !== 1) { throw new BridgePairingError( @@ -374,10 +633,12 @@ export class RedisBridgePairingStore { ); } const credentialDigest = digest(args.credential); - const [raw, activeDigest, pairingGeneration] = await this.redis.mget( + const [raw, activeDigest, pairingGeneration, enrollmentRaw, requiredGeneration] = await this.redis.mget( credentialDigestKey(credentialDigest), workerIdentityKey(args.workerId), workerPairingGenerationKey(args.workerId), + workerEnrollmentKey(args.workerId), + workerEnrollmentRequiredKey(args.workerId), ); if (raw == null || activeDigest == null) { throw new BridgePairingError( @@ -386,6 +647,12 @@ export class RedisBridgePairingStore { ); } const stored = JSON.parse(raw) as StoredCredential; + if (stored.enrollmentGeneration != null || requiredGeneration != null || enrollmentRaw != null) { + this.checkEnrolledCredential( + stored, + this.checkedEnrollment(args.workerId, enrollmentRaw, pairingGeneration, requiredGeneration), + ); + } if (activeDigest !== credentialDigest) { const activeRaw = await this.redis.get( credentialDigestKey(activeDigest), @@ -439,6 +706,177 @@ export class RedisBridgePairingStore { }; } + async createRecoveryChallenge( + workerId: string, + request: BridgeRecoveryStartProofInput, + signature: string, + ): Promise { + if (this.recovery == null) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine recovery is disabled'); + } + const [raw, pairingGeneration, requiredGeneration] = await this.redis.mget( + workerEnrollmentKey(workerId), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + ); + const enrollment = this.checkedEnrollment(workerId, raw, pairingGeneration, requiredGeneration); + const proofTime = Date.parse(request.timestamp); + if ( + String(request.operation) !== 'credential.challenge' || + request.serverId !== enrollment.serverId || + request.workerId !== workerId || + !/^[A-Za-z0-9_-]{43}$/.test(request.nonce) || + !Number.isFinite(proofTime) || + Math.abs(Date.now() - proofTime) > PROOF_CLOCK_SKEW_MS || + !verifyBridgeRecoveryStart(enrollment.publicKey, request, signature) + ) { + throw new BridgePairingError('PROOF_INVALID', 'Machine recovery challenge proof is invalid'); + } + const challenge: BridgeRecoveryChallenge = { + operation: 'credential.recover', + serverId: enrollment.serverId, + workerId, + enrollmentGeneration: enrollment.generation, + challenge: randomBytes(32).toString('base64url'), + expiresAt: new Date( + Date.now() + (this.recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS) * 1000, + ).toISOString(), + }; + const created = await this.redis.eval( + CREATE_RECOVERY_CHALLENGE_SCRIPT, + 6, + workerEnrollmentKey(workerId), + recoveryRateKey(workerId, enrollment.generation, 'start'), + recoveryChallengeKey(challenge.challenge), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + recoveryStartNonceKey(workerId, request.nonce), + raw!, + String(this.recovery.maxChallengesPerMinute ?? DEFAULT_CHALLENGES_PER_MINUTE), + JSON.stringify(challenge), + String(enrollment.pairingGeneration), + String(this.recovery.challengeTtlSeconds ?? DEFAULT_CHALLENGE_TTL_SECONDS), + enrollment.generation, + String(RECOVERY_START_NONCE_TTL_SECONDS), + ); + if (created === -2) { + throw new BridgePairingError('PROOF_REPLAYED', 'Machine recovery challenge proof was already used'); + } + if (created === -1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many machine recovery challenges'); + } + if (created !== 1) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine enrollment is unavailable or revoked'); + } + return challenge; + } + + async recoverCredential( + workerId: string, + proof: BridgeRecoveryChallenge, + signature: string, + ): Promise { + if (this.recovery == null) { + throw new BridgePairingError('ENROLLMENT_INVALID', 'Machine recovery is disabled'); + } + const challengeKey = recoveryChallengeKey(proof.challenge); + const [enrollmentRaw, challengeRaw, pairingGeneration, requiredGeneration] = await this.redis.mget( + workerEnrollmentKey(workerId), + challengeKey, + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + ); + const enrollment = this.checkedEnrollment( + workerId, enrollmentRaw, pairingGeneration, requiredGeneration, + ); + let parsed: unknown; + try { + parsed = challengeRaw == null ? undefined : JSON.parse(challengeRaw); + } catch { + parsed = undefined; + } + const saved = ( + typeof parsed === 'object' && parsed != null && !Array.isArray(parsed) + ? parsed + : {} + ) as Partial; + if ( + saved.operation !== 'credential.recover' || + saved.serverId !== enrollment.serverId || + saved.workerId !== workerId || + saved.enrollmentGeneration !== enrollment.generation || + saved.challenge !== proof.challenge || + saved.expiresAt !== proof.expiresAt || + String(proof.operation) !== saved.operation || + saved.serverId !== proof.serverId || + saved.workerId !== proof.workerId || + saved.enrollmentGeneration !== proof.enrollmentGeneration || + typeof saved.expiresAt !== 'string' || + !Number.isFinite(Date.parse(saved.expiresAt)) || + Date.parse(saved.expiresAt) <= Date.now() + ) { + throw new BridgePairingError('CHALLENGE_INVALID', 'Machine recovery challenge is invalid or expired'); + } + // Invalid signatures can only exhaust the high-entropy challenge they know, + // never the enrolled worker's shared quota. The worker-wide counter is + // charged inside completion, after the key proof and replay checks. + const attempts = await this.redis.eval( + LIMIT_RECOVERY_ATTEMPTS_SCRIPT, + 1, + recoveryChallengeAttemptKey(proof.challenge), + String(this.recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE), + ); + if (attempts !== 1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many machine recovery attempts'); + } + if (!verifyBridgeRecovery(enrollment.publicKey, saved as BridgeRecoveryChallenge, signature)) { + throw new BridgePairingError('PROOF_INVALID', 'Machine recovery signature is invalid'); + } + + const credential = randomBytes(32).toString('base64url'); + const credentialDigest = digest(credential); + const expiresAt = new Date(Date.now() + this.credentialTtlSeconds * 1000).toISOString(); + const stored: StoredCredential = { + workerId, + identityId: enrollment.identityId, + publicKey: enrollment.publicKey, + expiresAt, + binding: enrollment.binding, + enrollmentGeneration: enrollment.generation, + }; + // The same Redis decision consumes the proof and issues the credential. + // A revoke or replacement on another replica wins by invalidating the + // enrollment/generation comparison, with no window to recreate trust. + const issued = await this.redis.eval( + COMPLETE_RECOVERY_CHALLENGE_SCRIPT, + 8, + workerEnrollmentKey(workerId), + challengeKey, + credentialDigestKey(credentialDigest), + workerIdentityKey(workerId), + workerStableIdentityKey(workerId), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + recoveryRateKey(workerId, enrollment.generation, 'complete'), + enrollmentRaw!, + challengeRaw!, + String(enrollment.pairingGeneration), + enrollment.identityId, + JSON.stringify(stored), + String(this.credentialTtlSeconds), + credentialDigest, + enrollment.generation, + String(this.recovery.maxAttemptsPerMinute ?? DEFAULT_RECOVERY_ATTEMPTS_PER_MINUTE), + ); + if (issued === -1) { + throw new BridgePairingError('RECOVERY_RATE_LIMITED', 'Too many machine recoveries'); + } + if (issued !== 1) { + throw new BridgePairingError('CHALLENGE_INVALID', 'Machine recovery challenge is invalid or expired'); + } + return { workerId, credential, expiresAt }; + } + async revoke(workerId: string): Promise { await this.removeLegacyPairings(workerId); // Fence redemption and consume the currently indexed code atomically. An @@ -446,7 +884,7 @@ export class RedisBridgePairingStore { // that linearizes afterward installs a distinct generation and code. await this.redis.eval( REVOKE_PAIRING_SCRIPT, - 7, + 8, workerPairingIndexKey(workerId), workerPairingGenerationKey(workerId), workerIdentityKey(workerId), @@ -454,6 +892,7 @@ export class RedisBridgePairingStore { `${PREFIX}:worker:${encodeURIComponent(workerId)}`, `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation`, `${PREFIX}:worker:${encodeURIComponent(workerId)}:ready`, + workerEnrollmentKey(workerId), `${PREFIX}:credential:`, `${PREFIX}:worker:${encodeURIComponent(workerId)}:incarnation:`, ); @@ -652,12 +1091,26 @@ export class RedisBridgePairingStore { ); } const previous = JSON.parse(previousRaw) as StoredCredential; + let enrollment: StoredEnrollment | undefined; + let enrollmentRaw: string | null = null; + const [raw, generation, requiredGeneration] = await this.redis.mget( + workerEnrollmentKey(workerId), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), + ); + if (previous.enrollmentGeneration != null || requiredGeneration != null || raw != null) { + enrollment = this.checkedEnrollment(workerId, raw, generation, requiredGeneration); + this.checkEnrolledCredential(previous, enrollment); + enrollmentRaw = raw; + } return await this.issueCredential( workerId, previous.publicKey, previousDigest, previous.binding, previous.identityId ?? null, + enrollment, + enrollmentRaw, ); } @@ -667,6 +1120,8 @@ export class RedisBridgePairingStore { previousDigest?: string, binding?: BridgeWorkerBinding, identityId?: string | null, + enrollment?: StoredEnrollment, + enrollmentRaw?: string | null, ): Promise { const credential = randomBytes(32).toString('base64url'); const credentialDigest = digest(credential); @@ -683,20 +1138,27 @@ export class RedisBridgePairingStore { publicKey, expiresAt, binding, + ...(enrollment == null ? {} : { enrollmentGeneration: enrollment.generation }), }; if (previousDigest !== undefined) { const rotated = await this.redis.eval( ROTATE_CREDENTIAL_SCRIPT, - 4, + 7, workerIdentityKey(workerId), credentialDigestKey(previousDigest), credentialDigestKey(credentialDigest), workerStableIdentityKey(workerId), + workerEnrollmentKey(workerId), + workerPairingGenerationKey(workerId), + workerEnrollmentRequiredKey(workerId), previousDigest, credentialDigest, JSON.stringify(stored), String(this.credentialTtlSeconds), stableIdentityId ?? '', + enrollmentRaw ?? '', + String(enrollment?.pairingGeneration ?? ''), + enrollment?.generation ?? '', ); if (rotated !== 1) { throw new BridgePairingError( diff --git a/service/src/bridge/recovery-router.test.ts b/service/src/bridge/recovery-router.test.ts new file mode 100644 index 00000000..28ea4b1e --- /dev/null +++ b/service/src/bridge/recovery-router.test.ts @@ -0,0 +1,213 @@ +import { randomBytes } from 'crypto'; +import { createServer, type Server } from 'http'; + +import { afterEach, describe, expect, test } from 'bun:test'; +import express, { json } from 'express'; +import RedisMock from 'ioredis-mock'; + +import type Redis from 'ioredis'; +import type { BridgeRecoveryChallengeRequest, BridgeRecoveryChallengeResponse } from '../../../packages/code/src/protocol'; + +import { + createBridgeIdentity, + signBridgeRecovery, + signBridgeRecoveryStart, +} from '../../../packages/code/src/identity'; +import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { RedisBridgePairingStore } from './pairing'; +import { createBridgeRouter } from './router'; +import { RedisBridgeStore } from './store'; + +const redis = new RedisMock() as unknown as Redis; +const workerId = 'http-recovery-worker'; +const serverId = 'https://code.example.test'; +const servers: Server[] = []; + +function signedStart(privateKey: string): BridgeRecoveryChallengeRequest { + const request = { + operation: 'credential.challenge' as const, + serverId, + workerId, + timestamp: new Date().toISOString(), + nonce: randomBytes(32).toString('base64url'), + }; + return { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + ...request, + signature: signBridgeRecoveryStart(privateKey, request), + }; +} + +async function startRouter(pairings: RedisBridgePairingStore): Promise { + const app = express(); + app.set('trust proxy', 1); + app.use(json()); + app.use('/v1/bridge', createBridgeRouter({ + store: new RedisBridgeStore(redis), + pairings, + authMode: 'paired', + adminToken: 'operator-only', + configuredWorkerId: workerId, + })); + const server = createServer(app); + servers.push(server); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + return `http://127.0.0.1:${address.port}/v1/bridge/workers/${workerId}/credentials`; +} + +function post( + url: string, + body: object, + headers: Record = {}, +): Promise { + return fetch(url, { + method: 'POST', + headers: { 'Content-Type': 'application/json', ...headers }, + body: JSON.stringify(body), + }); +} + +afterEach(async () => { + await Promise.all(servers.splice(0).map(server => new Promise((resolve, reject) => { + server.close(error => error ? reject(error) : resolve()); + }))); + await redis.flushall(); +}); + +describe('machine credential recovery HTTP API', () => { + test('does not expose recovery until the server identity is enabled', async () => { + const baseUrl = await startRouter(new RedisBridgePairingStore(redis)); + const response = await post(`${baseUrl}/challenge`, { protocolVersion: BRIDGE_PROTOCOL_VERSION }); + expect(response.status).toBe(404); + }); + + test('recovers using the stored key without administrator authentication and never leaks the binding', async () => { + const pairings = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId, + }); + const identity = createBridgeIdentity(); + const pairing = await pairings.issue(workerId, { + tenantId: 'tenant-one', principal: { type: 'user', id: 'owner-one' }, + }); + await pairings.redeem({ workerId, code: pairing.code, publicKey: identity.publicKey }); + const baseUrl = await startRouter(pairings); + const unsigned = await post(`${baseUrl}/challenge`, { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + }); + expect(unsigned.status).toBe(400); + const outsider = createBridgeIdentity(); + const forged = await post(`${baseUrl}/challenge`, signedStart(outsider.privateKey)); + expect(forged.status).toBe(401); + await expect(forged.json()).resolves.toMatchObject({ code: 'PROOF_INVALID' }); + + const challengeResponse = await post(`${baseUrl}/challenge`, signedStart(identity.privateKey)); + expect(challengeResponse.status).toBe(200); + const challenge = (await challengeResponse.json()) as BridgeRecoveryChallengeResponse; + expect(challenge).toMatchObject({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + serverId, + workerId, + operation: 'credential.recover', + }); + expect(JSON.stringify(challenge)).not.toMatch(/tenant-one|owner-one|privateKey/); + + const signature = signBridgeRecovery(identity.privateKey, challenge); + const rejected = await post(`${baseUrl}/recover`, { + ...challenge, + serverId: 'https://unrelated.example.test', + signature, + }); + expect(rejected.status).toBe(401); + await expect(rejected.json()).resolves.toMatchObject({ code: 'CHALLENGE_INVALID' }); + + const recovered = await post(`${baseUrl}/recover`, { ...challenge, signature }); + expect(recovered.status).toBe(200); + const credential = (await recovered.json()) as { workerId: string; credential: string }; + expect(credential.workerId).toBe(workerId); + expect(credential.credential.length).toBeGreaterThanOrEqual(32); + const replay = await post(`${baseUrl}/recover`, { ...challenge, signature }); + expect(replay.status).toBe(401); + await expect(replay.json()).resolves.toMatchObject({ code: 'CHALLENGE_INVALID' }); + }); + + test('limits forged starts before signature work across replicas despite spoofed proxy headers', async () => { + const first = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId, maxUntrustedRequestsPerMinute: 2, + }); + const second = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId, maxUntrustedRequestsPerMinute: 2, + }); + const identity = createBridgeIdentity(); + const attacker = createBridgeIdentity(); + const pairing = await first.issue(workerId); + await first.redeem({ workerId, code: pairing.code, publicKey: identity.publicKey }); + const firstUrl = await startRouter(first); + const secondUrl = await startRouter(second); + let signatureChecks = 0; + for (const store of [first, second]) { + const original = store.createRecoveryChallenge.bind(store); + store.createRecoveryChallenge = async ( + ...args + ): ReturnType => { + signatureChecks += 1; + return original(...args); + }; + } + + const forged = async (url: string, suffix: number): Promise => post( + `${url}/challenge`, signedStart(attacker.privateKey), + { 'X-Forwarded-For': `198.51.100.${suffix}` }, + ); + const firstResponse = await forged(firstUrl, 1); + const secondResponse = await forged(secondUrl, 2); + const blocked = await forged( + firstUrl.replace(`/workers/${workerId}/`, '/workers/attacker-selected-worker/'), + 3, + ); + expect([firstResponse.status, secondResponse.status, blocked.status]).toEqual([401, 401, 429]); + await expect(blocked.json()).resolves.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + expect(Number(blocked.headers.get('retry-after'))).toBeGreaterThan(0); + expect(signatureChecks).toBe(2); + expect(await redis.keys('codeapi:bridge:v1:recovery:rate:start:*')).toEqual([]); + + // Bogus completions use another untrusted bucket, not the worker's signed quota. + const fake = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'credential.recover', + serverId, + workerId, + enrollmentGeneration: 'a'.repeat(24), + challenge: randomBytes(32).toString('base64url'), + expiresAt: new Date(Date.now() + 60_000).toISOString(), + signature: 'a'.repeat(86), + }; + const rejected = await post(`${secondUrl}/recover`, fake); + expect(rejected.status).toBe(401); + expect(await redis.keys('codeapi:bridge:v1:recovery:rate:complete:*')).toEqual([]); + const recovered = await post(`${secondUrl}/recover`, fake); + expect(recovered.status).toBe(401); + const blockedCompletion = await post(`${secondUrl}/recover`, fake); + expect(blockedCompletion.status).toBe(429); + await expect(blockedCompletion.json()).resolves.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + }); + + test('limits recovery challenges across two API routers sharing Redis', async () => { + const first = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId: 'https://code.example.test', maxChallengesPerMinute: 1, + }); + const second = new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId: 'https://code.example.test', maxChallengesPerMinute: 1, + }); + const identity = createBridgeIdentity(); + const pairing = await first.issue(workerId); + await first.redeem({ workerId, code: pairing.code, publicKey: identity.publicKey }); + const baseUrl = await startRouter(second); + const initial = signedStart(identity.privateKey); + await first.createRecoveryChallenge(workerId, initial, initial.signature); + const denied = await post(`${baseUrl}/challenge`, signedStart(identity.privateKey)); + expect(denied.status).toBe(429); + await expect(denied.json()).resolves.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + }); +}); diff --git a/service/src/bridge/recovery.test.ts b/service/src/bridge/recovery.test.ts new file mode 100644 index 00000000..443e7277 --- /dev/null +++ b/service/src/bridge/recovery.test.ts @@ -0,0 +1,518 @@ +import { afterEach, describe, expect, test } from 'bun:test'; +import { createHash, randomBytes } from 'crypto'; +import RedisMock from 'ioredis-mock'; + +import type Redis from 'ioredis'; +import type { + BridgeRecoveryProofInput, + BridgeRecoveryStartProofInput, +} from '../../../packages/code/src/identity'; +import type { + BridgeRecoveryChallenge, + BridgeRecoveryOptions, + BridgeWorkerBinding, + BridgeWorkerCredential, +} from './pairing'; + +import { + createBridgeIdentity, + signBridgeRecovery, + signBridgeRecoveryStart, + signBridgeRequest, +} from '../../../packages/code/src/identity'; +import { RedisBridgePairingStore } from './pairing'; +import { RedisBridgeStore } from './store'; + +const redis = new RedisMock() as unknown as Redis; +const serverId = 'https://code.example.test'; +const workerId = 'durable-worker'; +const binding: BridgeWorkerBinding = { + tenantId: 'tenant-one', + principal: { type: 'user', id: 'owner-one' }, +}; + +function recoverableStore(options: Partial = {}): RedisBridgePairingStore { + return new RedisBridgePairingStore(redis, 600, 300, 5_000, '', { + serverId, + ...options, + }); +} + +function authorizedRequest( + privateKey: string, + credential: string, + nonce: string, +): Parameters[0] { + const proof = { + credential, + method: 'POST', + path: '/v1/bridge/workers/register', + timestamp: new Date().toISOString(), + nonce, + body: JSON.stringify({ protocolVersion: 1, workerId }), + }; + return { + ...proof, + workerId, + signature: signBridgeRequest(privateKey, proof), + }; +} + +async function enroll( + store: RedisBridgePairingStore, + publicKey: string, +): Promise { + const pairing = await store.issue(workerId, binding); + return store.redeem({ workerId, code: pairing.code, publicKey }); +} + +function recoveryStart( + privateKey: string, + nonce = randomBytes(32).toString('base64url'), + overrides: Partial = {}, +): { proof: BridgeRecoveryStartProofInput; signature: string } { + const proof: BridgeRecoveryStartProofInput = { + operation: 'credential.challenge', + serverId, + workerId, + timestamp: new Date().toISOString(), + nonce, + ...overrides, + }; + return { proof, signature: signBridgeRecoveryStart(privateKey, proof) }; +} + +async function challengeFor( + store: RedisBridgePairingStore, + privateKey: string, +): Promise { + const { proof, signature } = recoveryStart(privateKey); + return store.createRecoveryChallenge(workerId, proof, signature); +} + +async function recover( + store: RedisBridgePairingStore, + privateKey: string, +): Promise<{ challenge: BridgeRecoveryChallenge; credential: BridgeWorkerCredential }> { + const challenge = await challengeFor(store, privateKey); + const credential = await store.recoverCredential( + workerId, + challenge, + signBridgeRecovery(privateKey, challenge), + ); + return { challenge, credential }; +} + +afterEach(async () => { + await redis.flushall(); +}); + +describe('durable bridge enrollment', () => { + test('keeps legacy pairing and refresh compatible until recovery is enabled', async () => { + const legacy = new RedisBridgePairingStore(redis); + const identity = createBridgeIdentity(); + const issued = await enroll(legacy, identity.publicKey); + expect(await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`)).toBeNull(); + + const replica = recoverableStore(); + const { proof, signature } = recoveryStart(identity.privateKey); + await expect(replica.createRecoveryChallenge(workerId, proof, signature)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + const rotated = await replica.rotate(workerId); + await expect( + replica.authorize(authorizedRequest(identity.privateKey, rotated.credential, 'legacy-proof')), + ).resolves.toMatchObject({ workerId, binding }); + expect(issued.credential).not.toBe(rotated.credential); + }); + + test('recovers after access expiry and restart with the same identity and binding', async () => { + const identity = createBridgeIdentity(); + const firstReplica = recoverableStore(); + const issued = await enroll(firstReplica, identity.publicKey); + const original = await firstReplica.authorize( + authorizedRequest(identity.privateKey, issued.credential, 'before-outage'), + ); + const digest = createHash('sha256').update(issued.credential).digest('hex'); + await redis.del( + `codeapi:bridge:v1:credential:${digest}`, + `codeapi:bridge:v1:identity:${workerId}`, + `codeapi:bridge:v1:stable-identity:${workerId}`, + ); + await expect(firstReplica.rotate(workerId)).rejects.toMatchObject({ + code: 'CREDENTIAL_INVALID', + }); + + const restartedReplica = recoverableStore(); + const { challenge, credential } = await recover(restartedReplica, identity.privateKey); + expect(challenge).toMatchObject({ + operation: 'credential.recover', + serverId, + workerId, + }); + const restored = await firstReplica.authorize( + authorizedRequest(identity.privateKey, credential.credential, 'after-outage'), + ); + expect(restored).toMatchObject({ identityId: original.identityId, binding }); + expect(credential.expiresAt).toBeString(); + const rotated = await restartedReplica.rotate(workerId, restored.credentialId); + await expect(firstReplica.authorize( + authorizedRequest(identity.privateKey, rotated.credential, 'after-rotation'), + )).resolves.toMatchObject({ identityId: original.identityId, binding }); + }); + + test('requires the enrolled key and binds proofs to server, worker, generation, operation, and expiry', async () => { + const enrolled = createBridgeIdentity(); + const outsider = createBridgeIdentity(); + const store = recoverableStore(); + await enroll(store, enrolled.publicKey); + const challenge = await challengeFor(store, enrolled.privateKey); + + await expect(store.recoverCredential( + workerId, challenge, signBridgeRecovery(outsider.privateKey, challenge), + )).rejects.toMatchObject({ code: 'PROOF_INVALID' }); + for (const modified of [ + { ...challenge, serverId: 'https://other.example.test' }, + { ...challenge, workerId: 'another-worker' }, + { ...challenge, enrollmentGeneration: '0'.repeat(24) }, + { ...challenge, operation: 'credential.recover-other' as 'credential.recover' }, + { ...challenge, expiresAt: new Date(Date.now() + 120_000).toISOString() }, + ]) { + await expect(store.recoverCredential( + workerId, + modified, + signBridgeRecovery(enrolled.privateKey, modified as BridgeRecoveryProofInput), + )).rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + } + const signature = signBridgeRecovery(enrolled.privateKey, challenge); + await store.recoverCredential(workerId, challenge, signature); + await expect(store.recoverCredential(workerId, challenge, signature)) + .rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + }); + + test('rejects a signed challenge after its short-lived Redis window expires', async () => { + const identity = createBridgeIdentity(); + const store = recoverableStore({ challengeTtlSeconds: 1 }); + await enroll(store, identity.publicKey); + const challenge = await challengeFor(store, identity.privateKey); + await new Promise((resolve) => setTimeout(resolve, 1_100)); + await expect(store.recoverCredential( + workerId, challenge, signBridgeRecovery(identity.privateKey, challenge), + )).rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + }); + + test('retries a lost recovery response without changing the enrolled identity', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore(); + const originalCredential = await enroll(first, identity.publicKey); + const original = await first.authorize( + authorizedRequest(identity.privateKey, originalCredential.credential, 'before-lost-response'), + ); + await recover(first, identity.privateKey); // The caller lost this credential response. + const second = recoverableStore(); + const retried = await recover(second, identity.privateKey); + const auth = await second.authorize( + authorizedRequest(identity.privateKey, retried.credential.credential, 'after-lost-response'), + ); + expect(auth.identityId).toBe(original.identityId); + expect(auth.binding).toEqual(binding); + const enrolled = JSON.parse((await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`))!) as { + generation: string; + }; + expect(await redis.get(`codeapi:bridge:v1:enrollment-required:${workerId}`)) + .toBe(enrolled.generation); + }); + + test('rejects expired and missing enrollment even when an access credential remains live', async () => { + const identity = createBridgeIdentity(); + const store = recoverableStore({ enrollmentTtlSeconds: 1 }); + const issued = await enroll(store, identity.publicKey); + await new Promise((resolve) => setTimeout(resolve, 1_100)); + await expect(challengeFor(store, identity.privateKey)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(store.rotate(workerId)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(store.authorize( + authorizedRequest(identity.privateKey, issued.credential, 'expired-enrollment'), + )).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + }); + + test('never treats an unmarked access token as legacy after authorization state is lost', async () => { + const identity = createBridgeIdentity(); + const store = recoverableStore(); + const issued = await enroll(store, identity.publicKey); + const credentialKey = `codeapi:bridge:v1:credential:${createHash('sha256').update(issued.credential).digest('hex')}`; + const raw = JSON.parse((await redis.get(credentialKey))!) as { enrollmentGeneration?: string }; + delete raw.enrollmentGeneration; // Simulate a refresh by a pre-recovery replica. + await redis.set(credentialKey, JSON.stringify(raw), 'EX', 300); + await redis.del(`codeapi:bridge:v1:enrollment:${workerId}`); + + expect(await redis.get(`codeapi:bridge:v1:enrollment-required:${workerId}`)).not.toBeNull(); + await expect(store.authorize( + authorizedRequest(identity.privateKey, issued.credential, 'lost-authorization'), + )).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(store.rotate(workerId)).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await expect(challengeFor(store, identity.privateKey)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + await redis.set(`codeapi:bridge:v1:enrollment:${workerId}`, 'null'); + await expect(challengeFor(store, identity.privateKey)) + .rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + }); + + test('re-pairing explicitly supersedes a previous key and binds to the new principal', async () => { + const first = createBridgeIdentity(); + const second = createBridgeIdentity(); + const store = recoverableStore(); + const initial = await enroll(store, first.publicKey); + const pending = await challengeFor(store, first.privateKey); + const replacement = await store.issue(workerId, { + tenantId: 'tenant-two', principal: { type: 'user', id: 'owner-two' }, + }); + await store.redeem({ workerId, code: replacement.code, publicKey: second.publicKey }); + + await expect(store.recoverCredential( + workerId, pending, signBridgeRecovery(first.privateKey, pending), + )).rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + await expect(store.authorize( + authorizedRequest(first.privateKey, initial.credential, 'superseded-key'), + )).rejects.toMatchObject({ code: 'CREDENTIAL_INVALID' }); + const { credential } = await recover(store, second.privateKey); + await expect(store.authorize( + authorizedRequest(second.privateKey, credential.credential, 'new-owner'), + )).resolves.toMatchObject({ + binding: { tenantId: 'tenant-two', principal: { type: 'user', id: 'owner-two' } }, + }); + }); + + test('concurrent replicas consume a recovery challenge and count it only once', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ maxAttemptsPerMinute: 2 }); + const second = recoverableStore({ maxAttemptsPerMinute: 2 }); + await enroll(first, identity.publicKey); + const challenge = await challengeFor(first, identity.privateKey); + const signature = signBridgeRecovery(identity.privateKey, challenge); + const attempts = await Promise.allSettled([ + first.recoverCredential(workerId, challenge, signature), + second.recoverCredential(workerId, challenge, signature), + ]); + const issued = attempts.filter( + (result): result is PromiseFulfilledResult => + result.status === 'fulfilled', + ); + const rejected = attempts.filter( + (result): result is PromiseRejectedResult => result.status === 'rejected', + ); + expect(issued).toHaveLength(1); + expect(rejected).toHaveLength(1); + expect(rejected[0]).toMatchObject({ reason: { code: 'CHALLENGE_INVALID' } }); + const digest = createHash('sha256').update(issued[0].value.credential).digest('hex'); + expect(await redis.get(`codeapi:bridge:v1:identity:${workerId}`)).toBe(digest); + expect(await redis.get(`codeapi:bridge:v1:recovery:rate:complete:${workerId}:${challenge.enrollmentGeneration}`)) + .toBe('1'); + }); + + test('revocation beats a signed recovery pending on a different replica', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore(); + const second = recoverableStore(); + await enroll(first, identity.publicKey); + const challenge = await challengeFor(first, identity.privateKey); + const originalEval = redis.eval.bind(redis); + let release!: () => void; + let enter!: () => void; + const paused = new Promise((resolve) => { enter = resolve; }); + const resume = new Promise((resolve) => { release = resolve; }); + redis.eval = (async (script: string, ...args: unknown[]) => { + if (script.includes('local stableIdentity = redis.call')) { + enter(); + await resume; + } + return (originalEval as (...evalArgs: unknown[]) => Promise)(script, ...args); + }) as Redis['eval']; + try { + const pending = first.recoverCredential( + workerId, challenge, signBridgeRecovery(identity.privateKey, challenge), + ); + await paused; + await second.revoke(workerId); + release(); + await expect(pending).rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + expect(await redis.get(`codeapi:bridge:v1:identity:${workerId}`)).toBeNull(); + expect(await redis.get(`codeapi:bridge:v1:enrollment:${workerId}`)).toBeNull(); + } finally { + redis.eval = originalEval as Redis['eval']; + release(); + } + }); + + test('isolates untrusted recovery limits by socket peer and endpoint from signed machine quotas', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ + maxUntrustedRequestsPerMinute: 1, maxChallengesPerMinute: 1, maxAttemptsPerMinute: 1, + }); + const second = recoverableStore({ + maxUntrustedRequestsPerMinute: 1, maxChallengesPerMinute: 1, maxAttemptsPerMinute: 1, + }); + await enroll(first, identity.publicKey); + await first.limitUntrustedRecovery('192.0.2.1', 'challenge'); + await expect(second.limitUntrustedRecovery('192.0.2.1', 'challenge')) + .rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + await expect(second.limitUntrustedRecovery('192.0.2.2', 'challenge')) + .resolves.toBeUndefined(); + await expect(second.limitUntrustedRecovery('192.0.2.1', 'recover')) + .resolves.toBeUndefined(); + expect((await redis.keys('codeapi:bridge:v1:recovery:rate:start:*')).length).toBe(0); + const { credential } = await recover(second, identity.privateKey); + expect(credential.workerId).toBe(workerId); + }); + + test('only a signed, fresh, unused start request consumes the worker challenge budget', async () => { + const identity = createBridgeIdentity(); + const outsider = createBridgeIdentity(); + const first = recoverableStore({ maxChallengesPerMinute: 2 }); + const second = recoverableStore({ maxChallengesPerMinute: 2 }); + await enroll(first, identity.publicKey); + + for (let index = 0; index < 20; index += 1) { + const forged = recoveryStart(outsider.privateKey); + await expect(first.createRecoveryChallenge(workerId, forged.proof, forged.signature)) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + } + const otherServer = recoveryStart(identity.privateKey, undefined, { + serverId: 'https://unrelated.example.test', + }); + await expect(first.createRecoveryChallenge(workerId, otherServer.proof, otherServer.signature)) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + const stale = recoveryStart(identity.privateKey, undefined, { + timestamp: new Date(Date.now() - 5 * 60_000).toISOString(), + }); + await expect(first.createRecoveryChallenge(workerId, stale.proof, stale.signature)) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + + const valid = recoveryStart(identity.privateKey); + await first.createRecoveryChallenge(workerId, valid.proof, valid.signature); + await expect(second.createRecoveryChallenge(workerId, valid.proof, valid.signature)) + .rejects.toMatchObject({ code: 'PROOF_REPLAYED' }); + await challengeFor(second, identity.privateKey); + await expect(challengeFor(first, identity.privateKey)) + .rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + }); + + test('fabricated completions never spend the worker budget or block a fresh signed recovery', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ maxAttemptsPerMinute: 1 }); + const second = recoverableStore({ maxAttemptsPerMinute: 1 }); + await enroll(first, identity.publicKey); + const real = await challengeFor(first, identity.privateKey); + + for (let index = 0; index < 35; index += 1) { + const fake = { ...real, challenge: randomBytes(32).toString('base64url') }; + await expect(first.recoverCredential(workerId, fake, 'forged')) + .rejects.toMatchObject({ code: 'CHALLENGE_INVALID' }); + } + await first.recoverCredential(workerId, real, signBridgeRecovery(identity.privateKey, real)); + const next = await challengeFor(second, identity.privateKey); + await expect(second.recoverCredential( + workerId, next, signBridgeRecovery(identity.privateKey, next), + )).rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + }); + + test('invalid signatures exhaust only their own challenge, not the enrolled worker', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore({ maxAttemptsPerMinute: 1 }); + const second = recoverableStore({ maxAttemptsPerMinute: 1 }); + await enroll(first, identity.publicKey); + const attacked = await challengeFor(first, identity.privateKey); + await expect(first.recoverCredential(workerId, attacked, 'forged')) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + await expect(second.recoverCredential( + workerId, attacked, signBridgeRecovery(identity.privateKey, attacked), + )).rejects.toMatchObject({ code: 'RECOVERY_RATE_LIMITED' }); + const fresh = await challengeFor(second, identity.privateKey); + await expect(first.recoverCredential( + workerId, fresh, signBridgeRecovery(identity.privateKey, fresh), + )).resolves.toMatchObject({ workerId }); + }); + + test('key replacement starts new signed challenge and recovery budgets', async () => { + const oldKey = createBridgeIdentity(); + const newKey = createBridgeIdentity(); + const store = recoverableStore({ maxChallengesPerMinute: 1, maxAttemptsPerMinute: 1 }); + await enroll(store, oldKey.publicKey); + const oldChallenge = await challengeFor(store, oldKey.privateKey); + await store.recoverCredential( + workerId, oldChallenge, signBridgeRecovery(oldKey.privateKey, oldChallenge), + ); + const replacement = await store.issue(workerId, binding); + await store.redeem({ workerId, code: replacement.code, publicKey: newKey.publicKey }); + + await expect(challengeFor(store, oldKey.privateKey)) + .rejects.toMatchObject({ code: 'PROOF_INVALID' }); + const newChallenge = await challengeFor(store, newKey.privateKey); + await expect(store.recoverCredential( + workerId, newChallenge, signBridgeRecovery(newKey.privateKey, newChallenge), + )).resolves.toMatchObject({ workerId }); + }); + + test('revocation fences a previously verified start proof before a challenge is written', async () => { + const identity = createBridgeIdentity(); + const first = recoverableStore(); + const second = recoverableStore(); + await enroll(first, identity.publicKey); + const { proof, signature } = recoveryStart(identity.privateKey); + const originalEval = redis.eval.bind(redis); + let release!: () => void; + let enter!: () => void; + const paused = new Promise((resolve) => { enter = resolve; }); + const resume = new Promise((resolve) => { release = resolve; }); + redis.eval = (async (script: string, ...args: unknown[]) => { + if (script.includes('KEYS[6]) == 1 then return -2')) { + enter(); + await resume; + } + return (originalEval as (...evalArgs: unknown[]) => Promise)(script, ...args); + }) as Redis['eval']; + try { + const pending = first.createRecoveryChallenge(workerId, proof, signature); + await paused; + await second.revoke(workerId); + release(); + await expect(pending).rejects.toMatchObject({ code: 'ENROLLMENT_INVALID' }); + expect(await redis.keys('codeapi:bridge:v1:recovery:challenge:*')).toEqual([]); + } finally { + redis.eval = originalEval as Redis['eval']; + release(); + } + }); + + test('recovering credentials never clears worker or workspace quarantine', async () => { + const identity = createBridgeIdentity(); + const store = recoverableStore(); + await enroll(store, identity.publicKey); + const workerQuarantine = `codeapi:bridge:v1:worker:${workerId}:incarnation:incarnation-00000001:quarantined`; + const workspaceQuarantine = `codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update('session-one').digest('hex')}:quarantined`; + await redis.set(workerQuarantine, '1'); + await redis.set(workspaceQuarantine, '1'); + const { credential } = await recover(store, identity.privateKey); + expect(await redis.get(workerQuarantine)).toBe('1'); + expect(await redis.get(workspaceQuarantine)).toBe('1'); + const auth = await store.authorize( + authorizedRequest(identity.privateKey, credential.credential, 'quarantined-machine'), + ); + await expect(new RedisBridgeStore(redis).register({ + protocolVersion: 1, + workerId, + incarnationId: 'incarnation-00000001', + capabilities: { statefulWorkspace: true, sandboxProfile: 'nsjail', runtimes: ['bash'] }, + }, auth)).rejects.toMatchObject({ code: 'WORKER_QUARANTINED' }); + }); + + test('refuses non-HTTPS or non-origin deployment identity and unbounded recovery policy', () => { + for (const invalid of ['http://code.example.test', 'https://code.example.test/path', 'https://user@code.example.test']) { + expect(() => recoverableStore({ serverId: invalid })).toThrow(); + } + expect(() => recoverableStore({ maxAttemptsPerMinute: 0 })).toThrow(); + expect(() => recoverableStore({ maxUntrustedRequestsPerMinute: 0 })).toThrow(); + expect(() => recoverableStore({ maxUntrustedRequestsPerMinute: 1201 })).toThrow(); + expect(() => recoverableStore({ challengeTtlSeconds: 301 })).toThrow(); + }); +}); diff --git a/service/src/bridge/router.test.ts b/service/src/bridge/router.test.ts index 63640111..5e76876e 100644 --- a/service/src/bridge/router.test.ts +++ b/service/src/bridge/router.test.ts @@ -167,6 +167,55 @@ describe('paired bridge HTTP API', () => { }); }); + for (const laneGit of [false, true]) { + test(`registration ${laneGit ? 'offers' : 'omits'} lane_git when the setting is ${laneGit ? 'on' : 'off'}`, async () => { + const app = express(); + app.use(json()); + app.use( + '/v1/bridge', + createBridgeRouter({ + store: new RedisBridgeStore(redis, undefined, undefined, 1, laneGit), + pairings: new RedisBridgePairingStore(redis), + authMode: 'static', + adminToken: 'strong-administrator-bootstrap-token', + configuredWorkerId: 'vm-1', + }), + ); + server = createServer(app); + await new Promise((resolve) => server?.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Expected TCP listener'); + const response = await fetch(`http://127.0.0.1:${address.port}/v1/bridge/workers/register`, { + method: 'POST', + headers: { + Authorization: 'Bearer strong-administrator-bootstrap-token', + 'Content-Type': 'application/json', + }, + body: JSON.stringify({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'vm-1', + incarnationId: 'incarnation-00000001', + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'nsjail', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['read_file', 'execute_command'], + workspaces: [{ id: 'primary' }], + ...(laneGit ? { commandResultFeatures: ['lane_git'] } : {}), + }, + }, + }), + }); + expect(response.status).toBe(200); + const body = (await response.json()) as Record; + expect(body.supportedWorkspaceListFileFeatures).toEqual(['after_path']); + if (laneGit) expect(body.supportedWorkspaceCommandResultFeatures).toEqual(['lane_git']); + else expect('supportedWorkspaceCommandResultFeatures' in body).toBe(false); + }); + } + test('rejects a malformed optional binding for a configured worker', async () => { const app = express(); app.use(json()); @@ -432,8 +481,13 @@ describe('paired bridge HTTP API', () => { ], supportedWorkspaceWriteFileModes: ['replace', 'create'], supportedWorkspaceEditFileModes: ['single', 'batch'], - supportedWorkspaceEditFileFeatures: ['expected_base_sha256'], + supportedWorkspaceEditFileFeatures: [ + 'expected_base_sha256', + 'tolerant_match', + 'replace_all', + ], supportedWorkspaceListFileFeatures: ['after_path'], + supportedWorkspaceToolErrorCodes: ['NOT_FOUND'], }); const crossDeploymentRevoke = await fetch( diff --git a/service/src/bridge/router.ts b/service/src/bridge/router.ts index 5b319733..123f8c75 100644 --- a/service/src/bridge/router.ts +++ b/service/src/bridge/router.ts @@ -12,6 +12,7 @@ import { isValidBridgeWorkerCapabilities, isValidBridgeWorkerId, isWorkspaceToolErrorCode, + WORKSPACE_EDIT_FILE_FEATURES, } from '../../../packages/code/src/protocol'; import { BridgePairingError, RedisBridgePairingStore } from './pairing'; import { BridgeStoreError, RedisBridgeStore } from './store'; @@ -310,6 +311,114 @@ export function createBridgeRouter(options: BridgeRouterOptions): Router { } })); + const untrustedRecoveryLimit = (operation: 'challenge' | 'recover'): RequestHandler => + (req, res, next) => { + if (options.authMode !== 'paired' || !options.pairings.recoveryEnabled) { + next(); + return; + } + // req.ip trusts X-Forwarded-For in our server. Use the connection peer so + // untrusted headers and arbitrary worker IDs cannot create new buckets. + void options.pairings.limitUntrustedRecovery(req.socket.remoteAddress ?? '', operation) + .then(() => next(), (error: unknown) => { + if (error instanceof BridgePairingError && error.code === 'RECOVERY_RATE_LIMITED') { + res.set('Retry-After', '60').status(429).json({ error: error.message, code: error.code }); + return; + } + next(error); + }); + }; + + router.post('/workers/:workerId/credentials/challenge', untrustedRecoveryLimit('challenge'), asyncRoute(async (req, res) => { + if (options.authMode !== 'paired' || !options.pairings.recoveryEnabled) { + res.status(404).json({ error: 'Machine recovery is disabled' }); + return; + } + const workerId = req.params.workerId; + const body = isRecord(req.body) ? req.body : {}; + if ( + !validWorkerId(workerId) || !configuredWorker(workerId) || + body.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + body.operation !== 'credential.challenge' || + typeof body.serverId !== 'string' || body.serverId.length > 256 || + body.workerId !== workerId || + typeof body.timestamp !== 'string' || body.timestamp.length > 64 || + typeof body.nonce !== 'string' || !/^[A-Za-z0-9_-]{43}$/.test(body.nonce) || + typeof body.signature !== 'string' || !/^[A-Za-z0-9_-]{86}$/.test(body.signature) + ) { + res.status(400).json({ error: 'Invalid machine recovery challenge request' }); + return; + } + try { + const challenge = await options.pairings.createRecoveryChallenge( + workerId, + { + operation: 'credential.challenge', + serverId: body.serverId, + workerId, + timestamp: body.timestamp, + nonce: body.nonce, + }, + body.signature, + ); + res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, ...challenge }); + } catch (error) { + if (error instanceof BridgePairingError) { + res.status(error.code === 'RECOVERY_RATE_LIMITED' ? 429 : 401) + .json({ error: error.message, code: error.code }); + return; + } + throw error; + } + })); + + router.post('/workers/:workerId/credentials/recover', untrustedRecoveryLimit('recover'), asyncRoute(async (req, res) => { + if (options.authMode !== 'paired' || !options.pairings.recoveryEnabled) { + res.status(404).json({ error: 'Machine recovery is disabled' }); + return; + } + const workerId = req.params.workerId; + const body = isRecord(req.body) ? req.body : {}; + if ( + !validWorkerId(workerId) || !configuredWorker(workerId) || + body.protocolVersion !== BRIDGE_PROTOCOL_VERSION || + body.operation !== 'credential.recover' || + typeof body.serverId !== 'string' || body.serverId.length > 256 || + typeof body.enrollmentGeneration !== 'string' || + !/^[A-Za-z0-9_-]{24}$/.test(body.enrollmentGeneration) || + typeof body.challenge !== 'string' || + !/^[A-Za-z0-9_-]{43}$/.test(body.challenge) || + typeof body.expiresAt !== 'string' || body.expiresAt.length > 64 || + typeof body.signature !== 'string' || + !/^[A-Za-z0-9_-]{86}$/.test(body.signature) + ) { + res.status(400).json({ error: 'Invalid machine recovery proof' }); + return; + } + try { + const credential = await options.pairings.recoverCredential( + workerId, + { + operation: 'credential.recover', + serverId: body.serverId, + workerId, + enrollmentGeneration: body.enrollmentGeneration, + challenge: body.challenge, + expiresAt: body.expiresAt, + }, + body.signature, + ); + res.json({ protocolVersion: BRIDGE_PROTOCOL_VERSION, ...credential }); + } catch (error) { + if (error instanceof BridgePairingError) { + res.status(error.code === 'RECOVERY_RATE_LIMITED' ? 429 : 401) + .json({ error: error.message, code: error.code }); + return; + } + throw error; + } + })); + router.post( '/workers/:workerId/revoke', adminAuth, @@ -427,10 +536,15 @@ router.post( ], supportedWorkspaceWriteFileModes: ['replace', 'create'], supportedWorkspaceEditFileModes: ['single', 'batch'], - supportedWorkspaceEditFileFeatures: ['expected_base_sha256'], + supportedWorkspaceEditFileFeatures: [...WORKSPACE_EDIT_FILE_FEATURES], supportedWorkspaceListFileFeatures: ['after_path'], + ...(options.store.laneGitEnabled + ? { supportedWorkspaceCommandResultFeatures: ['lane_git'] } + : {}), supportedWorkspaceProgrammaticLanguages: ['bash'], supportedWorkspaceInstanceTypes: ['git_worktree'], + supportedWorkspaceScopes: ['git_linked_worktree'], + supportedWorkspaceToolErrorCodes: ['NOT_FOUND'], }); } catch (error) { if (error instanceof BridgeStoreError) { @@ -768,6 +882,5 @@ router.post( }), ); - return router; } diff --git a/service/src/bridge/slots.test.ts b/service/src/bridge/slots.test.ts index dfe89495..1efe3b97 100644 --- a/service/src/bridge/slots.test.ts +++ b/service/src/bridge/slots.test.ts @@ -3,6 +3,7 @@ import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; import { BridgeAdmissionQueue } from './admission'; import { BridgeWorkspaceSlots } from './slots'; +import { workspaceIsolationKey } from '../../../packages/code/src/protocol'; const redis = new RedisMock() as unknown as Redis; const admission = new BridgeAdmissionQueue(redis); @@ -13,20 +14,27 @@ const prefix = `codeapi:bridge:v1:worker:${workerId}`; afterEach(async () => { await redis.flushall(); }); -async function enqueue(assignmentId: string, workspaceId: string) { +async function enqueue(assignmentId: string, workspaceId: string, capacity = 2) { await redis.set(`${prefix}:incarnation`, incarnationId); - await redis.set(`${prefix}:workspace-slot-capacity`, '2'); + await redis.set(`${prefix}:workspace-slot-capacity`, String(capacity)); await admission.enter(workerId, assignmentId, Date.now() + 5000, workspaceId); return { workerId, incarnationId, assignmentId, workspaceId, - capacity: 2, + capacity, expiresAtMs: Date.now() + 10000, }; } +async function finish(assignmentId: string) { + await slots.release(workerId, incarnationId, assignmentId); + await admission.leave(workerId, assignmentId); +} + +const lane = (name: string, parent = 'repo') => workspaceIsolationKey(parent, undefined, name); + test('slots admit independent workspaces, skip a busy root, and bound capacity', async () => { const a = await enqueue('a', 'root-a'); const a2 = await enqueue('a2', 'root-a'); @@ -91,3 +99,51 @@ test('releasing a long slot shortens the aggregate expiry to remaining work', as await slots.release(workerId, incarnationId, 'a'); expect(await redis.pttl(`${prefix}:lock`)).toBeLessThanOrEqual(3000); }); + +test('sibling linked-worktree lanes share their checkout, which waits for both', async () => { + const a = await enqueue('a', lane('task-a'), 3); + const b = await enqueue('b', lane('task-b'), 3); + const root = await enqueue('root', 'repo', 3); + expect(await slots.reserve(a)).toBe(0); + expect(await slots.reserve(b)).toBe(1); + expect(await slots.reserve(root)).toBeUndefined(); + await finish('a'); + expect(await slots.reserve(root)).toBeUndefined(); + await finish('b'); + expect(await slots.reserve(root)).toBe(0); +}); + +test('a busy checkout holds every lane beneath it but not other checkouts', async () => { + const root = await enqueue('root', 'repo', 3); + const a = await enqueue('a', lane('task-a'), 3); + const other = await enqueue('other', lane('task-a', 'other-repo'), 3); + expect(await slots.reserve(root)).toBe(0); + expect(await slots.reserve(a)).toBeUndefined(); + expect(await slots.reserve(other)).toBe(1); + await finish('root'); + expect(await slots.reserve(a)).toBe(0); +}); + +test('a waiting checkout holds back newer lanes so root work cannot starve', async () => { + const a = await enqueue('a', lane('task-a'), 3); + expect(await slots.reserve(a)).toBe(0); + const root = await enqueue('root', 'repo', 3); + const b = await enqueue('b', lane('task-b'), 3); + const other = await enqueue('other', 'other-repo', 3); + expect(await slots.reserve(root)).toBeUndefined(); + expect(await slots.reserve(b)).toBeUndefined(); + expect(await slots.reserve(other)).toBe(1); + await finish('a'); + expect(await slots.reserve(b)).toBeUndefined(); + expect(await slots.reserve(root)).toBe(0); +}); + +test('lanes nest beneath a conversation checkout, not the source root', async () => { + const instance = workspaceIsolationKey('repo', 'c'.repeat(64)); + const conversation = await enqueue('conversation', instance, 3); + const nested = await enqueue('nested', workspaceIsolationKey('repo', 'c'.repeat(64), 'task-a'), 3); + const source = await enqueue('source', lane('task-a'), 3); + expect(await slots.reserve(conversation)).toBe(0); + expect(await slots.reserve(nested)).toBeUndefined(); + expect(await slots.reserve(source)).toBe(1); +}); diff --git a/service/src/bridge/slots.ts b/service/src/bridge/slots.ts index 9c9e7d3b..f748054b 100644 --- a/service/src/bridge/slots.ts +++ b/service/src/bridge/slots.ts @@ -1,4 +1,5 @@ import type Redis from 'ioredis'; +import { durableAdmissionFence } from './admission'; /** Hard bound keeps every atomic scheduling scan constant-sized. */ export const MAX_WORKSPACE_LEASE_SLOTS = 8; @@ -6,6 +7,12 @@ export const MAX_WORKSPACE_LEASE_SLOTS = 8; /** Reservations share the legacy admission queue and aggregate worker lock. * Thus a serial dispatcher or replacement incarnation cannot race active slots. * Workspace mutation uncertainty is fenced separately by the assignment store. + * + * Keys are hierarchical: a linked-worktree lane (`\0linked-worktree\0\0`) + * conflicts with itself and with its parent checkout, and a checkout conflicts + * with every lane beneath it. Sibling lanes run concurrently. A waiting checkout + * request also holds back newer lanes beneath it, so root operations such as + * `git worktree add` cannot be starved by a steady stream of lane work. */ export class BridgeWorkspaceSlots { constructor(private readonly redis: Redis) {} @@ -31,6 +38,8 @@ export class BridgeWorkspaceSlots { workspaceId: string; capacity: number; expiresAtMs: number; + refreshOwned?: boolean; + guard?: { key: string; claimKey: string; token: string }; }): Promise { if ( !Number.isSafeInteger(args.capacity) || @@ -42,15 +51,34 @@ export class BridgeWorkspaceSlots { ) { throw new Error('Invalid workspace slot reservation'); } + const keys = this.keys(args.workerId); + if (args.guard != null) keys.push(args.guard.key, args.guard.claimKey); const result = Number( await this.redis.eval( [ + ...(args.guard == null ? [] : durableAdmissionFence('KEYS[9]', 'KEYS[10]', 'ARGV[8]', -3)), "if redis.call('GET', KEYS[4]) ~= ARGV[1] then return -2 end", "if (redis.call('GET', KEYS[8]) or '1') ~= ARGV[4] then return -2 end", + // Parent of a linked-worktree lane key; nil for a checkout key. + 'local lanePrefix = "\\0linked-worktree\\0"', + 'local function parentOf(key)', + ' if string.sub(key, 1, #lanePrefix) ~= lanePrefix then return nil end', + ' local last = nil', + ' local cursor = #lanePrefix + 1', + ' while true do', + ' local found = string.find(key, "\\0", cursor, true)', + ' if not found then break end', + ' last = found', + ' cursor = found + 1', + ' end', + ' if last == nil or last <= #lanePrefix + 1 then return nil end', + ' return string.sub(key, #lanePrefix + 1, last - 1)', + 'end', "local lock = redis.call('GET', KEYS[2])", "local owner = 'workspace-slots:' .. ARGV[1]", 'if lock and lock ~= owner then return -1 end', 'local busy = {}', + 'local busyChildren = {}', 'local free = nil', 'local latest = tonumber(ARGV[5])', `for slot = 0, ${MAX_WORKSPACE_LEASE_SLOTS - 1} do`, @@ -61,14 +89,30 @@ export class BridgeWorkspaceSlots { " redis.call('HDEL', KEYS[1], 'a:' .. slot, 'i:' .. slot, 'w:' .. slot, 'e:' .. slot)", ' occupied = false', ' else', - ' if entry[1] == ARGV[2] then return slot end', + ' if entry[1] == ARGV[2] then', + ' if ARGV[7] == \'1\' then', + ' redis.call(\'HSET\', KEYS[1], \'e:\' .. slot, ARGV[5])', + ' for _, key in ipairs({KEYS[1], KEYS[2], KEYS[3]}) do', + ' if redis.call(\'PTTL\', key) < tonumber(ARGV[5]) - tonumber(ARGV[6]) then redis.call(\'PEXPIREAT\', key, ARGV[5]) end', + ' end', + ' end', + ' return slot', + ' end', ' busy[entry[3]] = true', + ' local busyParent = parentOf(entry[3])', + ' if busyParent then busyChildren[busyParent] = true end', ' latest = math.max(latest, tonumber(entry[4]))', ' end', ' end', ' if not occupied and free == nil and slot < tonumber(ARGV[4]) then free = slot end', 'end', - 'if free == nil or busy[ARGV[3]] then return -1 end', + 'local function conflicts(key)', + ' if busy[key] then return true end', + ' local parent = parentOf(key)', + ' if parent then return busy[parent] == true end', + ' return busyChildren[key] == true', + 'end', + 'if free == nil or conflicts(ARGV[3]) then return -1 end', // Expiry removes queue metadata, never a workspace uncertainty fence. "local expired = redis.call('ZRANGEBYSCORE', KEYS[6], '-inf', ARGV[6])", 'for _, id in ipairs(expired) do', @@ -78,11 +122,15 @@ export class BridgeWorkspaceSlots { 'end', "local pending = redis.call('ZRANGE', KEYS[5], 0, 31)", 'local selected = nil', + 'local waitingCheckouts = {}', 'for _, id in ipairs(pending) do', " local workspace = redis.call('HGET', KEYS[7], id)", // An older serial request remains a barrier until its dispatcher finishes. ' if not workspace then return -1 end', - ' if not busy[workspace] then selected = id; break end', + ' local parent = parentOf(workspace)', + ' local held = conflicts(workspace) or (parent ~= nil and waitingCheckouts[parent] == true)', + ' if not held then selected = id; break end', + ' if parent == nil then waitingCheckouts[workspace] = true end', 'end', 'if selected ~= ARGV[2] then return -1 end', "redis.call('HSET', KEYS[1], 'a:' .. free, ARGV[2], 'i:' .. free, ARGV[1], 'w:' .. free, ARGV[3], 'e:' .. free, ARGV[5])", @@ -91,14 +139,16 @@ export class BridgeWorkspaceSlots { "redis.call('SET', KEYS[3], ARGV[1], 'PXAT', latest)", 'return free', ].join('\n'), - 8, - ...this.keys(args.workerId), + keys.length, + ...keys, args.incarnationId, args.assignmentId, args.workspaceId, args.capacity, args.expiresAtMs, Date.now(), + args.refreshOwned === true ? '1' : '0', + args.guard?.token ?? '', ), ); if (result === -2) diff --git a/service/src/bridge/store.ts b/service/src/bridge/store.ts index 2ffeec66..0ca74c45 100644 --- a/service/src/bridge/store.ts +++ b/service/src/bridge/store.ts @@ -18,10 +18,14 @@ import { isWorkspaceToolRequest, isWorkspaceToolResult, workspaceIsolationKey, + workspaceIsolationParent, } from '../../../packages/code/src/protocol'; import type { BridgeWorkerBinding } from './pairing'; -import { BridgeAdmissionQueue } from './admission'; +import logger from '../logger'; +import { hasLaneGit, laneGitPolicy } from './lane-git'; +import { BridgeAdmissionQueue, durableAdmissionFence } from './admission'; import { BridgeWorkspaceSlots } from './slots'; +import type { StoredWorkspaceRequest } from '../workspace-tools/requests'; const PREFIX = 'codeapi:bridge:v1'; const POLL_INTERVAL_MS = 100; @@ -65,10 +69,46 @@ export class BridgeStoreError extends Error { } } +function classifyPreEnqueueExpiry( + error: unknown, + args: { workspaceRequest?: WorkspaceToolRequest; workspaceId?: string; signal: AbortSignal }, +): unknown { + // A queue deadline reached before the assignment enqueue attempt is a + // definite non-execution, including expiry during the initial Redis reads. + if ( + (args.workspaceRequest != null || args.workspaceId != null) && + !args.signal.aborted && + error instanceof BridgeStoreError && + error.code === 'ASSIGNMENT_EXPIRED' + ) { + return new BridgeStoreError( + 'WORKSPACE_QUEUE_TIMEOUT', + 'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.', + ); + } + return error; +} + interface StoredAssignment extends CodeBridgeAssignment { leaseTokenHash: string; workerIdentityId?: string; workspaceFence?: string; + durableRequestKey?: string; +} + +export type DurableWorkspaceAssignment = StoredAssignment; + +export interface DurableWorkspaceGuard { + key: string; + claimKey: string; + token: string; +} + +export interface DurableWorkspaceOutcome { + state: 'completed' | 'failed' | 'cancelled'; + result?: WorkspaceToolResult; + error?: { code: string; message: string }; + settlement?: CodeBridgeWorkspaceSettlement; } type AssignmentOwnership = Pick< @@ -83,8 +123,28 @@ type AssignmentOwnership = Pick< | 'workerIdentityId' | 'expiresAt' | 'runtimeSessionId' + | 'durableRequestKey' >; +const NATIVE_WORKSPACE_FENCE_PREFIX = 'native-workspace:'; + +/** Fence keys of the linked-worktree lanes enqueued beneath a checkout fence. */ +function workspaceLaneFenceIndexKey(checkoutFenceKey: string): string { + return `${checkoutFenceKey}:lanes`; +} + +/** The fence of the checkout a linked-worktree lane nests beneath. While that + * checkout is quarantined, its lanes may not start either. */ +function workspaceFenceParent(fence: string): string | undefined { + if (!fence.startsWith(NATIVE_WORKSPACE_FENCE_PREFIX)) return undefined; + const parent = workspaceIsolationParent( + fence.slice(NATIVE_WORKSPACE_FENCE_PREFIX.length), + ); + return parent === undefined + ? undefined + : `${NATIVE_WORKSPACE_FENCE_PREFIX}${parent}`; +} + function workspaceFenceReceiptKey(assignmentId: string): string { return `${assignmentKey(assignmentId)}:workspace-fence-owner`; } @@ -108,6 +168,10 @@ export interface BridgeWorkerStatus { capabilities?: BridgeWorkerRegistration['capabilities']; } +function isWorkspaceMutation(request: WorkspaceToolRequest): boolean { + return request.operation === 'write_file' || request.operation === 'edit_file' || request.operation === 'execute_command'; +} + function supportsWorkspaceTool( registration: RegisteredBridgeWorker, request: WorkspaceToolRequest, @@ -131,6 +195,12 @@ function supportsWorkspaceTool( ) { return false; } + if ( + request.worktree !== undefined && + workspace.workspaceScopes?.includes('git_linked_worktree') !== true + ) { + return false; + } if (request.operation === 'list_files' && request.afterPath !== undefined) { return capabilities?.listFileFeatures?.includes('after_path') === true; } @@ -148,12 +218,21 @@ function supportsWorkspaceTool( const mode = request.edits === undefined ? 'single' : 'batch'; const modes = capabilities?.editFileModes; const supportsMode = modes == null ? mode === 'single' : modes.includes(mode); - if (request.operation === 'preview_edit') return supportsMode; + const features = capabilities?.editFileFeatures ?? []; + const supportsMatching = + request.matching === undefined || features.includes('tolerant_match'); + const supportsReplaceAll = + request.edits?.some((edit) => edit.replaceAll !== undefined) !== true || + features.includes('replace_all'); + if (request.operation === 'preview_edit') { + return supportsMode && supportsMatching && supportsReplaceAll; + } return ( supportsMode && + supportsMatching && + supportsReplaceAll && (request.expectedBaseSha256 === undefined || - capabilities?.editFileFeatures?.includes('expected_base_sha256') === - true) + features.includes('expected_base_sha256')) ); } return true; @@ -164,6 +243,7 @@ function supportsWorkspaceProgrammatic( workspaceId: string, language: string, workspaceInstanceId?: string, + worktree?: string, ): boolean { const capabilities = registration.capabilities.workspaceTools; const workspace = capabilities?.workspaces.find( @@ -173,6 +253,8 @@ function supportsWorkspaceProgrammatic( workspace != null && (workspaceInstanceId === undefined || workspace.workspaceInstances?.includes('git_worktree') === true) && + (worktree === undefined || + workspace.workspaceScopes?.includes('git_linked_worktree') === true) && capabilities?.operations.includes('execute_command') === true && (workspace.operations == null || workspace.operations.includes('execute_command')) && @@ -194,11 +276,24 @@ function workspaceInstanceId(body: t.PayloadBody): string | undefined { return undefined; } +function workspaceWorktree(body: t.PayloadBody): string | undefined { + if ( + typeof body === 'object' && + body != null && + 'workspace_worktree' in body && + typeof body.workspace_worktree === 'string' + ) { + return body.workspace_worktree; + } + return undefined; +} + export function workspaceAdmissionId( workspaceId: string, instanceId?: string, + worktree?: string, ): string { - return workspaceIsolationKey(workspaceId, instanceId); + return workspaceIsolationKey(workspaceId, instanceId, worktree); } function workerKey(workerId: string): string { @@ -290,6 +385,12 @@ function assignmentKey(assignmentId: string): string { return `${PREFIX}:assignment:${assignmentId}`; } +function withoutLaneGit(settlement: AnyCodeBridgeSettlement): AnyCodeBridgeSettlement { + if (settlement.status !== 'fulfilled') return settlement; + const { laneGit: _laneGit, ...result } = settlement.result as unknown as Record; + return { ...settlement, result } as AnyCodeBridgeSettlement; +} + function settlementKey(assignmentId: string): string { return `${PREFIX}:assignment:${assignmentId}:settlement`; } @@ -310,6 +411,12 @@ function assignmentTtlSeconds(deadlineAtMs: number): number { return Math.max(1, Math.ceil((deadlineAtMs - Date.now()) / 1000) + 30); } +function assignmentOutcomeTtlSeconds(assignment: AssignmentOwnership): number { + return assignment.durableRequestKey == null + ? assignmentTtlSeconds(Date.parse(assignment.expiresAt)) + : 24 * 60 * 60; +} + async function delay(ms: number, signal?: AbortSignal): Promise { if (signal?.aborted === true) return; await new Promise((resolve) => { @@ -373,6 +480,8 @@ export class RedisBridgeStore { private readonly workerTtlSeconds = DEFAULT_WORKER_TTL_SECONDS, private readonly redisCommandTimeoutMs = DEFAULT_REDIS_COMMAND_TIMEOUT_MS, private readonly maxWorkspaceLeaseSlots = 1, + /** Accept `laneGit` on command results and offer `lane_git` at registration. Off by default. */ + public readonly laneGitEnabled = false, ) { if ( !Number.isSafeInteger(maxWorkspaceLeaseSlots) || @@ -704,6 +813,127 @@ export class RedisBridgeStore { } } + async prepareDurableWorkspaceTool(args: { + workerId: string; tenantId: string; requireTenantBinding: boolean; + request: WorkspaceToolRequest; executionTimeoutMs: number; queueWaitMs: number; + }): Promise { + if (!isWorkspaceToolRequest(args.request) || + !Number.isSafeInteger(args.executionTimeoutMs) || args.executionTimeoutMs < 1 || args.executionTimeoutMs > 305_000 || + !Number.isSafeInteger(args.queueWaitMs) || args.queueWaitMs < 1 || args.queueWaitMs > 300_000) { + throw new BridgeStoreError('ASSIGNMENT_INVALID', 'Invalid durable workspace request'); + } + const current = await boundedCommand(this.dispatchableRegistration(args.workerId), this.redisCommandTimeoutMs, 'Durable worker registration'); + if (current == null) throw new BridgeStoreError('WORKER_OFFLINE', 'Code environment is offline'); + const registration = current.registration; + if ((args.requireTenantBinding && registration.binding == null) || + (registration.binding != null && registration.binding.tenantId !== args.tenantId)) { + throw new BridgeStoreError('WORKER_UNAUTHORIZED', 'Code environment is not authorized for this tenant'); + } + if ((registration.capabilities.workspaceLeaseSlots ?? 1) > this.maxWorkspaceLeaseSlots || + !supportsWorkspaceTool(registration, args.request)) { + throw new BridgeStoreError('WORKER_MISMATCH', 'Code environment does not support the requested workspace operation'); + } + return registration; + } + + async advanceDurableWorkspaceTool( + record: StoredWorkspaceRequest, + guard: DurableWorkspaceGuard, + ): Promise { + if (record.state === 'queued') { + if (record.cancelRequested === true) return { state: 'cancelled' }; + if (Date.now() >= record.queueDeadlineAtMs) return { + state: 'failed', error: { code: 'WORKSPACE_QUEUE_TIMEOUT', message: 'Workspace admission deadline expired. The operation was not started.' }, + }; + const registration = await this.prepareDurableWorkspaceTool({ + ...record, queueWaitMs: Math.max(1, record.queueDeadlineAtMs - Date.now()), + }); + if (registration.incarnationId !== record.registration.incarnationId || + registration.identityId !== record.registration.identityId || + registration.binding?.tenantId !== record.registration.binding?.tenantId) { + throw new BridgeStoreError('WORKER_FENCED', 'Code environment changed while the request was waiting'); + } + const admission = new BridgeAdmissionQueue(this.redis); + const workspace = workspaceAdmissionId(record.request.workspaceId, record.request.workspaceInstanceId, record.request.worktree); + const capacity = registration.capabilities.workspaceLeaseSlots ?? 1; + if (capacity === 1 && !await admission.isHead(record.workerId, record.id)) return; + const expiresAtMs = Date.now() + record.executionTimeoutMs; + const ttlSeconds = assignmentTtlSeconds(expiresAtMs); + let slot: number | undefined; + if (capacity > 1) { + slot = await new BridgeWorkspaceSlots(this.redis).reserve({ + workerId: record.workerId, incarnationId: registration.incarnationId, + assignmentId: record.id, workspaceId: workspace, capacity, + expiresAtMs: Date.now() + ttlSeconds * 1000, refreshOwned: true, guard, + }); + if (slot === undefined) return; + } else if (!await this.acquireLock(record.workerId, record.id, registration.incarnationId, ttlSeconds, true, guard)) return; + if (Date.now() >= record.queueDeadlineAtMs) return { + state: 'failed', error: { code: 'WORKSPACE_QUEUE_TIMEOUT', message: 'Workspace admission deadline expired. The operation was not started.' }, + }; + const leaseToken = randomBytes(32).toString('base64url'); + const assignment: StoredAssignment = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + assignmentId: record.id, workerId: record.workerId, + incarnationId: registration.incarnationId, + generation: await this.redis.incr(generationKey(record.workerId)), + leaseToken, leaseTokenHash: tokenHash(leaseToken), + workerIdentityId: registration.identityId, + workspaceFence: `${NATIVE_WORKSPACE_FENCE_PREFIX}${workspace}`, + workspaceLeaseSlot: slot, executionKind: 'workspace_tool', durableRequestKey: record.key, + expiresAt: new Date(Date.now() + record.executionTimeoutMs).toISOString(), + request: record.request, + }; + const ready = await this.dispatchableRegistration(record.workerId); + if (ready == null) throw new BridgeStoreError('WORKER_OFFLINE', 'Code environment is temporarily unavailable'); + if (ready.registration.incarnationId !== registration.incarnationId || + ready.registration.identityId !== registration.identityId || + ready.registration.binding?.tenantId !== registration.binding?.tenantId || + !supportsWorkspaceTool(ready.registration, record.request)) { + throw new BridgeStoreError('WORKER_FENCED', 'Code environment changed before admission'); + } + await this.enqueueForActiveIncarnation(assignment, assignmentTtlSeconds(Date.parse(assignment.expiresAt)), ready.readyToken, guard); + return; + } + const assignment = record.assignment; + if (assignment == null) throw new Error('Durable admitted request has no assignment'); + let settlement: CodeBridgeWorkspaceSettlement; + try { + const raw = await boundedCommand(this.redis.get(settlementKey(record.id)), this.redisCommandTimeoutMs, 'Durable settlement lookup'); + if (raw == null && record.cancelRequested !== true && Date.now() < Date.parse(assignment.expiresAt)) return; + if (raw != null) settlement = JSON.parse(raw) as CodeBridgeWorkspaceSettlement; + else { + const controller = new AbortController(); + if (record.cancelRequested === true) controller.abort(); + settlement = await this.waitForSettlement(assignment, Date.parse(assignment.expiresAt), controller.signal) as unknown as CodeBridgeWorkspaceSettlement; + } + } catch (error) { + if (!(error instanceof BridgeStoreError) || error.code !== 'ASSIGNMENT_EXPIRED') throw error; + if (record.cancelRequested === true && !isWorkspaceMutation(record.request)) return { state: 'cancelled' }; + return { state: 'failed', error: { code: error.code, message: 'Assignment ended without a confirmed result. Do not replay this operation.' } }; + } + if (settlement.status === 'fulfilled') { + if (!isWorkspaceToolResult(record.request, settlement.result, record.registration.capabilities.workspaceTools)) { + return { state: 'failed', error: { code: 'RESULT_INVALID', message: 'Code environment returned an invalid workspace result. Do not replay this operation.' } }; + } + return { state: 'completed', result: settlement.result, settlement }; + } + return { + state: record.cancelRequested === true ? 'cancelled' : 'failed', + error: { code: settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED', message: settlement.error }, settlement, + }; + } + + async finishDurableWorkspaceTool(record: StoredWorkspaceRequest): Promise { + if (record.assignment != null) { + if (record.settlement != null) await this.commitPendingWorkspace(record.assignment, record.settlement, record.key); + await this.cleanupDispatch(record.workerId, record.id, record.assignment); + } else { + await this.cleanupUnassignedSlot(record.workerId, record.registration.incarnationId, record.id); + } + await new BridgeAdmissionQueue(this.redis).leave(record.workerId, record.id); + } + async dispatchWorkspaceTool(args: { workerId: string; tenantId?: string; @@ -778,12 +1008,17 @@ export class RedisBridgeStore { 'Invalid workspace execution budget', ); } - this.assertDispatchActive(args.signal, args.deadlineAtMs); - const dispatchable = await this.dispatchCommand( - () => this.dispatchableRegistration(args.workerId), - args, - 'Bridge worker registration read', - ); + let dispatchable: Awaited>; + try { + this.assertDispatchActive(args.signal, args.deadlineAtMs); + dispatchable = await this.dispatchCommand( + () => this.dispatchableRegistration(args.workerId), + args, + 'Bridge worker registration read', + ); + } catch (error) { + throw classifyPreEnqueueExpiry(error, args); + } if (dispatchable == null) { throw new BridgeStoreError( 'WORKER_OFFLINE', @@ -837,6 +1072,7 @@ export class RedisBridgeStore { args.workspaceId, args.body.language, workspaceInstanceId(args.body), + workspaceWorktree(args.body), ) ) { throw new BridgeStoreError( @@ -844,17 +1080,20 @@ export class RedisBridgeStore { `Bridge worker ${args.workerId} does not advertise programmatic execution for the selected workspace`, ); } - if ( - args.runtimeSessionId !== undefined && - (await this.dispatchCommand( - () => - this.redis.exists( - workspaceQuarantineKey(args.workerId, args.runtimeSessionId ?? ''), - ), - args, - 'Bridge workspace fence read', - )) === 1 - ) { + let quarantined = false; + const runtimeSessionId = args.runtimeSessionId; + if (runtimeSessionId !== undefined) { + try { + quarantined = (await this.dispatchCommand( + () => this.redis.exists(workspaceQuarantineKey(args.workerId, runtimeSessionId)), + args, + 'Bridge workspace fence read', + )) === 1; + } catch (error) { + throw classifyPreEnqueueExpiry(error, args); + } + } + if (quarantined) { throw new BridgeStoreError( 'WORKSPACE_QUARANTINED', 'Bridge workspace is quarantined after an incomplete result commit', @@ -863,10 +1102,6 @@ export class RedisBridgeStore { const assignmentId = randomBytes(18).toString('base64url'); const leaseToken = randomBytes(32).toString('base64url'); - // The lock is acquired before admission finishes; it must outlive the later execution deadline. - const ttlSeconds = assignmentTtlSeconds( - args.deadlineAtMs + (args.executionTimeoutMs ?? 0), - ); const lockIncarnationId = registration.incarnationId; let assignment: StoredAssignment | undefined; let enqueueAttempted = false; @@ -875,12 +1110,15 @@ export class RedisBridgeStore { args.workspaceRequest?.workspaceId ?? args.workspaceId; const selectedWorkspaceInstanceId = args.workspaceRequest?.workspaceInstanceId ?? workspaceInstanceId(args.body); + const selectedWorktree = + args.workspaceRequest?.worktree ?? workspaceWorktree(args.body); const selectedWorkspaceAdmissionId = selectedWorkspaceId == null ? undefined : workspaceAdmissionId( selectedWorkspaceId, selectedWorkspaceInstanceId, + selectedWorktree, ); const workspaceSlots = selectedWorkspaceId != null && @@ -931,6 +1169,14 @@ export class RedisBridgeStore { ); continue; } + // A queued request may have waited nearly its full admission budget. + // Reserve for the *remaining* absolute deadline or a fresh execution + // budget, not the original queue window plus execution again. + const ttlSeconds = assignmentTtlSeconds( + args.executionTimeoutMs === undefined + ? args.deadlineAtMs + : Date.now() + args.executionTimeoutMs, + ); if (workspaceSlots != null) { workspaceLeaseSlot = await this.dispatchCommand( () => @@ -1001,6 +1247,7 @@ export class RedisBridgeStore { args.workspaceId, args.body.language, workspaceInstanceId(args.body), + workspaceWorktree(args.body), )) ) { throw new BridgeStoreError( @@ -1027,13 +1274,13 @@ export class RedisBridgeStore { leaseToken, leaseTokenHash: tokenHash(leaseToken), ...(selectedWorkspaceAdmissionId == null ? {} : { - workspaceFence: `native-workspace:${selectedWorkspaceAdmissionId}`, + workspaceFence: `${NATIVE_WORKSPACE_FENCE_PREFIX}${selectedWorkspaceAdmissionId}`, }), ...(workspaceLeaseSlot === undefined ? {} : { workspaceLeaseSlot, - workspaceFence: `native-workspace:${selectedWorkspaceAdmissionId!}`, + workspaceFence: `${NATIVE_WORKSPACE_FENCE_PREFIX}${selectedWorkspaceAdmissionId!}`, }), ...(registration.identityId != null ? { workerIdentityId: registration.identityId } @@ -1070,7 +1317,7 @@ export class RedisBridgeStore { enqueueAttempted = true; return this.enqueueForActiveIncarnation( assignment!, - ttlSeconds, + assignmentTtlSeconds(Date.parse(assignment!.expiresAt)), readyToken, ); }, @@ -1123,6 +1370,7 @@ export class RedisBridgeStore { args.workspaceId, args.body.language, workspaceInstanceId(args.body), + workspaceWorktree(args.body), ) ) { throw new BridgeStoreError( @@ -1177,15 +1425,7 @@ export class RedisBridgeStore { } } catch (error) { // Once enqueue starts, even a lost Redis response may hide execution. - if ( - admission != null && !enqueueAttempted && !args.signal.aborted && - error instanceof BridgeStoreError && error.code === 'ASSIGNMENT_EXPIRED' - ) { - throw new BridgeStoreError( - 'WORKSPACE_QUEUE_TIMEOUT', - 'Workspace capacity was unavailable before the queue deadline. The operation was not started. Wait for active work to finish or select an independent workspace on a machine with available capacity.', - ); - } + if (!enqueueAttempted) throw classifyPreEnqueueExpiry(error, args); throw error; } finally { if (admission != null) { @@ -1333,6 +1573,7 @@ export class RedisBridgeStore { leaseTokenHash: _leaseTokenHash, workerIdentityId: _workerIdentityId, workspaceFence: _workspaceFence, + durableRequestKey: _durableRequestKey, ...wireAssignment } = assignment; return { @@ -1560,12 +1801,25 @@ export class RedisBridgeStore { ); return; } - const serializedSettlement = JSON.stringify(settlement); + let serializedSettlement = JSON.stringify(settlement); const existingSettlement = await this.leaseCommand( this.redis.get(settlementKey(assignmentId)), signal, 'Bridge settlement existing read', ); + if ( + existingSettlement != null && + existingSettlement !== serializedSettlement && + hasLaneGit(settlement.status === 'fulfilled' ? settlement.result : undefined) + ) { + // A retry of a committed settlement must stay recognizable whichever way this replica's + // lane Git setting differs from the replica that committed it. + const stripped = withoutLaneGit(settlement); + if (existingSettlement === JSON.stringify(stripped)) { + settlement = stripped; + serializedSettlement = existingSettlement; + } + } if ( existingSettlement != null && existingSettlement !== serializedSettlement @@ -1625,7 +1879,14 @@ export class RedisBridgeStore { 'Bridge assignment has expired', ); } - const ttlSeconds = assignmentTtlSeconds(Date.parse(assignment.expiresAt)); + // Uses the assignment and registration already read above: no extra round trips on the + // settlement path, which has only a short grace after the command ran. + const sanitized = this.sanitizeLaneGit(assignmentId, settlement, assignment, registration); + if (sanitized !== settlement) { + settlement = sanitized; + serializedSettlement = JSON.stringify(settlement); + } + const ttlSeconds = assignmentOutcomeTtlSeconds(assignment); const settlementKeys = [ assignmentKey(assignmentId), settlementKey(assignmentId), @@ -1733,6 +1994,38 @@ export class RedisBridgeStore { } } + /** + * Validate the optional `laneGit` of a fulfilled execute_command settlement before it is + * stored or forwarded. An invalid, unexpected or unadvertised value is dropped, never an + * error: the command's own result must still reach the caller. Pure over records the caller + * already read, so it adds no Redis round trips. + */ + private sanitizeLaneGit( + assignmentId: string, + settlement: AnyCodeBridgeSettlement, + assignment: StoredAssignment, + registration: RegisteredBridgeWorker | undefined, + ): AnyCodeBridgeSettlement { + if ( + settlement.status !== 'fulfilled' || + !hasLaneGit(settlement.result) || + assignment.executionKind !== 'workspace_tool' || + (assignment.request as WorkspaceToolRequest).operation !== 'execute_command' + ) { + return settlement; + } + const { dropped } = laneGitPolicy({ + result: settlement.result as unknown as Record, + enabled: this.laneGitEnabled, + advertised: + registration?.capabilities.workspaceTools?.commandResultFeatures?.includes('lane_git') === true, + }); + if (dropped === undefined) return settlement; + // Reason only: never the branch, the head or anything else from the result. + logger.debug('Dropped lane Git from a command settlement', { reason: dropped, assignmentId }); + return withoutLaneGit(settlement); + } + async cancelled( workerId: string, incarnationId: string, @@ -1904,7 +2197,7 @@ export class RedisBridgeStore { raw.epoch, assignmentId, JSON.stringify(settlement), - assignmentTtlSeconds(Date.parse(receipt.expiresAt)), + assignmentOutcomeTtlSeconds(receipt), quarantine ? '1' : '0', ), signal, @@ -2049,14 +2342,12 @@ export class RedisBridgeStore { const cancelledMutation = signal.aborted && (assignment.executionKind === 'workspace_programmatic' || - (workspaceRequest != null && - (workspaceRequest.operation === 'write_file' || - workspaceRequest.operation === 'edit_file' || - workspaceRequest.operation === 'execute_command'))); - if (cancelledMutation) { + (workspaceRequest != null && isWorkspaceMutation(workspaceRequest))); + const cancelledDurableWorkspace = signal.aborted && assignment.durableRequestKey != null && workspaceRequest != null; + if (cancelledMutation || cancelledDurableWorkspace) { try { // Keep the acknowledged assignment available long enough for the - // worker to terminate its process tree and commit a clean rejection. + // worker to stop its operation and commit a clean rejection. // Closing it first makes that rejection impossible to acknowledge and // leaves the worker's durable mutation guard armed. await this.cancel(assignment.assignmentId, assignment); @@ -2156,11 +2447,29 @@ export class RedisBridgeStore { assignment: StoredAssignment, ttlSeconds: number, readyToken?: string, + guard?: DurableWorkspaceGuard, ): Promise { const script = [ + 'local keyCount = tonumber(ARGV[10])', + ...(guard == null ? [] : [ + ...durableAdmissionFence('KEYS[keyCount + 1]', 'KEYS[keyCount + 2]', 'ARGV[11]', -2), + ...(assignment.workspaceLeaseSlot === undefined + ? ['if redis.call(\'GET\', KEYS[keyCount + 3]) ~= ARGV[4] or redis.call(\'GET\', KEYS[4]) ~= ARGV[1] then return -2 end'] + : [ + `if redis.call('HGET', KEYS[keyCount + 3], 'a:${assignment.workspaceLeaseSlot}') ~= ARGV[4] then return -2 end`, + `if redis.call('HGET', KEYS[keyCount + 3], 'i:${assignment.workspaceLeaseSlot}') ~= ARGV[1] then return -2 end`, + ]), + ]), "if redis.call('GET', KEYS[1]) ~= ARGV[1] then return 0 end", 'if ARGV[7] ~= "" and redis.call(\'GET\', KEYS[6]) ~= ARGV[7] then return 0 end', - "if #KEYS >= 7 and redis.call('EXISTS', KEYS[7]) == 1 then return -1 end", + 'if keyCount >= 7 and redis.call(\'EXISTS\', KEYS[7]) == 1 then return -1 end', + // A lane refuses a quarantined checkout. A checkout reaches enqueue only once no + // lane beneath it holds a slot, so any lane fence still indexed here is stuck. + "if ARGV[9] == 'lane' and redis.call('EXISTS', KEYS[10]) == 1 then return -1 end", + "if ARGV[9] == 'checkout' then", + ' for i = 11, keyCount do if redis.call(\'EXISTS\', KEYS[i]) == 1 then return -1 end end', + ' for i = 11, keyCount do redis.call(\'SREM\', KEYS[10], KEYS[i]) end', + 'end', 'redis.call(\'SET\', KEYS[2], ARGV[2], \"EX\", ARGV[3])', "redis.call('RPUSH', KEYS[3], ARGV[4])", "redis.call('EXPIRE', KEYS[3], ARGV[3])", @@ -2168,14 +2477,18 @@ export class RedisBridgeStore { ? ['redis.call(\'SET\', KEYS[4], ARGV[1], \"PX\", ARGV[5])'] : []), 'redis.call(\'SET\', KEYS[5], "1", \"PXAT\", ARGV[6])', - "if #KEYS >= 7 then redis.call('SET', KEYS[7], ARGV[4]) end", - 'if #KEYS == 9 then', + 'if keyCount >= 7 then redis.call(\'SET\', KEYS[7], ARGV[4]) end', + "if ARGV[9] == 'lane' then redis.call('SADD', KEYS[11], KEYS[7]) end", + 'if keyCount >= 9 then', " local epoch = redis.call('GET', KEYS[9])", - " if type(epoch) ~= 'string' then epoch = '0'; redis.call('SET', KEYS[9], epoch, 'EX', ARGV[3]) end", - " if redis.call('PTTL', KEYS[9]) < tonumber(ARGV[3]) * 1000 then redis.call('EXPIRE', KEYS[9], ARGV[3]) end", + ' if type(epoch) ~= \'string\' then epoch = \'0\'; redis.call(\'SET\', KEYS[9], epoch, \'EX\', ARGV[12]) end', + ' if redis.call(\'PTTL\', KEYS[9]) < tonumber(ARGV[12]) * 1000 then redis.call(\'EXPIRE\', KEYS[9], ARGV[12]) end', " redis.call('HSET', KEYS[8], 'metadata', ARGV[8], 'epoch', epoch)", - " redis.call('EXPIRE', KEYS[8], ARGV[3])", + ' redis.call(\'EXPIRE\', KEYS[8], ARGV[12])', 'end', + ...(guard == null ? [] : [ + 'redis.call(\'HSET\', KEYS[keyCount + 1], \'state\', \'admitted\', \'assignment\', ARGV[2], \'admittedAtMs\', admissionNowMs)', + ]), 'return 1', ].join('\n'); const keys = [ @@ -2208,13 +2521,36 @@ export class RedisBridgeStore { leaseTokenHash: assignment.leaseTokenHash, workerIdentityId: assignment.workerIdentityId, expiresAt: assignment.expiresAt, + durableRequestKey: assignment.durableRequestKey, }; + let fenceScope: '' | 'lane' | 'checkout' = ''; if (assignment.workspaceLeaseSlot !== undefined) { keys.push( workspaceFenceReceiptKey(assignment.assignmentId), `${workspaceQuarantineKey(assignment.workerId, assignment.workspaceFence!)}:epoch`, ); + const parent = workspaceFenceParent(assignment.workspaceFence!); + if (parent !== undefined) { + const parentFence = workspaceQuarantineKey(assignment.workerId, parent); + keys.push(parentFence, workspaceLaneFenceIndexKey(parentFence)); + fenceScope = 'lane'; + } else { + const index = workspaceLaneFenceIndexKey( + workspaceQuarantineKey(assignment.workerId, assignment.workspaceFence!), + ); + const laneFences = await boundedCommand( + this.redis.smembers(index), + this.redisCommandTimeoutMs, + 'Bridge linked worktree fence index read', + ); + keys.push(index, ...laneFences); + fenceScope = 'checkout'; + } } + const keyCount = keys.length; + if (guard != null) keys.push(guard.key, guard.claimKey, + assignment.workspaceLeaseSlot === undefined ? lockKey(assignment.workerId) + : `${workerKey(assignment.workerId)}:workspace-slots`); const result = await this.redis.eval( script, keys.length, @@ -2227,6 +2563,10 @@ export class RedisBridgeStore { String(Date.parse(assignment.expiresAt)), readyToken ?? '', JSON.stringify(receipt), + fenceScope, + keyCount, + guard?.token ?? '', + guard == null ? ttlSeconds : assignmentOutcomeTtlSeconds(assignment), ); if (Number(result) === -1) { throw new BridgeStoreError( @@ -2242,21 +2582,31 @@ export class RedisBridgeStore { assignmentId: string, incarnationId: string, ttlSeconds: number, + reuseOwned = false, + guard?: DurableWorkspaceGuard, ): Promise { const script = [ + ...(guard == null ? [] : [ + ...durableAdmissionFence('KEYS[3]', 'KEYS[4]', 'ARGV[5]', 0), + 'if redis.call(\'GET\', KEYS[5]) ~= ARGV[2] then return 0 end', + ]), + 'if ARGV[4] == \'1\' and redis.call(\'GET\', KEYS[1]) == ARGV[1] and redis.call(\'GET\', KEYS[2]) == ARGV[2] then redis.call(\'PEXPIRE\', KEYS[1], ARGV[3]); redis.call(\'PEXPIRE\', KEYS[2], ARGV[3]); return 1 end', 'if redis.call(\'EXISTS\', KEYS[1]) == 1 then return 0 end', 'redis.call(\'SET\', KEYS[1], ARGV[1], \"PX\", ARGV[3])', 'redis.call(\'SET\', KEYS[2], ARGV[2], \"PX\", ARGV[3])', 'return 1', ].join('\n'); + const keys = [lockKey(workerId), lockIncarnationKey(workerId)]; + if (guard != null) keys.push(guard.key, guard.claimKey, workerIncarnationKey(workerId)); const result = await this.redis.eval( script, - 2, - lockKey(workerId), - lockIncarnationKey(workerId), + keys.length, + ...keys, assignmentId, incarnationId, String(ttlSeconds * 1000), + reuseOwned ? '1' : '0', + guard?.token ?? '', ); return Number(result) === 1; } @@ -2304,6 +2654,7 @@ export class RedisBridgeStore { private async commitPendingWorkspace( assignment: StoredAssignment, settlement: AnyCodeBridgeSettlement, + durableKey?: string, ): Promise { if (assignment.workspaceLeaseSlot !== undefined) { const committed = Number( @@ -2342,8 +2693,17 @@ export class RedisBridgeStore { } const runtimeSessionId = assignmentWorkspace(assignment)!; const script = [ + ...(durableKey == null ? [] : [ + 'if redis.call(\'EXISTS\', KEYS[2]) == 1 then return 1 end', + ]), "if redis.call('GET', KEYS[1]) == ARGV[1] then", - " return redis.call('DEL', KEYS[1])", + ...(durableKey == null + ? [' return redis.call(\'DEL\', KEYS[1])'] + : [ + ' redis.call(\'DEL\', KEYS[1])', + ' redis.call(\'SET\', KEYS[2], \'1\', \'PX\', 86400000)', + ' return 1', + ]), 'end', 'return 0', ].join('\n'); @@ -2351,8 +2711,9 @@ export class RedisBridgeStore { await boundedCommand( this.redis.eval( script, - 1, + durableKey == null ? 1 : 2, workspaceQuarantineKey(assignment.workerId, runtimeSessionId), + ...(durableKey == null ? [] : [`${durableKey}:committed:${assignment.assignmentId}`]), assignment.assignmentId, ), // Once settlement wins, caller cancellation must not prevent its diff --git a/service/src/bridge/worker-admission.test.ts b/service/src/bridge/worker-admission.test.ts index 655748b1..c77ac19e 100644 --- a/service/src/bridge/worker-admission.test.ts +++ b/service/src/bridge/worker-admission.test.ts @@ -124,6 +124,32 @@ test('an expired queued call never reaches the worker and does not strand later await third; }); +test('an already-expired workspace deadline is a definite queue timeout before registration', async () => { + await register(); + await expect(dispatch('expired-before-read', new AbortController(), -1)).rejects.toMatchObject({ + code: 'WORKSPACE_QUEUE_TIMEOUT', + }); + expect(await redis.zcard(`codeapi:bridge:v1:worker:${workerId}:admission`)).toBe(0); + expect(await store.lease(workerId, incarnationId, 20)).toBeUndefined(); +}); + +test('expiry during the registration read never becomes an ambiguous assignment error', async () => { + await register(); + const registrationRead = spyOn(redis, 'mget').mockImplementation(async () => { + await new Promise(resolve => setTimeout(resolve, 25)); + throw new Error('registration read outlived its queue budget'); + }); + try { + await expect(dispatch('expired-during-read', new AbortController(), 1)).rejects.toMatchObject({ + code: 'WORKSPACE_QUEUE_TIMEOUT', + }); + } finally { + registrationRead.mockRestore(); + } + expect(await redis.zcard(`codeapi:bridge:v1:worker:${workerId}:admission`)).toBe(0); + expect(await store.lease(workerId, incarnationId, 20)).toBeUndefined(); +}); + test('a queued request is rejected if the worker withdraws its capability', async () => { await register(); const first = dispatch('first'); @@ -161,6 +187,72 @@ test('execution receives a fresh budget after waiting and the lock covers long c await second; }); +test('a long queue allowance does not extend serial lock or assignment TTLs after admission', async () => { + await register(); + const active = dispatch('first'); + const activeAssignment = await store.lease(workerId, incarnationId, 1000); + const controller = new AbortController(); + const queued = dispatch('second', controller, 300_000, 305_000); + void queued.catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 150)); + await settle(activeAssignment); + await active; + + const assignment = await store.lease(workerId, incarnationId, 1000); + if (assignment == null) { + controller.abort(); + await queued.catch(() => undefined); + throw new Error('Queued request was not leased'); + } + expect(assignment.request).toMatchObject({ path: 'second' }); + try { + const expiresAtMs = Date.parse(assignment.expiresAt); + for (const key of [ + `codeapi:bridge:v1:worker:${workerId}:lock`, + `codeapi:bridge:v1:worker:${workerId}:lock:incarnation`, + `codeapi:bridge:v1:assignment:${assignment.assignmentId}`, + ]) { + const ttlMs = await redis.pttl(key); + const remainingMs = expiresAtMs - Date.now(); + expect(ttlMs).toBeGreaterThan(remainingMs + 28_000); + expect(ttlMs).toBeLessThan(remainingMs + 32_000); + } + } finally { + await settle(assignment); + await queued; + } +}); + +test('absolute-deadline callers keep only their remaining deadline in the serial lock TTL', async () => { + await register(); + const active = dispatch('first'); + const activeAssignment = await store.lease(workerId, incarnationId, 1000); + const controller = new AbortController(); + const queued = dispatch('absolute', controller, 2_500); + void queued.catch(() => undefined); + await new Promise((resolve) => setTimeout(resolve, 1450)); + await settle(activeAssignment); + await active; + + const assignment = await store.lease(workerId, incarnationId, 1000); + if (assignment == null) { + controller.abort(); + await queued.catch(() => undefined); + throw new Error('Queued request was not leased'); + } + expect(assignment.request).toMatchObject({ path: 'absolute' }); + try { + const remainingMs = Date.parse(assignment.expiresAt) - Date.now(); + expect(remainingMs).toBeGreaterThan(0); + const ttlMs = await redis.pttl(`codeapi:bridge:v1:worker:${workerId}:lock`); + expect(ttlMs).toBeGreaterThan(remainingMs + 28_000); + expect(ttlMs).toBeLessThan(remainingMs + 31_000); + } finally { + await settle(assignment); + await queued; + } +}); + test('execution expires independently of an unused queue allowance', async () => { await register(); const completion = dispatch('short', new AbortController(), 5000, 150); diff --git a/service/src/bridge/workspace-store.test.ts b/service/src/bridge/workspace-store.test.ts index f248aa40..90f68da3 100644 --- a/service/src/bridge/workspace-store.test.ts +++ b/service/src/bridge/workspace-store.test.ts @@ -2,8 +2,9 @@ import { afterEach, expect, test } from 'bun:test'; import RedisMock from 'ioredis-mock'; import type Redis from 'ioredis'; +import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; -import { BRIDGE_PROTOCOL_VERSION } from '../../../packages/code/src/protocol'; +import { BRIDGE_PROTOCOL_VERSION, isValidBridgeWorkerCapabilities } from '../../../packages/code/src/protocol'; import { RedisBridgeStore } from './store'; const redis = new RedisMock() as unknown as Redis; @@ -643,6 +644,131 @@ test('rejects fenced edits from workers without the negotiated feature', async ( expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); }); +const featureGatedEdits: Array<[string, WorkspaceToolRequest]> = [ + [ + 'tolerant matching', + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + matching: 'tolerant', + edits: [{ oldText: 'before', newText: 'after' }], + }, + ], + [ + 'tolerant previews', + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'preview_edit', + workspaceId: 'primary', + path: 'notes.txt', + matching: 'tolerant', + edits: [{ oldText: 'before', newText: 'after' }], + }, + ], + [ + 'replaceAll edits', + { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'edit_file', + workspaceId: 'primary', + path: 'notes.txt', + edits: [{ oldText: 'before', newText: 'after', replaceAll: true }], + }, + ], +]; + +test.each(featureGatedEdits)( + 'rejects %s from workers without the negotiated feature', + async (_label, request) => { + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: true, + sandboxProfile: 'nsjail', + runtimes: ['bash'], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['preview_edit', 'edit_file'], + editFileModes: ['single', 'batch'], + editFileFeatures: ['expected_base_sha256'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + + await expect( + store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request, + deadlineAtMs: Date.now() + 1_000, + signal: new AbortController().signal, + }), + ).rejects.toMatchObject({ code: 'WORKER_MISMATCH' }); + expect(await redis.keys('codeapi:bridge:v1:assignment:*')).toHaveLength(0); + }, +); + +test('preview-only workers can dispatch negotiated tolerant and replace-all previews', async () => { + expect(isValidBridgeWorkerCapabilities({ + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['preview_edit'], + editFileModes: ['single', 'batch'], + editFileFeatures: ['tolerant_match', 'replace_all'], + workspaces: [{ id: 'primary' }], + }, + })).toBe(true); + await store.register({ + protocolVersion: BRIDGE_PROTOCOL_VERSION, + workerId: 'workspace-worker', + incarnationId, + capabilities: { + statefulWorkspace: false, + sandboxProfile: 'native-srt', + runtimes: [], + workspaceTools: { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operations: ['preview_edit'], + editFileModes: ['single', 'batch'], + editFileFeatures: ['tolerant_match', 'replace_all'], + workspaces: [{ id: 'primary' }], + }, + }, + }); + const request = { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + operation: 'preview_edit' as const, + workspaceId: 'primary', + path: 'notes.txt', + matching: 'tolerant' as const, + edits: [{ oldText: 'before', newText: 'after', replaceAll: true }], + }; + const completion = store.dispatchWorkspaceTool({ + workerId: 'workspace-worker', + request, + deadlineAtMs: Date.now() + 5_000, + signal: new AbortController().signal, + }); + const assignment = await store.lease('workspace-worker', incarnationId, 1_000); + expect(assignment).toMatchObject({ executionKind: 'workspace_tool', request }); + await store.settle('workspace-worker', assignment!.assignmentId, { + protocolVersion: BRIDGE_PROTOCOL_VERSION, + generation: assignment!.generation, + leaseToken: assignment!.leaseToken, + incarnationId, + status: 'rejected', + error: 'preview tested', + }); + await expect(completion).resolves.toMatchObject({ status: 'rejected', error: 'preview tested' }); +}); + test('rejects batch previews from workers without the negotiated mode', async () => { await store.register({ protocolVersion: BRIDGE_PROTOCOL_VERSION, diff --git a/service/src/config.spec.ts b/service/src/config.spec.ts index 69b416d8..3c2ca9d2 100644 --- a/service/src/config.spec.ts +++ b/service/src/config.spec.ts @@ -67,15 +67,15 @@ describe('runtime version configuration', () => { }); }); - it('maps Bun JavaScript and TypeScript requests to Bun 1.3.14', () => { + it('maps Bun JavaScript and TypeScript requests to Bun 1.4.2', () => { expect(languageConfig[Languages.js]).toMatchObject({ language: 'bun-js', - version: '1.3.14', + version: '1.4.2', fileName: 'index.js', }); expect(languageConfig[Languages.ts]).toMatchObject({ language: 'bun-ts', - version: '1.3.14', + version: '1.4.2', fileName: 'main.ts', }); }); diff --git a/service/src/config.ts b/service/src/config.ts index 55570dfe..b3e397b8 100644 --- a/service/src/config.ts +++ b/service/src/config.ts @@ -10,10 +10,10 @@ import { export const languageConfig: Record = { [Languages.bash]: { language: 'bash', version: '5.2.0', fileName: 'script.sh' }, - [Languages.js]: { language: 'bun-js', version: '1.3.14', fileName: 'index.js' }, + [Languages.js]: { language: 'bun-js', version: '1.4.2', fileName: 'index.js' }, [Languages.node]: { language: 'node', version: '24.15.0', fileName: 'index.js' }, [Languages.py]: { language: 'python', version: '3.14.4', fileName: 'main.py' }, - [Languages.ts]: { language: 'bun-ts', version: '1.3.14', fileName: 'main.ts' }, + [Languages.ts]: { language: 'bun-ts', version: '1.4.2', fileName: 'main.ts' }, }; const languageAliases: Record = { @@ -420,12 +420,33 @@ export const env = { BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: Number( process.env.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS ?? 1, ), + /** + * Offer `lane_git` to workers and accept `laneGit` (`{ branch, head }`) on command results. + * Off by default: enable only after the LibreChat talking to this service accepts the field. + */ + BRIDGE_LANE_GIT: process.env.CODEAPI_BRIDGE_LANE_GIT === 'true', /** Outbound worker selected by the remote-bridge backend. */ BRIDGE_WORKER_ID: process.env.CODEAPI_BRIDGE_WORKER_ID ?? '', /** Static compatibility auth or short-lived proof-of-possession credentials. */ BRIDGE_AUTH_MODE: bridgeAuthMode, /** Enrollment and lease credential shared only with the configured worker. */ BRIDGE_TOKEN: process.env.CODEAPI_BRIDGE_TOKEN ?? '', + /** Opt-in stable HTTPS origin of this Code API deployment (shared by all replicas). */ + BRIDGE_RECOVERY_SERVER_ID: process.env.CODEAPI_BRIDGE_RECOVERY_SERVER_ID ?? '', + /** Zero preserves machine authorization until explicit revocation. */ + BRIDGE_ENROLLMENT_TTL_SECONDS: Number(process.env.CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS ?? 0), + BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: Number( + process.env.CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS ?? 60, + ), + BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE: Number( + process.env.CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE ?? 12, + ), + BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE: Number( + process.env.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE ?? 30, + ), + BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE: Number( + process.env.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE ?? 240, + ), /** * Runtime session affinity for stateful sandbox backends. * - `stateless` (default): no runtime sessions; `runtime_session_hint` ignored. diff --git a/service/src/egress-gateway-client.test.ts b/service/src/egress-gateway-client.test.ts index c3d7ab9c..ef9e3fb7 100644 --- a/service/src/egress-gateway-client.test.ts +++ b/service/src/egress-gateway-client.test.ts @@ -21,6 +21,7 @@ type AxiosPostCall = { headers?: Record; signal?: AbortSignal; timeout?: number; + maxRedirects?: number; }; }; @@ -109,6 +110,7 @@ describe('egress gateway client', () => { allowedToolNames: ['query_clickhouse'], }, { signal: controller.signal }); + expect(calls.map(call => call.config.maxRedirects)).toEqual([0, 0, 0]); expect(calls.map(call => call.config.timeout)).toEqual([12_345, 12_345, 12_345]); expect(calls.map(call => call.config.signal)).toEqual([ controller.signal, @@ -147,6 +149,7 @@ describe('egress gateway client', () => { await revokeGatewayEgressGrant({ grantId: 'grant_123', reason: 'completed' }); await revokeGatewayEgressGrant({ grantId: 'grant_123', reason: 'failed', timeoutMs: 321 }); + expect(calls.map(call => call.config.maxRedirects)).toEqual([0, 0]); expect(calls.map(call => call.config.timeout)).toEqual([987, 321]); }); diff --git a/service/src/egress-gateway-client.ts b/service/src/egress-gateway-client.ts index ebcef4ad..67b7dca6 100644 --- a/service/src/egress-gateway-client.ts +++ b/service/src/egress-gateway-client.ts @@ -48,6 +48,7 @@ export async function createGatewayEgressGrant(args: { gatewayUrl('/internal/egress-grants'), body, { + maxRedirects: 0, headers: injectTraceHeaders(gatewayHeaders({ 'Content-Type': 'application/json' }, isSynthetic)), signal, timeout: env.EGRESS_GATEWAY_REQUEST_TIMEOUT_MS, @@ -73,6 +74,7 @@ export async function restoreGatewaySandboxResult { }); describe('egress gateway routes', () => { + test('rejects file-server redirects before forwarding internal credentials', async () => { + let redirected = 0; + let contacted = 0; + const target = Bun.serve({ + port: 0, + fetch() { + redirected++; + return new Response('unexpected'); + }, + }); + const upstream = Bun.serve({ + port: 0, + fetch(req) { + contacted++; + expect(req.headers.get(INTERNAL_SERVICE_TOKEN_HEADER)).toBe(INTERNAL_TOKEN); + return new Response(null, { + status: 307, + headers: { Location: `http://127.0.0.1:${target.port}/stolen` }, + }); + }, + }); + try { + globalThis.fetch = originalFetch; + env.EGRESS_GATEWAY_FILE_SERVER_URL = `http://127.0.0.1:${upstream.port}`; + const response = await gatewayFetch( + `/sessions/${sessionHandle({ dir: 'read', sessionId: 'sess_input' })}/objects/${objectHandle({})}`, + { headers: grantHeader() }, + ); + expect(response.status).toBe(500); + expect(contacted).toBe(1); + expect(redirected).toBe(0); + } finally { + upstream.stop(true); + target.stop(true); + } + }); + test('protects internal grant create, restore, and revoke routes', async () => { const createBody = JSON.stringify({ payload: payload(), claims: executionClaims() }); const unauthorized = await gatewayFetch('/internal/egress-grants', { diff --git a/service/src/egress-gateway.ts b/service/src/egress-gateway.ts index 873db173..2af379c1 100644 --- a/service/src/egress-gateway.ts +++ b/service/src/egress-gateway.ts @@ -544,7 +544,7 @@ app.get('/sessions/:sessionHandle/objects', async (req, res) => { `/sessions/${encodeURIComponent(sessionId)}/objects`, '?detail=normalized', ), - { headers: injectTraceHeaders(internalServiceHeaders({ Accept: 'application/json' })) }, + { redirect: 'error', headers: injectTraceHeaders(internalServiceHeaders({ Accept: 'application/json' })) }, ); if (!upstream.ok) { return pipeFetchResponse(upstream, res); @@ -638,7 +638,7 @@ app.post('/input-manifest', express.json({ limit: '4mb' }), async (req, res) => controller.signal.throwIfAborted(); const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_FILE_SERVER_URL, `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(objectId)}/metadata`), - { headers: injectTraceHeaders(internalServiceHeaders()), signal: controller.signal }); + { redirect: 'error', headers: injectTraceHeaders(internalServiceHeaders()), signal: controller.signal }); if (!upstream.ok) { await upstream.body?.cancel(); // Recheck failures individually through the existing retry/classification path. @@ -668,7 +668,7 @@ app.get('/sessions/:sessionHandle/objects/:objectHandle/metadata', async (req, r await recordEgressRead(grant); const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_FILE_SERVER_URL, `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(object.id)}/metadata`), - { headers: injectTraceHeaders(internalServiceHeaders()) }); + { redirect: 'error', headers: injectTraceHeaders(internalServiceHeaders()) }); if (!upstream.ok) return pipeFetchResponse(upstream, res); return res.json(inputMetadata(await upstream.json(), grant, sessionId, object.id)); } catch (error) { @@ -694,7 +694,7 @@ app.get('/sessions/:sessionHandle/objects/:objectHandle', async (req, res) => { env.EGRESS_GATEWAY_FILE_SERVER_URL, `/sessions/${encodeURIComponent(sessionId)}/objects/${encodeURIComponent(object.id)}`, ), - { headers: injectTraceHeaders(internalServiceHeaders(expectedVersion ? { 'X-CodeAPI-Input-Version': expectedVersion } : {})) }, + { redirect: 'error', headers: injectTraceHeaders(internalServiceHeaders(expectedVersion ? { 'X-CodeAPI-Input-Version': expectedVersion } : {})) }, ); const headerOverrides = isOpaqueObjectContentDisposition( upstream.headers.get('content-disposition'), @@ -750,6 +750,7 @@ app.put('/sessions/:sessionHandle/objects/:fileId', async (req, res) => { ), { method: 'PUT', + redirect: 'error', headers, body: req as unknown as BodyInit, duplex: 'half', @@ -784,8 +785,8 @@ app.post('/tool-call', async (req, res) => { if (!executionId || !callId || !opaqueCallbackToken) { /* Generic 404 so a sandbox attacker probing paths cannot tell that * `/tool-call` is the live route by reading the error body. The - * body and headers must MATCH the tool-call socket proxy's own 404 - * for unknown paths (see api/src/tool-call-socket-proxy.ts) byte- + * body and headers must match the generic unknown-route response + * for unknown paths byte- * for-byte — case included — otherwise the sandbox can still * fingerprint the real route by comparing response shapes. */ logger.warn( @@ -832,6 +833,7 @@ app.post('/tool-call', async (req, res) => { await recordEgressToolCall(callback.grant_id, executionId); const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_TOOL_CALL_SERVER_URL, '/tool-call'), { method: 'POST', + redirect: 'error', headers: { ...injectTraceHeaders(), 'Content-Type': req.header('content-type') ?? 'application/json', @@ -847,6 +849,7 @@ app.post('/tool-call', async (req, res) => { await recordEgressToolCall(callback.grant_id, executionId); const upstream = await fetch(forwardUrl(env.EGRESS_GATEWAY_TOOL_CALL_SERVER_URL, '/tool-call'), { method: 'POST', + redirect: 'error', headers: { ...injectTraceHeaders(), 'Content-Type': req.header('content-type') ?? 'application/json', diff --git a/service/src/file-server.ts b/service/src/file-server.ts index b96392f3..00b91e4a 100644 --- a/service/src/file-server.ts +++ b/service/src/file-server.ts @@ -11,11 +11,11 @@ import { sendFileDownload } from './file-download'; import path from 'path'; import IORedis from 'ioredis'; import express from 'express'; +import { createMinioClient } from './minio-client'; import { nanoid } from 'nanoid'; import { PassThrough } from 'stream'; import { pipeline } from 'stream/promises'; -import { Client } from 'minio'; -import type { BucketItem, BucketItemStat, ClientOptions } from 'minio'; +import type { BucketItem, BucketItemStat, Client } from 'minio'; import type { Readable } from 'stream'; import type * as tls from 'tls'; import type * as t from './types'; @@ -40,94 +40,6 @@ app.use(httpMetricsMiddleware); const bucketName = process.env.MINIO_BUCKET ?? 'test-bucket'; -type IamProviderModule = { IamAwsProvider?: new (opts: object) => unknown; default?: new (opts: object) => unknown }; - -async function createMinioClient(): Promise { - const irsaExplicit = process.env.MINIO_USE_IRSA?.toLowerCase() === 'true'; - const irsaEnvVars = Boolean(process.env.AWS_WEB_IDENTITY_TOKEN_FILE) && Boolean(process.env.AWS_ROLE_ARN); - const useIrsa = irsaExplicit || irsaEnvVars; - - const baseConfig: ClientOptions = { - endPoint: process.env.MINIO_ENDPOINT ?? 'localhost', - port: process.env.MINIO_NO_PORT?.toLowerCase() === 'true' ? undefined : parseInt(process.env.MINIO_PORT ?? '9000'), - useSSL: process.env.MINIO_USE_SSL?.toLowerCase() === 'true', - region: process.env.MINIO_REGION ?? process.env.AWS_REGION ?? 'us-east-1', - }; - - if (useIrsa) { - logger.info('Using IRSA (IamAwsProvider) for S3 authentication', { - tokenFile: process.env.AWS_WEB_IDENTITY_TOKEN_FILE, - roleArn: process.env.AWS_ROLE_ARN, - region: baseConfig.region, - }); - - /** IamAwsProvider exists in minio 8.0.6+ but isn't exported from main module - * Try multiple import paths for compatibility with different runtimes (bun, ts-node, node) - */ - let IamAwsProviderClass: new (opts: object) => unknown; - try { - const mod = await import('minio/dist/main/IamAwsProvider.js') as IamProviderModule; - IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; - } catch (primaryError) { - try { - // Fallback for bun: resolve path using require if available (CJS context) - let resolvePath = 'node_modules/minio/'; - try { - // eslint-disable-next-line @typescript-eslint/no-require-imports - resolvePath = require.resolve('minio').replace(/dist\/.*$/, ''); - } catch { - // require.resolve not available (ESM context), use default path - } - const mod = await import(`${resolvePath}dist/main/IamAwsProvider.js`) as IamProviderModule; - IamAwsProviderClass = (mod.IamAwsProvider ?? mod.default)!; - } catch (fallbackError) { - logger.error('Failed to load IamAwsProvider', { primaryError, fallbackError }); - throw new Error('Could not load IamAwsProvider for IRSA authentication. Ensure minio >= 8.0.6 is installed.'); - } - } - - const credentialsProvider = new IamAwsProviderClass({}); - - return new Client({ - ...baseConfig, - credentialsProvider: credentialsProvider as ClientOptions['credentialsProvider'], - }); - } - - if (!process.env.MINIO_ACCESS_KEY) { - /** No static pair and no IRSA: resolve through the AWS SDK default chain - * (ECS task role, EC2 instance profile, env, profile). minio's own - * IamAwsProvider cannot read Fargate container credentials. */ - logger.info('Using the AWS SDK default credential chain for MinIO/S3 authentication', { - region: baseConfig.region, - }); - const { S3Client } = await import('@aws-sdk/client-s3'); - const sdk = new S3Client({ region: baseConfig.region }); - const credentialsProvider = { - async getCredentials() { - const c = await sdk.config.credentials(); - return { - getAccessKey: () => c.accessKeyId, - getSecretKey: () => c.secretAccessKey, - getSessionToken: () => c.sessionToken, - }; - }, - }; - return new Client({ - ...baseConfig, - credentialsProvider: credentialsProvider as unknown as ClientOptions['credentialsProvider'], - }); - } - - logger.info('Using explicit credentials for MinIO/S3 authentication'); - return new Client({ - ...baseConfig, - accessKey: process.env.MINIO_ACCESS_KEY ?? '', - secretKey: process.env.MINIO_SECRET_KEY ?? '', - sessionToken: process.env.MINIO_SESSION_TOKEN, - }); -} - let minioClient: Client; let storageInitialized = false; diff --git a/service/src/hosted-app/microvm-runtime.ts b/service/src/hosted-app/microvm-runtime.ts index bc0ac61b..3cb10941 100644 --- a/service/src/hosted-app/microvm-runtime.ts +++ b/service/src/hosted-app/microvm-runtime.ts @@ -380,6 +380,7 @@ export class HostedAppMicrovmRuntime { token = await this.mintToken(vm.microvmId, this.config.controlPort, callerSignal); } const response = await this.deps.fetch(`${endpoint}/api/v2/health`, { + redirect: 'error', headers: { [token.headerName]: token.token, ...microvmPortHeaders(this.config.controlPort), @@ -417,6 +418,7 @@ export class HostedAppMicrovmRuntime { `${normalizeHostedAppMicrovmEndpoint(vm.endpoint ?? '')}/api/v2/hosted-app/start`, { method: 'POST', + redirect: 'error', headers: { [token.headerName]: token.token, ...microvmPortHeaders(this.config.controlPort), @@ -457,6 +459,7 @@ export class HostedAppMicrovmRuntime { const response = await this.deps.fetch( `${normalizeHostedAppMicrovmEndpoint(vm.endpoint ?? '')}/api/v2/hosted-app/status`, { + redirect: 'error', headers: { [token.headerName]: token.token, ...microvmPortHeaders(this.config.controlPort), diff --git a/service/src/internal-redirect-policy.test.ts b/service/src/internal-redirect-policy.test.ts new file mode 100644 index 00000000..577b468c --- /dev/null +++ b/service/src/internal-redirect-policy.test.ts @@ -0,0 +1,56 @@ +import { describe, expect, test } from 'bun:test'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as ts from 'typescript'; + +const transports: Record = { + 'egress-gateway-client.ts': 4, + 'egress-gateway.ts': 7, + 'hosted-app/microvm-runtime.ts': 3, + 'runtime-session/checkpoint.ts': 4, + 'runtime-session/files.ts': 1, + 'sandbox-backend/http.ts': 1, + 'sandbox-backend/lambda-microvm.ts': 2, + 'service/programmatic-router.ts': 3, + 'service/replay-state.ts': 2, + 'service/router.ts': 5, +}; + +// Every request to these internal endpoints must stay on its original origin. +describe('internal transport redirect policy', () => { + for (const [file, count] of Object.entries(transports)) { + test(`${file} rejects redirects on every internal request`, () => { + const source = ts.createSourceFile(file, + fs.readFileSync(path.join(__dirname, file), 'utf8'), ts.ScriptTarget.Latest, true); + let requests = 0; + function visit(node: ts.Node): void { + if (ts.isCallExpression(node)) { + const name = node.expression.getText(source); + const fetchCall = name === 'fetch' || name === 'this.deps.fetch'; + const axiosCall = name === 'axios' || /^axios\.(get|post|delete)$/.test(name); + if (fetchCall || axiosCall) { + requests++; + let index = 1; + if (name === 'axios') index = 0; + if (name === 'axios.post') index = 2; + let options = node.arguments[index]; + while (options && (ts.isAsExpression(options) || ts.isTypeAssertionExpression(options))) { + options = options.expression; + } + expect(options && ts.isObjectLiteralExpression(options)).toBe(true); + if (!options || !ts.isObjectLiteralExpression(options)) return; + const property = options.properties.find(prop => ts.isPropertyAssignment(prop) + && prop.name.getText(source) === (fetchCall ? 'redirect' : 'maxRedirects')); + expect(property && ts.isPropertyAssignment(property)).toBe(true); + if (property && ts.isPropertyAssignment(property)) { + expect(property.initializer.getText(source)).toBe(fetchCall ? '\'error\'' : '0'); + } + } + } + ts.forEachChild(node, visit); + } + visit(source); + expect(requests).toBe(count); + }); + } +}); diff --git a/service/src/minio-client.ts b/service/src/minio-client.ts index 6cc0f54d..4c6dd5ab 100644 --- a/service/src/minio-client.ts +++ b/service/src/minio-client.ts @@ -58,6 +58,30 @@ export async function createMinioClient(): Promise { }); } + if (!process.env.MINIO_ACCESS_KEY) { + /** No static pair and no IRSA: use the AWS SDK default chain, which supports + * ECS task roles as well as EC2 instance profiles, environment, and profiles. */ + logger.info('Using the AWS SDK default credential chain for MinIO/S3 authentication', { + region: baseConfig.region, + }); + const { S3Client } = await import('@aws-sdk/client-s3'); + const sdk = new S3Client({ region: baseConfig.region }); + const credentialsProvider = { + async getCredentials() { + const c = await sdk.config.credentials(); + return { + getAccessKey: () => c.accessKeyId, + getSecretKey: () => c.secretAccessKey, + getSessionToken: () => c.sessionToken, + }; + }, + }; + return new Client({ + ...baseConfig, + credentialsProvider: credentialsProvider as unknown as ClientOptions['credentialsProvider'], + }); + } + logger.info('Using explicit credentials for MinIO/S3 authentication'); return new Client({ ...baseConfig, diff --git a/service/src/minio-compat.test.ts b/service/src/minio-compat.test.ts new file mode 100644 index 00000000..255c6e00 --- /dev/null +++ b/service/src/minio-compat.test.ts @@ -0,0 +1,39 @@ +import { expect, test } from 'bun:test'; +import { Client } from 'minio'; + +test('patched MinIO preserves S3 object-list XML decoding', async () => { + const server = Bun.serve({ + port: 0, + fetch(req) { + const url = new URL(req.url); + expect(url.pathname).toBe('/testbucket'); + expect(url.searchParams.get('list-type')).toBe('2'); + return new Response(` + + testbucketinputs/1 + 1000false + inputs/a&b.csv2026-01-01T00:00:00.000Z + "test-etag"12STANDARD + +`, { headers: { 'Content-Type': 'application/xml' } }); + }, + }); + try { + const client = new Client({ + endPoint: '127.0.0.1', + port: server.port, + useSSL: false, + region: 'us-east-1', + accessKey: 'test-access-key', + secretKey: 'test-secret-key', + }); + const objects = []; + for await (const object of client.listObjectsV2('testbucket', 'inputs/', true)) { + objects.push(object); + } + expect(objects).toHaveLength(1); + expect(objects[0]).toMatchObject({ name: 'inputs/a&b.csv', size: 12, etag: 'test-etag' }); + } finally { + server.stop(true); + } +}); diff --git a/service/src/preamble.test.ts b/service/src/preamble.test.ts index 803e1839..b4293fa5 100644 --- a/service/src/preamble.test.ts +++ b/service/src/preamble.test.ts @@ -15,71 +15,16 @@ const baseConfig = { tools: [], }; -describe('generatePreamble — Unix-vs-TCP transport gate', () => { - /* Codex audit follow-up: a path-existence check (`os.path.exists`) on - * /tmp/tcs.sock was user-spoofable inside the sandbox — in - * legacy/no-proxy mode, user code could `open('/tmp/tcs.sock', 'w')` - * and force the next tool call into the unix-socket branch (where - * connect() then fails with ENOTSOCK) instead of falling back to TCP. - * The fix probes a real AF_UNIX connect() at preamble import time - * (before user code runs) and caches the verdict. These tests assert - * the rendered preamble keeps that property. */ - - test('renders a probe that performs an AF_UNIX connect, not a path-existence check', () => { - const preamble = generatePreamble(baseConfig); - expect(preamble).toContain('def _probe_tool_call_socket():'); - /* The probe must construct an AF_UNIX socket and call connect(). A - * file-existence check would not detect a regular file or stale - * inode planted by the user. */ - expect(preamble).toMatch(/AF_UNIX/); - expect(preamble).toMatch(/\.connect\(_TOOL_CALL_SOCKET\)/); - /* Regression guard against reintroducing the user-spoofable check. */ - expect(preamble).not.toMatch( - /if\s+os\.path\.exists\(_TOOL_CALL_SOCKET\)/, - ); - }); - - test('caches the probe verdict at module load (before user code can plant a spoof)', () => { +describe('generatePreamble — anonymous pipe transport', () => { + test('uses fixed FIFO descriptors and fails closed without a pipe-enabled runner', () => { const preamble = generatePreamble(baseConfig); - /* The probe call must appear at top level of the preamble, NOT - * inside _do_request. Otherwise a user could plant a regular file - * at the path between calls and flip the gate per-request. */ - const probeCallIdx = preamble.indexOf( - '_USE_TOOL_CALL_SOCKET = _probe_tool_call_socket()', - ); - expect(probeCallIdx).toBeGreaterThan(-1); - /* _do_request must consult the cached verdict, not re-probe. */ - const doReqMatch = preamble.match( - /def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/, - ); - expect(doReqMatch).not.toBeNull(); - expect(doReqMatch![0]).toContain('_USE_TOOL_CALL_SOCKET'); - expect(doReqMatch![0]).not.toContain('_probe_tool_call_socket('); - }); - - test('keeps the TCP fallback path for legacy / no-proxy deployments', () => { - const preamble = generatePreamble(baseConfig); - /* The fallback must still construct the URL from _CALLBACK_URL and - * delegate to _tcp_request. Without this, runners without the - * proxy bind-mount would have no way to reach the orchestrator. */ - const doReqMatch = preamble.match( - /def\s+_do_request[\s\S]*?(?=\ndef\s|\nclass\s|\Z)/, - ); - expect(doReqMatch).not.toBeNull(); - expect(doReqMatch![0]).toContain('_CALLBACK_URL + path'); - expect(doReqMatch![0]).toContain('_tcp_request('); - }); - - test('the literal socket path stays inside the preamble, not in os.environ', () => { - const preamble = generatePreamble(baseConfig); - /* Hardening item from PR #1648: TOOL_CALL_SOCKET is no longer - * exported. The path must remain hardcoded so the preamble does - * not depend on env-var injection. */ - expect(preamble).toContain('_TOOL_CALL_SOCKET = "/tmp/tcs.sock"'); - expect(preamble).not.toMatch( - /os\.environ\.get\(['"]TOOL_CALL_SOCKET['"]/, - ); - expect(preamble).not.toMatch(/os\.environ\[['"]TOOL_CALL_SOCKET['"]\]/); + expect(preamble).toContain('_PIPE_WRITE_FD = 3'); + expect(preamble).toContain('_PIPE_READ_FD = 4'); + expect(preamble).toContain('_stat.S_ISFIFO(os.fstat(_fd).st_mode)'); + expect(preamble).toContain('blocking tool calls require a pipe-enabled runner'); + expect(preamble).not.toContain('_tcp_request'); + expect(preamble).not.toContain('/tmp/tcs.sock'); + expect(preamble).not.toContain('import socket'); }); }); @@ -131,14 +76,14 @@ describe('extractPendingFromStdout — input hash metadata', () => { }); }); -describe('createProgrammaticPayload — tool-call socket opt-in', () => { +describe('createProgrammaticPayload — tool-call pipe capability', () => { const req = { body: { code: 'print("ok")', }, } as Parameters[0]['req']; - test('requests the sandbox socket for blocking PTC only', () => { + test('requests the sandbox pipe capability for blocking PTC only', () => { const payload = createProgrammaticPayload({ req, session_id: 'session-blocking', @@ -152,7 +97,7 @@ describe('createProgrammaticPayload — tool-call socket opt-in', () => { expect(payload.tool_call_socket).toBe(true); }); - test('does not request the sandbox socket for replay PTC', () => { + test('does not request the sandbox pipe capability for replay PTC', () => { const payload = createProgrammaticPayload({ req, session_id: 'session-replay', diff --git a/service/src/preamble.ts b/service/src/preamble.ts index 94685c6c..4e8003c2 100644 --- a/service/src/preamble.ts +++ b/service/src/preamble.ts @@ -324,7 +324,7 @@ ${nameComment} _input = {${inputDict}} * Generates dual-mode stubs that work with or without await */ export function generatePreamble(config: PreambleConfig): string { - const { callbackUrl, callbackToken, executionId, tools } = config; + const { callbackToken, executionId, tools } = config; // Header with HTTP callback infrastructure (dual-mode version) let preamble = ` @@ -337,73 +337,137 @@ import json import sys import os import asyncio -import socket as _socket -import http.client as _http_client from typing import Any, Dict, List, Optional, Union -from urllib import request, error as urllib_error -_CALLBACK_URL = "${callbackUrl}" -_CALLBACK_TOKEN = "${callbackToken}" -_EXECUTION_ID = "${executionId}" +_CALLBACK_TOKEN = ${JSON.stringify(callbackToken)} +_EXECUTION_ID = ${JSON.stringify(executionId)} _TOOL_CALL_COUNTER = 0 _TOOL_CALL_LOCK = asyncio.Lock() -# The proxy bind-mounts this fixed path into every sandbox; the env var -# version was dropped to keep the path off os.environ introspection. -_TOOL_CALL_SOCKET = "/tmp/tcs.sock" - -class _UnixHTTPConnection(_http_client.HTTPConnection): - """HTTPConnection that tunnels through a Unix domain socket""" - def __init__(self, socket_path): - super().__init__("localhost") - self._socket_path = socket_path - def connect(self): - self.sock = _socket.socket(_socket.AF_UNIX, _socket.SOCK_STREAM) - self.sock.settimeout(self.timeout) - self.sock.connect(self._socket_path) - -def _unix_request(method, path, body=None, headers=None, timeout=300): - conn = _UnixHTTPConnection(_TOOL_CALL_SOCKET) - conn.timeout = timeout - conn.connect() - conn.request(method, path, body=body, headers=headers or {}) - resp = conn.getresponse() - data = resp.read() - conn.close() - return resp.status, data - -def _tcp_request(method, url, body=None, headers=None, timeout=300): - req = request.Request(url, data=body, headers=headers or {}, method=method) - with request.urlopen(req, timeout=timeout) as resp: - return resp.status, resp.read() - -def _probe_tool_call_socket(): - # One-shot probe: is the path a live AF_UNIX listener? A real - # connect() succeeds against the proxy; a regular file or stale - # node returns ENOTSOCK / ECONNREFUSED. Done at preamble import - # time, *before* user code runs, so a malicious user cannot spoof - # /tmp/tcs.sock to flip the gate (a previous os.path.exists check - # was user-spoofable in legacy/no-proxy mode). Each NsJail - # invocation gets a fresh /tmp tmpfs so the probe result cannot - # bleed across jobs. +import stat as _stat +import struct as _struct +import threading as _threading +import select as _select +import time as _time +from concurrent.futures import Future as _Future + +# These two anonymous pipe ends are granted only to an authorized invocation. +_PIPE_WRITE_FD = 3 +_PIPE_READ_FD = 4 +_PIPE_MAX_FRAME = 1024 * 1024 +_PIPE_OWNER_PID = os.getpid() +_PIPE_LOCK = _threading.Lock() +_PIPE_WRITE_LOCK = _threading.Lock() +_PIPE_PENDING = {} +_PIPE_SEQUENCE = 0 +_PIPE_READER = None +_PIPE_FAILURE = None + +# Spawned multiprocessing workers re-import this module without the granted +# descriptors. They may import tool definitions, but cannot use the capability. +if __name__ == "__mp_main__": + _PIPE_OWNER_PID = None +else: + for _fd in (_PIPE_WRITE_FD, _PIPE_READ_FD): + try: + if not _stat.S_ISFIFO(os.fstat(_fd).st_mode): + raise OSError("tool-call descriptor is not an anonymous pipe") + os.set_inheritable(_fd, False) + except OSError as _error: + raise RuntimeError("blocking tool calls require a pipe-enabled runner") from _error + os.set_blocking(_PIPE_WRITE_FD, False) + +def _pipe_fail(error): + global _PIPE_FAILURE + with _PIPE_LOCK: + _PIPE_FAILURE = error + pending = list(_PIPE_PENDING.values()) + _PIPE_PENDING.clear() + for future in pending: + if not future.done(): + future.set_exception(error) + +def _pipe_read_exact(size): + chunks = [] + remaining = size + while remaining: + chunk = os.read(_PIPE_READ_FD, remaining) + if not chunk: + raise EOFError("tool-call broker disconnected") + chunks.append(chunk) + remaining -= len(chunk) + return b"".join(chunks) + +def _pipe_read_responses(): try: - s = _socket.socket(_socket.AF_UNIX, _socket.SOCK_STREAM) - s.settimeout(0.5) - s.connect(_TOOL_CALL_SOCKET) - s.close() - return True - except OSError: - return False - -_USE_TOOL_CALL_SOCKET = _probe_tool_call_socket() + while True: + size = _struct.unpack("!I", _pipe_read_exact(4))[0] + if not 0 < size <= _PIPE_MAX_FRAME: + raise ValueError("invalid tool-call response size") + response = json.loads(_pipe_read_exact(size).decode("utf-8")) + if not isinstance(response, dict) or not isinstance(response.get("status"), int) or not isinstance(response.get("body"), str): + raise ValueError("invalid tool-call response") + with _PIPE_LOCK: + future = _PIPE_PENDING.pop(response.get("id"), None) + # A late response after a caller timeout has no pending waiter. + if future is not None: + future.set_result((response["status"], response["body"].encode("utf-8"))) + except Exception as error: + _pipe_fail(error) def _do_request(method, path, body=None, headers=None, timeout=300): - # Cached at import time (see _probe_tool_call_socket). User code - # cannot influence this decision because it has not yet executed. - if _USE_TOOL_CALL_SOCKET: - return _unix_request(method, path, body, headers, timeout) - url = _CALLBACK_URL + path - return _tcp_request(method, url, body, headers, timeout) + global _PIPE_READER, _PIPE_SEQUENCE + if os.getpid() != _PIPE_OWNER_PID: + raise RuntimeError("tool calls must run in the invocation's primary process") + if method != "POST" or path != "/tool-call": + raise ValueError("unsupported tool-call route") + deadline = _time.monotonic() + timeout + future = _Future() + with _PIPE_LOCK: + if _PIPE_FAILURE is not None: + raise _PIPE_FAILURE + if len(_PIPE_PENDING) >= 16: + raise RuntimeError("too many concurrent tool calls") + if _PIPE_READER is None: + _PIPE_READER = _threading.Thread(target=_pipe_read_responses, daemon=True) + _PIPE_READER.start() + _PIPE_SEQUENCE += 1 + sequence = _PIPE_SEQUENCE + _PIPE_PENDING[sequence] = future + try: + payload = json.dumps({ + "id": sequence, + "headers": {key.lower(): value for key, value in (headers or {}).items()}, + "body": (body or b"").decode("utf-8") + }).encode("utf-8") + if not 0 < len(payload) <= _PIPE_MAX_FRAME: + raise ValueError("tool-call request too large") + framed = _struct.pack("!I", len(payload)) + payload + # Serialize writes, but allow concurrent upstream requests and responses. + if not _PIPE_WRITE_LOCK.acquire(timeout=max(0, deadline - _time.monotonic())): + raise TimeoutError("tool-call write timeout") + try: + remaining = memoryview(framed) + while remaining: + wait = deadline - _time.monotonic() + if wait <= 0 or not _select.select([], [_PIPE_WRITE_FD], [], wait)[1]: + raise TimeoutError("tool-call write timeout") + try: + written = os.write(_PIPE_WRITE_FD, remaining[:4096]) + except BlockingIOError: + continue + remaining = remaining[written:] + except Exception as error: + # A partial frame cannot be reused for the next request. + _pipe_fail(error) + os.close(_PIPE_WRITE_FD) + raise + finally: + _PIPE_WRITE_LOCK.release() + return future.result(timeout=max(0, deadline - _time.monotonic())) + finally: + with _PIPE_LOCK: + _PIPE_PENDING.pop(sequence, None) class ToolExecutionError(Exception): """Raised when a tool call fails""" @@ -500,7 +564,7 @@ async def _execute_tool_internal_async(tool_name: str, tool_input: Dict[str, Any * History is delivered as a file (not an env var) so very large histories * are not bounded by the Linux ARG_MAX / MAX_ARG_STRLEN ceiling. * - * No socket bind mount, no Tool Call Server callback, no long-poll. + * No pipe capability, no Tool Call Server callback, no long-poll. */ export function generateReplayPreamble(config: ReplayPreambleConfig): string { const { executionId, tools } = config; @@ -519,7 +583,7 @@ import os import asyncio from typing import Any, Dict, List, Optional, Union -_EXECUTION_ID = "${executionId}" +_EXECUTION_ID = ${JSON.stringify(executionId)} _PTC_SENTINEL_START = "${scopedStart}" _PTC_SENTINEL_END = "${scopedEnd}" _PTC_HISTORY_PATH = os.environ.get("PTC_HISTORY_PATH") or "${PTC_HISTORY_SANDBOX_PATH}" diff --git a/service/src/queue.ts b/service/src/queue.ts index bbd789ac..ad546190 100644 --- a/service/src/queue.ts +++ b/service/src/queue.ts @@ -20,8 +20,8 @@ import type { import logger from './logger'; import { redisKeepAliveOptions, redisReconnectDelay } from './redis-options'; import { bullmqQueueJobs, registerBullmqQueueMetricsCollector } from './metrics'; -import { bullmqPrefix } from './redis-connection'; import { JobCancellationRegistry } from './job-cancellation'; +import { bullmqPrefix } from './redis-connection'; const retryStrategy: CommonRedisOptions['retryStrategy'] = (times) => { logger.warn(`Retrying Redis connection attempt ${times}`); diff --git a/service/src/runtime-session/checkpoint.ts b/service/src/runtime-session/checkpoint.ts index 3562cd17..e49d9290 100644 --- a/service/src/runtime-session/checkpoint.ts +++ b/service/src/runtime-session/checkpoint.ts @@ -94,6 +94,7 @@ export async function pullCheckpoint( ? AbortSignal.any([args.signal, transferDeadlineSignal]) : transferDeadlineSignal; const response = await axios.get(`${args.endpointBase}/api/v2/session/checkpoint`, { + maxRedirects: 0, headers: { [token.headerName]: token.token, ...microvmPortHeaders(config.port), @@ -125,6 +126,7 @@ export async function pushRestore( ): Promise { const token = await args.mintToken(); await axios.post(`${args.endpointBase}/api/v2/session/restore`, fs.createReadStream(data.path), { + maxRedirects: 0, headers: { [token.headerName]: token.token, ...microvmPortHeaders(config.port), @@ -153,6 +155,7 @@ export async function probeInputs( `${args.endpointBase}/api/v2/session/inputs/probe`, { refs }, { + maxRedirects: 0, headers: { [token.headerName]: token.token, ...microvmPortHeaders(config.port), @@ -197,6 +200,7 @@ export async function pushInputs( * unlink the archive first and the stream then emits an unhandled ENOENT. */ const requestData = typeof data === 'function' ? data() : data; await axios.post(`${args.endpointBase}/api/v2/session/inputs`, requestData, { + maxRedirects: 0, headers: { [token.headerName]: token.token, ...microvmPortHeaders(config.port), diff --git a/service/src/runtime-session/files.ts b/service/src/runtime-session/files.ts index f17998dd..dd53508d 100644 --- a/service/src/runtime-session/files.ts +++ b/service/src/runtime-session/files.ts @@ -202,6 +202,7 @@ async function fetchFileObjectToPath( const url = `${baseUrl}/sessions/${encodeURIComponent(ref.storage_session_id)}/objects/${encodeURIComponent(ref.id)}`; try { const response = await axios.get(url, { + maxRedirects: 0, headers: internalServiceHeaders(), responseType: 'stream', timeout: opts.timeoutMs, diff --git a/service/src/sandbox-backend/http.test.ts b/service/src/sandbox-backend/http.test.ts index d3370646..403772be 100644 --- a/service/src/sandbox-backend/http.test.ts +++ b/service/src/sandbox-backend/http.test.ts @@ -14,7 +14,7 @@ type CapturedRequest = { let server: ReturnType; let captured: CapturedRequest[] = []; -let nextResponse: { status: number; body: unknown; delayMs?: number } = { status: 200, body: {} }; +let nextResponse: { status: number; body: unknown; delayMs?: number; location?: string } = { status: 200, body: {} }; const savedEndpoint = env.SANDBOX_ENDPOINT; @@ -33,7 +33,7 @@ beforeAll(() => { } return new Response(JSON.stringify(nextResponse.body), { status: nextResponse.status, - headers: { 'Content-Type': 'application/json' }, + headers: { 'Content-Type': 'application/json', ...(nextResponse.location ? { Location: nextResponse.location } : {}) }, }); }, }); @@ -104,6 +104,26 @@ describe('HttpSandboxBackend', () => { expect(result).toEqual(responseBody); }); + test('rejects redirects without sending the signed execution to another endpoint', async () => { + let redirected = 0; + const target = Bun.serve({ + port: 0, + fetch() { + redirected++; + return Response.json({ session_id: 'unexpected' }); + }, + }); + try { + for (const status of [302, 307, 308]) { + nextResponse = { status, body: {}, location: `http://127.0.0.1:${target.port}/stolen` }; + await expect(new HttpSandboxBackend().execute(request(), context())).rejects.toThrow(); + } + expect(redirected).toBe(0); + } finally { + target.stop(true); + } + }); + test('does not mutate the signed request body', async () => { const req = request(); const before = JSON.stringify(req.body); diff --git a/service/src/sandbox-backend/http.ts b/service/src/sandbox-backend/http.ts index ce5f5c0a..1ebdfcbe 100644 --- a/service/src/sandbox-backend/http.ts +++ b/service/src/sandbox-backend/http.ts @@ -20,6 +20,7 @@ export class HttpSandboxBackend implements SandboxBackend { `${env.SANDBOX_ENDPOINT}/${Jobs.execute}`, req.body, { + maxRedirects: 0, headers: injectTraceHeaders(req.headers), signal: ctx.signal, } diff --git a/service/src/sandbox-backend/lambda-microvm.ts b/service/src/sandbox-backend/lambda-microvm.ts index 00e73db2..6e59d773 100644 --- a/service/src/sandbox-backend/lambda-microvm.ts +++ b/service/src/sandbox-backend/lambda-microvm.ts @@ -1449,6 +1449,7 @@ export class LambdaMicrovmSandboxBackend implements SandboxBackend { `${base}/api/v2/${Jobs.execute}`, req.body, { + maxRedirects: 0, headers: { ...injectTraceHeaders(req.headers), [token.headerName]: token.token, @@ -1656,6 +1657,7 @@ export class LambdaMicrovmSandboxBackend implements SandboxBackend { private async assertHealthy(base: string, token: string, ctx: SandboxExecuteContext): Promise { try { const response = await axios.get(`${base}/api/v2/health`, { + maxRedirects: 0, headers: { 'X-aws-proxy-auth': token, ...microvmPortHeaders(this.config.port) }, timeout: this.config.healthTimeoutMs, signal: ctx.signal, diff --git a/service/src/service/programmatic-router.ts b/service/src/service/programmatic-router.ts index a646b9ad..86b1cdbb 100644 --- a/service/src/service/programmatic-router.ts +++ b/service/src/service/programmatic-router.ts @@ -92,7 +92,11 @@ import { CODEAPI_BRIDGE_WORKSPACE_HEADER, resolveBridgeWorkerSelection, } from '../bridge/selection'; -import { isValidBridgeWorkerId, BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES } from '../../../packages/code/src/protocol'; +import { + isValidBridgeWorkerId, + isValidLinkedWorktreeName, + BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_INPUT_FILES, +} from '../../../packages/code/src/protocol'; import logger from '../logger'; import { type ExecutionState, @@ -315,7 +319,7 @@ function waitForExecutionState( () => axios.get( `${env.TOOL_CALL_SERVER_URL}/sessions/${id}/pending`, - { headers: internalServiceHeaders() }, + { maxRedirects: 0, headers: internalServiceHeaders() }, ), 'Get pending tool calls', ); @@ -505,6 +509,7 @@ async function handleReplayInitial( bridgeWorkerId?: string; workspaceId?: string; workspaceInstanceId?: string; + workspaceWorktree?: string; }, cancellation: ReplayRequestCancellation, ): Promise { @@ -514,6 +519,7 @@ async function handleReplayInitial( bridgeWorkerId, workspaceId, workspaceInstanceId, + workspaceWorktree, } = params; const { code, tools, user_id, files } = req.body as t.ProgrammaticRequestBody; @@ -671,6 +677,7 @@ async function handleReplayInitial( bridgeWorkerId, workspaceId, workspaceInstanceId, + workspaceWorktree, executionProfile: env.EXECUTION_PROFILE, executionProfileSource: env.EXECUTION_PROFILE_SOURCE, sandboxBackend: resolveReplayStateSandboxBackend({ @@ -1291,6 +1298,7 @@ router.post( let bridgeWorkerId: string | undefined; let workspaceId: string | undefined; let workspaceInstanceId: string | undefined; + let workspaceWorktree: string | undefined; if (continuation_token == null || continuation_token === '') { try { const bridgeSelection = resolveBridgeWorkerSelection({ @@ -1341,6 +1349,15 @@ router.post( principalId: principal.userId, }); } + const requestedWorktree = rawBody.workspace_worktree; + if (requestedWorktree !== undefined) { + if (workspaceId == null || !isValidLinkedWorktreeName(requestedWorktree)) { + return res.status(400).json({ + error: 'Invalid code workspace worktree', + }); + } + workspaceWorktree = requestedWorktree; + } } catch (error) { if (error instanceof BridgeWorkerSelectionError) { return res @@ -1458,6 +1475,7 @@ router.post( bridgeWorkerId, workspaceId, workspaceInstanceId, + workspaceWorktree, }, cancellation); } if (workspaceId != null) { @@ -1580,7 +1598,7 @@ async function handleBlocking( error_message: r.error_message, })), }, - { headers: internalServiceHeaders() }, + { maxRedirects: 0, headers: internalServiceHeaders() }, ), 'Submit tool results', ); @@ -1765,7 +1783,7 @@ async function handleBlocking( timeout, tools, }, - { headers: internalServiceHeaders() }, + { maxRedirects: 0, headers: internalServiceHeaders() }, ), 'Create Tool Call Server session', ); diff --git a/service/src/service/programmatic-state.ts b/service/src/service/programmatic-state.ts index 38c72486..056fcbb5 100644 --- a/service/src/service/programmatic-state.ts +++ b/service/src/service/programmatic-state.ts @@ -40,6 +40,7 @@ export interface BuildReplayExecutionStateParams { bridgeWorkerId?: string; workspaceId?: string; workspaceInstanceId?: string; + workspaceWorktree?: string; sandboxBackend?: SandboxBackendName; executionProfile: ExecutionProfile; executionProfileSource: ExecutionProfileSource; @@ -70,6 +71,7 @@ export function buildReplayExecutionState( bridgeWorkerId: params.bridgeWorkerId, workspaceId: params.workspaceId, workspaceInstanceId: params.workspaceInstanceId, + workspaceWorktree: params.workspaceWorktree, sandboxBackend: params.sandboxBackend, executionProfile: params.executionProfile, executionProfileSource: params.executionProfileSource, @@ -86,12 +88,21 @@ export function buildReplayExecutionState( }; } -/** Bind the authenticated conversation checkout to every replay iteration. */ +/** Bind the authenticated conversation checkout and linked-worktree lane to every replay iteration. */ export function bindReplayWorkspaceInstance( payload: t.PayloadBody, - state: Pick, + state: Pick, ): t.PayloadBody { - return state.workspaceInstanceId == null - ? payload - : { ...payload, workspace_instance_id: state.workspaceInstanceId }; + if (state.workspaceInstanceId == null && state.workspaceWorktree == null) { + return payload; + } + return { + ...payload, + ...(state.workspaceInstanceId == null + ? {} + : { workspace_instance_id: state.workspaceInstanceId }), + ...(state.workspaceWorktree == null + ? {} + : { workspace_worktree: state.workspaceWorktree }), + }; } diff --git a/service/src/service/replay-state.ts b/service/src/service/replay-state.ts index 70aafea9..52eb902f 100644 --- a/service/src/service/replay-state.ts +++ b/service/src/service/replay-state.ts @@ -121,6 +121,8 @@ export interface ExecutionState { workspaceId?: string; /** Selected conversation checkout retained across every replay iteration. */ workspaceInstanceId?: string; + /** Selected linked-worktree lane retained across every replay iteration. */ + workspaceWorktree?: string; /** Original queue/backend target retained across replay continuations. */ sandboxBackend?: SandboxBackendName; /** Original producer profile retained so continuations use the same queue. */ @@ -802,6 +804,7 @@ export async function cleanupStaleExecutions(): Promise { if (state.mode === 'blocking') { await axios.delete(`${env.TOOL_CALL_SERVER_URL}/sessions/${state.execution_id}`, { + maxRedirects: 0, headers: internalServiceHeaders(), }) .catch(() => {}); @@ -837,6 +840,7 @@ export async function cleanupExecution(execution_id: string, mode: 'blocking' | if (mode === 'blocking') { ops.push( axios.delete(`${env.TOOL_CALL_SERVER_URL}/sessions/${execution_id}`, { + maxRedirects: 0, headers: internalServiceHeaders(), }).catch(() => {}), ); diff --git a/service/src/service/router.ts b/service/src/service/router.ts index 43d0ff4b..a977182e 100644 --- a/service/src/service/router.ts +++ b/service/src/service/router.ts @@ -71,6 +71,7 @@ async function putFileToFileServer( ): Promise { const response = await fetch(url, { method: 'PUT', + redirect: 'error', headers, body: Readable.toWeb(file) as unknown as ReadableStream, signal, @@ -416,6 +417,7 @@ router.get('/download/:session_id/:fileId', downloadLimiter, sessionAuth, async const response = await axios({ method: 'get', url: `${env.FILE_SERVER_URL}/sessions/${session_id}/objects/${fileId}`, + maxRedirects: 0, headers: internalServiceHeaders(), responseType: 'stream' }); @@ -909,6 +911,7 @@ router.get('/files/:session_id', fetchLimiter, sessionAuth, async (req: t.Authen try { const response = await axios.get(`${env.FILE_SERVER_URL}/sessions/${session_id}/objects`, { params: { detail }, + maxRedirects: 0, headers: internalServiceHeaders({ 'Accept': 'application/json' }) }); @@ -941,7 +944,7 @@ router.get('/sessions/:session_id/objects/:fileId', fetchLimiter, sessionAuth, a try { const response = await axios.get( `${env.FILE_SERVER_URL}/sessions/${session_id}/objects/${fileId}/metadata`, - { headers: internalServiceHeaders({ Accept: 'application/json' }) }, + { maxRedirects: 0, headers: internalServiceHeaders({ Accept: 'application/json' }) }, ); return res.status(200).json(response.data); @@ -972,7 +975,7 @@ const deleteSessionObject = async (req: t.AuthenticatedRequest, res: Response) = try { const response = await axios.delete( `${env.FILE_SERVER_URL}/sessions/${session_id}/objects/${fileId}`, - { headers: internalServiceHeaders() } + { maxRedirects: 0, headers: internalServiceHeaders() } ); await connection.del(`upload:${req.sessionKey}${session_id}${fileId}`); diff --git a/service/src/types/service.ts b/service/src/types/service.ts index 91c3ed89..b0375b38 100644 --- a/service/src/types/service.ts +++ b/service/src/types/service.ts @@ -206,6 +206,8 @@ export interface PayloadBody { version: string; /** Opaque conversation checkout selected and authenticated by the API. */ workspace_instance_id?: string; + /** Linked worktree lane at `.worktrees/` beneath the selected checkout. */ + workspace_worktree?: string; /** Stable identity shared by all replay iterations of one execution. */ execution_id?: string; replay_tool_count?: number; @@ -397,6 +399,8 @@ export interface ProgrammaticRequestBody { lang?: 'python' | 'bash'; /** Opaque conversation checkout binding for a selected native workspace. */ workspace_instance_id?: string; + /** Linked worktree lane at `.worktrees/` beneath the selected checkout. */ + workspace_worktree?: string; } export interface ProgrammaticToolCall { diff --git a/service/src/workers.ts b/service/src/workers.ts index b0f0dc4d..227978b8 100644 --- a/service/src/workers.ts +++ b/service/src/workers.ts @@ -50,11 +50,11 @@ import { validateQueuedExecutionProfile, validateQueuedSandboxBackend, } from './execution-profile'; -import { bullmqPrefix } from './redis-connection'; import { BRIDGE_WORKSPACE_PROGRAMMATIC_MAX_FILE_BYTES, programmaticTransferReserveMs, } from '../../packages/code/src/protocol'; +import { bullmqPrefix } from './redis-connection'; const { INSTANCE_ID } = env; const WORKER_ID = `${INSTANCE_ID}-${process.pid}`; diff --git a/service/src/workspace-tools/coordination.ts b/service/src/workspace-tools/coordination.ts new file mode 100644 index 00000000..0bcd5e91 --- /dev/null +++ b/service/src/workspace-tools/coordination.ts @@ -0,0 +1,21 @@ +import { createHash } from 'crypto'; + +export interface WorkspaceRequestCoordinationPolicy { + bridgeEnabled: boolean; + backend: string; + executionProfile: string; + authMode: string; + configuredWorkerId: string; + dynamicWorkers: boolean; + maxWorkspaceLeaseSlots: number; + maxCommandTimeoutMs: number; +} + +/** Only replicas with the same bridge policy may reconcile accepted work. */ +export function workspaceRequestCoordinationScope(policy: WorkspaceRequestCoordinationPolicy): string | undefined { + if (!policy.bridgeEnabled) return undefined; + return createHash('sha256').update(JSON.stringify([ + policy.backend, policy.executionProfile, policy.authMode, policy.configuredWorkerId, + policy.dynamicWorkers, policy.maxWorkspaceLeaseSlots, policy.maxCommandTimeoutMs, + ])).digest('hex'); +} diff --git a/service/src/workspace-tools/index.ts b/service/src/workspace-tools/index.ts index 6f105f16..3c7caee2 100644 --- a/service/src/workspace-tools/index.ts +++ b/service/src/workspace-tools/index.ts @@ -4,12 +4,59 @@ import { bridgeStore } from '../bridge'; import { env } from '../config'; import { executionLimiter } from '../middleware/limits'; import { createWorkspaceToolsRouter } from './router'; +import { RedisWorkspaceRequests } from './requests'; +import { connection } from '../queue'; +import logger from '../logger'; +import { checkServiceStartUp, checkServiceShutDown } from '../lifecycle'; +import { withSpan } from '../telemetry'; +import { isBridgeEnabled } from '../bridge/enabled'; +import { workspaceRequestCoordinationScope } from './coordination'; const router = Router(); -router.use('/workspace-tools/execute', executionLimiter); +const coordinationScope = workspaceRequestCoordinationScope({ + bridgeEnabled: isBridgeEnabled(), backend: env.SANDBOX_BACKEND, + executionProfile: env.EXECUTION_PROFILE, authMode: env.BRIDGE_AUTH_MODE, + configuredWorkerId: env.BRIDGE_WORKER_ID, dynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, + maxWorkspaceLeaseSlots: env.BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, maxCommandTimeoutMs: env.JOB_TIMEOUT, +}); +const requests = coordinationScope === undefined ? undefined : new RedisWorkspaceRequests(connection, bridgeStore, record => { + const attributes = { + 'codeapi.request.id': record.id, + 'codeapi.admission.state': record.state, + 'codeapi.worker.id': record.workerId, + 'codeapi.workspace.id': record.request.workspaceId, + 'codeapi.workspace.instance_id': record.request.workspaceInstanceId ?? '', + 'codeapi.workspace.worktree': record.request.worktree ?? '', + 'codeapi.admission.wait_ms': Math.max(0, (record.admittedAtMs ?? Math.min(record.finishedAtMs ?? Date.now(), record.queueDeadlineAtMs)) - record.createdAtMs), + 'codeapi.admission.error_code': record.error?.code ?? '', + }; + logger.info('Durable workspace admission transition', attributes); + void withSpan('codeapi.workspace.admission.transition', attributes, () => undefined).catch(error => { + logger.warn('Admission transition telemetry failed', { error }); + }); +}, coordinationScope); +if (requests != null) { + const coordinator = requests; + let reconciling = false; + let unavailable = false; + const reconciliation = setInterval(() => { + if (reconciling || checkServiceStartUp() || checkServiceShutDown()) return; + reconciling = true; + void coordinator.reconcile().then(() => { unavailable = false; }).catch(error => { + if (!unavailable) logger.warn('Durable workspace reconciliation unavailable', { error }); + unavailable = true; + }).finally(() => { reconciling = false; }); + }, 250); + reconciliation.unref(); +} +router.use(['/workspace-tools/execute', '/workspace-tools/requests'], (req, res, next) => { + if (req.method === 'POST') executionLimiter(req, res, next); + else next(); +}); router.use( createWorkspaceToolsRouter({ store: bridgeStore, + requests, backend: env.SANDBOX_BACKEND, configuredWorkerId: env.BRIDGE_WORKER_ID, dynamicWorkers: env.BRIDGE_DYNAMIC_WORKERS, diff --git a/service/src/workspace-tools/outcome.ts b/service/src/workspace-tools/outcome.ts index d965d515..e8dc100f 100644 --- a/service/src/workspace-tools/outcome.ts +++ b/service/src/workspace-tools/outcome.ts @@ -25,7 +25,7 @@ const earlyErrorCodes: Record = { 500: 'INTERNAL_ERROR', }; -export function getWorkspaceToolOutcome(res: Response): WorkspaceToolOutcome { +export function getWorkspaceToolOutcome(res: Response, route = '/workspace-tools/execute'): WorkspaceToolOutcome { const existing = outcomes.get(res); if (existing != null) return existing; const startedAt = performance.now(); @@ -39,7 +39,7 @@ export function getWorkspaceToolOutcome(res: Response): WorkspaceToolOutcome { outcomes.delete(res); const finished = res.writableFinished; logger.log(finished && res.statusCode < 400 ? 'info' : 'warn', 'Workspace tool request completed', { - route: '/workspace-tools/execute', + route, operation: outcome.operation, workerId: outcome.workerId, status: finished ? res.statusCode : undefined, diff --git a/service/src/workspace-tools/requests-router.test.ts b/service/src/workspace-tools/requests-router.test.ts new file mode 100644 index 00000000..856a7a38 --- /dev/null +++ b/service/src/workspace-tools/requests-router.test.ts @@ -0,0 +1,96 @@ +import { createServer } from 'node:http'; +import type { Server } from 'node:http'; +import { afterEach, expect, test } from 'bun:test'; +import express, { json } from 'express'; +import RedisMock from 'ioredis-mock'; +import type Redis from 'ioredis'; +import { applyPrincipal } from '../auth/principal'; +import { RedisBridgeStore } from '../bridge/store'; +import { createWorkspaceToolsRouter } from './router'; +import { RedisWorkspaceRequests } from './requests'; + +let server: Server | undefined; +afterEach(() => { server?.close(); server = undefined; }); +const body = { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }; +const requestId = 'http-request-000001'; + +async function setup(durable = true, rejectSynchronous = false): Promise { + const redis = new RedisMock() as unknown as Redis; + const bridge = new RedisBridgeStore(redis); + await bridge.register({ protocolVersion: 1, workerId: 'worker', incarnationId: 'incarnation-00000001', capabilities: { + sandboxProfile: 'native-srt', statefulWorkspace: false, runtimes: [], + workspaceTools: { protocolVersion: 1, operations: ['read_file'], workspaces: [{ id: 'primary' }] }, + } }); + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + if (req.header('X-Test-User') !== 'unauthenticated') applyPrincipal(req, { + tenantId: 'tenant', userId: req.header('X-Test-User') ?? 'user', principalSource: 'local', codeWorkerId: 'worker', + }); + next(); + }); + app.use(createWorkspaceToolsRouter({ + store: rejectSynchronous ? { async dispatchWorkspaceTool(): Promise { throw new Error('Durable URL dispatched synchronously'); } } : bridge, requests: durable ? new RedisWorkspaceRequests(redis, bridge) : undefined, + backend: 'remote-bridge', configuredWorkerId: 'worker', dynamicWorkers: false, + })); + server = createServer(app); + await new Promise(resolve => server!.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Missing listener'); + return `http://127.0.0.1:${address.port}`; +} + +function post(url: string, id = requestId, value = body): Promise { + return fetch(`${url}/workspace-tools/requests`, { method: 'POST', headers: { + 'Content-Type': 'application/json', 'X-LibreChat-Workspace-Request-Id': id, + }, body: JSON.stringify(value) }); +} + +test.each([false, true])('capability discovery explicitly distinguishes durable support (%s)', async durable => { + const url = await setup(durable); + expect(await (await fetch(`${url}/workspace-tools/capabilities`)).json()).toEqual({ durableWorkspaceRequests: durable ? 1 : 0 }); +}); + +test('submission, duplicate lookup and cancellation use the same principal-scoped handle without exposing leases', async () => { + const url = await setup(); + const accepted = await post(url); + expect(accepted.status).toBe(202); + expect(await accepted.json()).toMatchObject({ requestId, state: 'queued', queuePosition: 1 }); + const duplicate = await post(url); + expect(duplicate.status).toBe(202); + const status = await (await fetch(`${url}/workspace-tools/requests/${requestId}`)).json(); + expect(status).toMatchObject({ requestId, state: 'queued' }); + expect(status).not.toHaveProperty('assignment'); + expect(status).not.toHaveProperty('leaseToken'); + expect((await fetch(`${url}/workspace-tools/requests/${requestId}`, { headers: { 'X-Test-User': 'another' } })).status).toBe(404); + const cancelled = await fetch(`${url}/workspace-tools/requests/${requestId}`, { method: 'DELETE' }); + expect(await cancelled.json()).toMatchObject({ requestId, cancelRequested: true }); +}); + +test('missing IDs and conflicting requests fail without dispatching', async () => { + const url = await setup(); + expect((await post(url, '')).status).toBe(400); + expect((await post(url)).status).toBe(202); + const conflict = await post(url, requestId, { ...body, path: 'different' }); + expect(conflict.status).toBe(409); + expect(await conflict.json()).toMatchObject({ code: 'REQUEST_CONFLICT' }); +}); + +test('unauthenticated request status and capability discovery are rejected', async () => { + const url = await setup(); + for (const path of ['capabilities', `requests/${requestId}`]) { + expect((await fetch(`${url}/workspace-tools/${path}`, { headers: { 'X-Test-User': 'unauthenticated' } })).status).toBe(401); + } +}); + +test.each(['/workspace-tools/requests/', '/workspace-tools/REQUESTS'])('durable route aliases remain idempotent (%s)', async path => { + const url = await setup(true, true); + for (let attempt = 0; attempt < 2; attempt++) { + const response = await fetch(`${url}${path}`, { method: 'POST', headers: { + 'Content-Type': 'application/json', 'X-LibreChat-Workspace-Request-Id': requestId, + }, body: JSON.stringify(body) }); + expect(response.status).toBe(202); + expect(await response.json()).toMatchObject({ requestId, state: 'queued', queuePosition: 1 }); + } + expect(await (await fetch(`${url}/workspace-tools/requests/${requestId}`)).json()).toMatchObject({ state: 'queued' }); +}); diff --git a/service/src/workspace-tools/requests.test.ts b/service/src/workspace-tools/requests.test.ts new file mode 100644 index 00000000..a0f7d99f --- /dev/null +++ b/service/src/workspace-tools/requests.test.ts @@ -0,0 +1,671 @@ +import { afterEach, expect, spyOn, test } from 'bun:test'; +import { createHash } from 'node:crypto'; +import RedisMock from 'ioredis-mock'; +import Redis from 'ioredis'; +import { RedisBridgeStore } from '../bridge/store'; +import { BridgeAdmissionQueue } from '../bridge/admission'; +import type { CodeBridgeAssignment } from '../bridge/store'; +import type { WorkspaceToolRequest } from '../../../packages/code/src/protocol'; +import { RedisWorkspaceRequests } from './requests'; +import { workspaceRequestCoordinationScope } from './coordination'; +import type { WorkspaceRequestCoordinationPolicy } from './coordination'; + +const testRedisUrl = process.env.BRIDGE_TEST_REDIS_URL; +const redis = testRedisUrl !== undefined && testRedisUrl.length > 0 + ? new Redis(testRedisUrl) + : new RedisMock() as unknown as Redis; +const bridge = new RedisBridgeStore(redis, 600, 1000, 2); +const owner = { tenantId: 'tenant', userId: 'user' }; +const workerId = 'durable-worker'; +const incarnationId = 'incarnation-00000001'; +const requestId = 'request-00000000001'; +const activeKey = 'codeapi:workspace-requests:v1:active'; +const queueKey = `codeapi:bridge:v1:worker:${workerId}:admission`; + +// Only a disposable test Redis may be supplied. +afterEach(async () => { await redis.flushall(); }); + +async function setup(slots = 1): Promise { + const generation = await bridge.register({ protocolVersion: 1, workerId, incarnationId, + capabilities: { + sandboxProfile: 'native-srt', statefulWorkspace: false, runtimes: ['bash'], + ...(slots > 1 ? { workspaceLeaseSlots: slots, requiresReadyConfirmation: true } : {}), + workspaceTools: { protocolVersion: 1, operations: ['read_file', 'execute_command', 'list_files', 'search_text', 'preview_edit'], + workspaces: [{ id: 'primary' }, { id: 'independent' }] }, + }, + }); + if (slots > 1) await bridge.confirmReady(workerId, incarnationId, generation); + return new RedisWorkspaceRequests(redis, bridge); +} + +function submit(requests: RedisWorkspaceRequests, id = requestId, workspaceId = 'primary'): ReturnType { + return requests.submit({ owner, requestId: id, workerId, requireTenantBinding: false, + request: { protocolVersion: 1, operation: 'read_file', workspaceId, path: 'README.md' }, + queueWaitMs: 300_000, executionTimeoutMs: 30_000, + }); +} + +async function tick(requests: RedisWorkspaceRequests): Promise { + const keys = await redis.zrange(activeKey, 0, -1); + for (const key of keys) await redis.zadd(activeKey, 0, key); + await requests.reconcile(); +} + +async function settle(assignment: CodeBridgeAssignment, fulfilled = true): Promise { + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, + assignment.generation, assignment.leaseToken); + await bridge.settle(workerId, assignment.assignmentId, { + protocolVersion: 1, incarnationId, generation: assignment.generation, + leaseToken: assignment.leaseToken, + ...(fulfilled ? { status: 'fulfilled' as const, result: { protocolVersion: 1, operation: 'read_file', workspaceId: (assignment.request as { workspaceId: string }).workspaceId, path: 'README.md', content: 'hello', startLine: 1, endLine: 1, truncated: false } } + : { status: 'rejected' as const, error: 'stopped' }), + }); +} + +async function mutateRecord(update: (record: Record) => void): Promise { + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const record = JSON.parse((await redis.hget(key, 'record'))!); + update(record); + await redis.hset(key, 'record', JSON.stringify(record)); +} + +test('submission is idempotent across replicas, scoped by principal, and conflicts do not move FIFO position', async () => { + const requests = await setup(); + await submit(requests); + await submit(requests, 'request-00000000002'); + const replica = new RedisWorkspaceRequests(redis, bridge); + expect((await submit(replica)).queuePosition).toBe(1); + expect(await redis.zcard(queueKey)).toBe(2); + await expect(submit(replica, requestId, 'independent')).rejects.toThrow('different work'); + expect(await replica.get({ ...owner, userId: 'someone-else' }, requestId)).toBeUndefined(); + expect(await replica.cancel({ ...owner, tenantId: 'someone-else' }, requestId)).toBeUndefined(); +}); + +test('restart before dispatch preserves the queue and starts a full execution budget after sixty seconds waiting', async () => { + const requests = await setup(); + await submit(requests); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + await mutateRecord(record => { + record.createdAtMs = Number(record.createdAtMs) - 60_000; + record.queueDeadlineAtMs = Number(record.queueDeadlineAtMs) - 60_000; + }); + const record = JSON.parse((await redis.hget(key, 'record'))!); + await redis.hset(key, 'queueDeadlineAtMs', record.queueDeadlineAtMs); + const restarted = new RedisWorkspaceRequests(redis, new RedisBridgeStore(redis)); + await tick(restarted); + const assignment = await bridge.lease(workerId, incarnationId, 0); + expect(assignment).toBeDefined(); + expect(assignment!.remainingMs).toBeGreaterThan(29_000); + // Mock Redis TIME has subsecond clock skew. Execution still receives its full budget. + expect((await restarted.get(owner, requestId))?.queueWaitMs).toBeGreaterThan(59_000); + await settle(assignment!); + await tick(restarted); + expect(await restarted.get(owner, requestId)).toMatchObject({ state: 'completed', result: { content: 'hello' } }); +}); + +test.each([false, true])('restart around admission acknowledgement never enqueues another assignment (ack=%s)', async acknowledged => { + const requests = await setup(); + await submit(requests); + await tick(requests); + const assignment = await bridge.lease(workerId, incarnationId, 0); + if (acknowledged) await bridge.acknowledgeLease(workerId, incarnationId, assignment!.assignmentId, + assignment!.generation, assignment!.leaseToken); + const restarted = new RedisWorkspaceRequests(redis, new RedisBridgeStore(redis)); + await tick(restarted); + await submit(restarted); + expect((await bridge.lease(workerId, incarnationId, 0))?.assignmentId).toBe(assignment!.assignmentId); + expect(await redis.llen(`codeapi:bridge:v1:worker:${workerId}:incarnation:${incarnationId}:assignments`)).toBe(0); + await settle(assignment!); + await tick(restarted); + expect((await restarted.get(owner, requestId))?.state).toBe('completed'); +}); + +test('a lost enqueue acknowledgement is reconciled rather than replayed', async () => { + const requests = await setup(); + await submit(requests); + const original = redis.eval.bind(redis); + const failure = spyOn(redis, 'eval').mockImplementation((async (...args: Parameters) => { + const result = await original(...args); + if (String(args[0]).includes('\'state\', \'admitted\', \'assignment\'')) throw new Error('lost acknowledgement'); + return result; + }) as Redis['eval']); + try { await expect(tick(requests)).rejects.toThrow('lost acknowledgement'); } finally { failure.mockRestore(); } + const assignment = await bridge.lease(workerId, incarnationId, 0); + expect(assignment).toBeDefined(); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect((await requests.get(owner, requestId))?.state).toBe('admitted'); + await settle(assignment!); + await tick(requests); + expect((await requests.get(owner, requestId))?.state).toBe('completed'); +}); + +test('cancel before admission is durable and does not dispatch', async () => { + const requests = await setup(); + await submit(requests); + await requests.cancel(owner, requestId); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect((await requests.get(owner, requestId))?.state).toBe('cancelled'); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeUndefined(); + expect(await redis.zcard(queueKey)).toBe(0); +}); + +test('cancel racing a persisted fulfillment returns the winning result without consuming it', async () => { + const requests = await setup(); + await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await settle(assignment); + await requests.cancel(owner, requestId); + await tick(new RedisWorkspaceRequests(redis, bridge)); + const statuses = await Promise.all([requests.get(owner, requestId), requests.get(owner, requestId), requests.cancel(owner, requestId)]); + for (const status of statuses) expect(status).toMatchObject({ state: 'completed', result: { content: 'hello' } }); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeUndefined(); +}); + +test('restart after result persistence repeats only idempotent cleanup', async () => { + const requests = await setup(); + await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await settle(assignment); + const cleanup = spyOn(bridge, 'finishDurableWorkspaceTool').mockRejectedValue(new Error('process stopped')); + try { await expect(tick(requests)).rejects.toThrow('process stopped'); } finally { cleanup.mockRestore(); } + expect((await requests.get(owner, requestId))?.state).toBe('completed'); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect(await redis.zcard(activeKey)).toBe(0); + await submit(requests); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeUndefined(); +}); + +test('pending work rejects a replacement worker incarnation and never transfers to it', async () => { + const requests = await setup(); + await submit(requests); + await bridge.register({ protocolVersion: 1, workerId, incarnationId: 'replacement-00001', + capabilities: { sandboxProfile: 'native-srt', statefulWorkspace: false, runtimes: [], + workspaceTools: { protocolVersion: 1, operations: ['read_file'], workspaces: [{ id: 'primary' }] } }, + }); + await tick(requests); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'WORKER_FENCED' } }); +}); + +test('durable requests preserve negotiated independent-root concurrency', async () => { + const requests = await setup(2); + await submit(requests); await submit(requests, 'request-00000000002', 'independent'); + await tick(requests); await tick(requests); + const first = await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0); + const second = await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 1); + expect(first).toBeDefined(); expect(second).toBeDefined(); + await settle(first!); await settle(second!); await tick(requests); + expect((await requests.get(owner, requestId))?.state).toBe('completed'); + expect((await requests.get(owner, 'request-00000000002'))?.state).toBe('completed'); +}); + +test.each([1, 2])('restart after capacity reservation refreshes its own reservation without duplicating work (slots=%s)', async slots => { + const requests = await setup(slots); + await submit(requests); + const generation = spyOn(redis, 'incr').mockRejectedValue(new Error('process stopped before enqueue')); + try { await expect(tick(requests)).rejects.toThrow('process stopped before enqueue'); } finally { generation.mockRestore(); } + expect((await requests.get(owner, requestId))?.state).toBe('queued'); + await redis.pexpire(`codeapi:bridge:v1:worker:${workerId}:lock`, 5000); + await tick(new RedisWorkspaceRequests(redis, bridge)); + const assignment = await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0); + expect(assignment).toBeDefined(); + expect(await redis.pttl(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBeGreaterThan(30_000); +}); + +test.each([1, 2])('a stale coordinator cannot acquire capacity or dispatch (slots=%s)', async slots => { + const requests = await setup(slots); await submit(requests); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const record = JSON.parse((await redis.hget(key, 'record'))!); + await redis.set(`${key}:claim`, 'new-owner', 'PX', 10000); + await bridge.advanceDurableWorkspaceTool(record, { key, claimKey: `${key}:claim`, token: 'old-owner' }); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeUndefined(); + expect((await requests.get(owner, requestId))?.state).toBe('queued'); + expect(await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBeNull(); + expect(await redis.hlen(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`)).toBe(0); +}); + +test('a queued deadline expires as definitely unstarted work', async () => { + const requests = await setup(); await submit(requests); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + await mutateRecord(record => { record.queueDeadlineAtMs = Date.now() - 1; }); + await redis.hset(key, 'queueDeadlineAtMs', Date.now() - 1); + await tick(requests); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'WORKSPACE_QUEUE_TIMEOUT' } }); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeUndefined(); +}); + +test('an expired acknowledged command is unknown, remains fenced, and is not re-enqueued on restart', async () => { + const requests = await setup(); await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, assignment.generation, assignment.leaseToken); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const stored = JSON.parse((await redis.hget(key, 'assignment'))!); + stored.expiresAt = new Date(Date.now() - 1).toISOString(); + await redis.hset(key, 'assignment', JSON.stringify(stored)); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'ASSIGNMENT_EXPIRED' } }); + await submit(requests); + expect(await redis.llen(`codeapi:bridge:v1:worker:${workerId}:incarnation:${incarnationId}:assignments`)).toBe(0); + expect(await redis.get(`codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update('native-workspace:primary').digest('hex')}:quarantined`)).toBe(assignment.assignmentId); +}); + +test('cancel during command execution waits for clean rejection and does not replay', async () => { + const requests = await setup(); + await requests.submit({ owner, requestId, workerId, requireTenantBinding: false, + request: { protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'sleep 30' }, + queueWaitMs: 300_000, executionTimeoutMs: 35_000, + }); + await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, assignment.generation, assignment.leaseToken); + await requests.cancel(owner, requestId); + const cancelling = tick(requests); + const deadline = Date.now() + 1000; + while (!await bridge.cancelled(workerId, incarnationId, assignment.assignmentId)) { + if (Date.now() >= deadline) throw new Error('Worker never received cancellation'); + await new Promise(resolve => setTimeout(resolve, 5)); + } + await settle(assignment, false); + await cancelling; + expect((await requests.get(owner, requestId))?.state).toBe('cancelled'); +}); + +test('restart between workspace commit and cleanup preserves the result and releases the same assignment', async () => { + const requests = await setup(); await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; await settle(assignment); + const original = redis.eval.bind(redis); + let failed = false; + const failure = spyOn(redis, 'eval').mockImplementation((async (...args: Parameters) => { + if (!failed && String(args[0]).includes('local queued = redis.call(\'LREM\'')) { + failed = true; throw new Error('process stopped after commit'); + } + return original(...args); + }) as Redis['eval']); + try { await expect(tick(requests)).rejects.toThrow('process stopped after commit'); } finally { failure.mockRestore(); } + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect((await requests.get(owner, requestId))?.state).toBe('completed'); + expect(await redis.zcard(activeKey)).toBe(0); +}); + +test('simultaneous submissions from replicas keep one request identity and one FIFO entry', async () => { + const requests = await setup(); + const replica = new RedisWorkspaceRequests(redis, bridge); + const statuses = await Promise.all([submit(requests), submit(replica)]); + for (const status of statuses) expect(status).toMatchObject({ requestId, state: 'queued', queuePosition: 1 }); + expect(await redis.zcard(queueKey)).toBe(1); + expect(await redis.zcard(activeKey)).toBe(1); + await tick(requests); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeDefined(); +}); + +test.each([false, true])('temporary registration loss preserves FIFO until the original worker reconnects (replacement=%s)', async replacement => { + const requests = await setup(); + await submit(requests); await submit(requests, 'request-00000000002'); + // Registration/readiness TTL expiry, without losing accepted work. + const worker = `codeapi:bridge:v1:worker:${workerId}`; + await redis.del(worker, `${worker}:incarnation`, `${worker}:ready`); + const restarted = new RedisWorkspaceRequests(redis, bridge); + await tick(restarted); + expect(await restarted.get(owner, requestId)).toMatchObject({ state: 'queued', queuePosition: 1 }); + expect(await redis.zcard(activeKey)).toBe(2); + if (replacement) { + await bridge.register({ protocolVersion: 1, workerId, incarnationId: 'replacement-00001', + capabilities: { sandboxProfile: 'native-srt', statefulWorkspace: false, runtimes: [], + workspaceTools: { protocolVersion: 1, operations: ['read_file'], workspaces: [{ id: 'primary' }] } }, + }); + await tick(restarted); + expect(await restarted.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'WORKER_FENCED' } }); + expect(await bridge.lease(workerId, 'replacement-00001', 0)).toBeUndefined(); + } else { + await setup(); await tick(restarted); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + expect(assignment).toBeDefined(); + await settle(assignment); await tick(restarted); + expect((await restarted.get(owner, requestId))?.state).toBe('completed'); + } +}); + +test('an offline worker cannot extend the queue deadline', async () => { + const requests = await setup(); await submit(requests); + const worker = `codeapi:bridge:v1:worker:${workerId}`; + await redis.del(worker, `${worker}:incarnation`, `${worker}:ready`); + await tick(requests); + expect((await requests.get(owner, requestId))?.state).toBe('queued'); + await mutateRecord(record => { record.queueDeadlineAtMs = Date.now() - 1; }); + await tick(requests); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'failed', error: { code: 'WORKSPACE_QUEUE_TIMEOUT' } }); + expect(await redis.zcard(activeKey)).toBe(0); +}); + +test('durable reset epochs outlive their receipts and reject late quarantine after reset', async () => { + const requests = await setup(2); await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + await settle(assignment); await tick(requests); + const receipt = `codeapi:bridge:v1:assignment:${assignment.assignmentId}:workspace-fence-owner`; + const fence = `codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update('native-workspace:primary').digest('hex')}:quarantined`; + expect(await redis.pttl(`${fence}:epoch`)).toBeGreaterThanOrEqual(await redis.pttl(receipt) - 50); + expect(await redis.pttl(`${fence}:epoch`)).toBeGreaterThan(86_000_000); + // Simulate execution-ownership TTL expiry. The durable receipt and epoch survive. + await redis.del(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`, + `codeapi:bridge:v1:worker:${workerId}:lock`, `codeapi:bridge:v1:worker:${workerId}:lock:incarnation`); + await bridge.resetWorkspace(workerId, incarnationId, 'native-workspace:primary'); + await submit(requests, 'request-00000000002'); await tick(requests); + const next = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + await expect(bridge.settle(workerId, assignment.assignmentId, { + protocolVersion: 1, incarnationId, generation: assignment.generation, leaseToken: assignment.leaseToken, + status: 'rejected', error: 'delayed local cleanup failure', + }, undefined, undefined, true)).rejects.toMatchObject({ code: 'ASSIGNMENT_FENCED' }); + expect(await redis.get(fence)).toBe(next.assignmentId); + await settle(next); await tick(requests); + expect((await requests.get(owner, 'request-00000000002'))?.state).toBe('completed'); +}); + +const coordinationPolicy: WorkspaceRequestCoordinationPolicy = { + bridgeEnabled: true, backend: 'remote-bridge', executionProfile: 'default', authMode: 'paired', + configuredWorkerId: '', dynamicWorkers: true, maxWorkspaceLeaseSlots: 2, maxCommandTimeoutMs: 30_000, +}; + +test('disabled bridges do not participate and each scheduling policy has an isolated scope', () => { + expect(workspaceRequestCoordinationScope({ ...coordinationPolicy, bridgeEnabled: false })).toBeUndefined(); + const scope = workspaceRequestCoordinationScope(coordinationPolicy); + expect(workspaceRequestCoordinationScope({ ...coordinationPolicy })).toBe(scope); + const variants: Partial[] = [ + { maxWorkspaceLeaseSlots: 1 }, { backend: 'http' }, { executionProfile: 'stateful' }, + { authMode: 'static' }, { configuredWorkerId: 'another' }, { dynamicWorkers: false }, + { maxCommandTimeoutMs: 60_000 }, + ]; + for (const variant of variants) expect(workspaceRequestCoordinationScope({ ...coordinationPolicy, ...variant })).not.toBe(scope); +}); + +test('a different policy cannot claim or fail work accepted by a two-slot bridge API', async () => { + await setup(2); + const scope = workspaceRequestCoordinationScope(coordinationPolicy)!; + const requests = new RedisWorkspaceRequests(redis, bridge, undefined, scope); + await submit(requests); + const foreignScope = workspaceRequestCoordinationScope({ ...coordinationPolicy, maxWorkspaceLeaseSlots: 1 })!; + const foreign = new RedisWorkspaceRequests(redis, new RedisBridgeStore(redis), undefined, foreignScope); + await foreign.reconcile(); + expect((await requests.get(owner, requestId))?.state).toBe('queued'); + expect(await foreign.get(owner, requestId)).toBeUndefined(); + const restarted = new RedisWorkspaceRequests(redis, new RedisBridgeStore(redis, 600, 1000, 2), undefined, scope); + await restarted.reconcile(); + const assignment = await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0); + expect(assignment).toBeDefined(); +}); + +test('a late FIFO response cannot relock a cancelled request after the coordinator timed out', async () => { + const requests = await setup(); await submit(requests); + let release!: () => void; + let entered!: () => void; + let finished!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { entered = resolve; }); + const drained = new Promise(resolve => { finished = resolve; }); + const originalHead = BridgeAdmissionQueue.prototype.isHead; + const originalAdvance = bridge.advanceDurableWorkspaceTool.bind(bridge); + const head = spyOn(BridgeAdmissionQueue.prototype, 'isHead').mockImplementation(async (worker, id) => { + const result = await originalHead.call(new BridgeAdmissionQueue(redis), worker, id); + entered(); await gate; return result; + }); + const advance = spyOn(bridge, 'advanceDurableWorkspaceTool').mockImplementation(async (...args) => { + try { return await originalAdvance(...args); } finally { finished(); } + }); + const pending = tick(requests); + void pending.catch(() => undefined); + try { + await started; + await expect(pending).rejects.toThrow('Durable workspace transition timed out'); + head.mockRestore(); advance.mockRestore(); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const claimDeadline = Date.now() + 1000; + while (await redis.get(`${key}:claim`) != null) { + if (Date.now() >= claimDeadline) throw new Error('Timed-out coordinator did not release its claim'); + await new Promise(resolve => setTimeout(resolve, 5)); + } + await requests.cancel(owner, requestId); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect((await requests.get(owner, requestId))?.state).toBe('cancelled'); + expect(await redis.zcard(activeKey)).toBe(0); + release(); await drained; + expect(await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBeNull(); + await submit(requests, 'request-00000000002'); await tick(requests); + expect(await bridge.lease(workerId, incarnationId, 0)).toBeDefined(); + } finally { release(); head.mockRestore(); advance.mockRestore(); await drained; } +}, 15_000); + +test.each([1, 2])('Redis capacity writes delayed past cancellation cannot strand a reservation (slots=%s)', async slots => { + const requests = await setup(slots); await submit(requests); + let release!: () => void; + let entered!: () => void; + const gate = new Promise(resolve => { release = resolve; }); + const started = new Promise(resolve => { entered = resolve; }); + const original = redis.eval.bind(redis); + let held = false; + const delay = spyOn(redis, 'eval').mockImplementation((async (...args: Parameters) => { + const script = String(args[0]); + if (!held && script.includes('local admissionTime') && !script.includes('local keyCount')) { + held = true; entered(); await gate; + } + return original(...args); + }) as Redis['eval']); + const pending = tick(requests); + void pending.catch(() => undefined); + try { + await started; + const key = (await redis.zrange(activeKey, 0, -1))[0]; + // Expire this claim while its Redis command is still undelivered. + await redis.del(`${key}:claim`); + await requests.cancel(owner, requestId); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect((await requests.get(owner, requestId))?.state).toBe('cancelled'); + release(); await pending; + expect(await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBeNull(); + expect(await redis.hlen(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`)).toBe(0); + expect(await redis.zcard(activeKey)).toBe(0); + } finally { release(); delay.mockRestore(); await pending.catch(() => undefined); } +}); + +test.each([1, 2])('a durable enqueue refuses lost capacity ownership (slots=%s)', async slots => { + const requests = await setup(slots); await submit(requests); + const original = redis.eval.bind(redis); + const loss = spyOn(redis, 'eval').mockImplementation((async (...args: Parameters) => { + if (String(args[0]).includes('local keyCount')) { + await redis.del(`codeapi:bridge:v1:worker:${workerId}:lock`, + `codeapi:bridge:v1:worker:${workerId}:workspace-slots`); + } + return original(...args); + }) as Redis['eval']); + try { await tick(requests); } finally { loss.mockRestore(); } + expect((await requests.get(owner, requestId))?.state).toBe('queued'); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0)).toBeUndefined(); + await tick(requests); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0)).toBeDefined(); +}); + +test.each([1, 2])('capacity acquisition rejects terminal, cancelled, expired, and unclaimed requests (slots=%s)', async slots => { + for (const denial of ['terminal', 'cancelled', 'expired', 'unclaimed']) { + await redis.flushall(); + const requests = await setup(slots); await submit(requests); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const record = JSON.parse((await redis.hget(key, 'record'))!); + const token = 'current-coordinator'; + await redis.set(`${key}:claim`, token, 'PX', 10000); + if (denial === 'terminal') await redis.hset(key, 'state', 'cancelled'); + if (denial === 'cancelled') await redis.hset(key, 'cancelRequested', '1'); + if (denial === 'expired') await redis.hset(key, 'queueDeadlineAtMs', Date.now() - 2000); + if (denial === 'unclaimed') await redis.del(`${key}:claim`); + await bridge.advanceDurableWorkspaceTool(record, { key, claimKey: `${key}:claim`, token }); + expect(await redis.get(`codeapi:bridge:v1:worker:${workerId}:lock`)).toBeNull(); + expect(await redis.hlen(`codeapi:bridge:v1:worker:${workerId}:workspace-slots`)).toBe(0); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0)).toBeUndefined(); + } +}); + +test.each([false, true])('quarantine retains the winning durable outcome across two-minute recovery (settled=%s)', async settled => { + const requests = await setup(2); + await requests.submit({ owner, requestId, workerId, requireTenantBinding: false, + request: { protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, + queueWaitMs: 300_000, executionTimeoutMs: 30_000, + }); + await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, assignment.generation, assignment.leaseToken); + const firstError = 'Command stopped before local cleanup'; + const quarantineError = 'Workspace cleanup failed before settlement'; + const envelope = { protocolVersion: 1 as const, incarnationId, + generation: assignment.generation, leaseToken: assignment.leaseToken, + status: 'rejected' as const, + }; + if (settled) await bridge.settle(workerId, assignment.assignmentId, { ...envelope, error: firstError }); + await bridge.settle(workerId, assignment.assignmentId, { ...envelope, error: quarantineError }, undefined, undefined, true); + const resultKey = `codeapi:bridge:v1:assignment:${assignment.assignmentId}:settlement`; + const receiptKey = `codeapi:bridge:v1:assignment:${assignment.assignmentId}:workspace-fence-owner`; + const receipt = JSON.parse((await redis.hget(receiptKey, 'metadata'))!); + expect(receipt.durableRequestKey).toBe((await redis.zrange(activeKey, 0, -1))[0]); + const remaining = await redis.pttl(resultKey); + expect(remaining).toBeGreaterThan(86_000_000); + expect(await redis.get(`codeapi:bridge:v1:assignment:${assignment.assignmentId}`)).toBeNull(); + // Advance retained storage and the coordinator clock without a two-minute sleep. + await redis.pexpire(resultKey, Math.max(0, remaining - 120_000)); + const clock = spyOn(Date, 'now').mockReturnValue(Date.now() + 120_000); + const restarted = new RedisWorkspaceRequests(redis, new RedisBridgeStore(redis, 600, 1000, 2)); + try { + await tick(restarted); + expect(await restarted.get(owner, requestId)).toMatchObject({ + state: 'failed', error: { code: 'WORKSPACE_TOOL_REJECTED', message: settled ? firstError : quarantineError }, + }); + expect(await redis.pttl(resultKey)).toBeGreaterThan(85_000_000); + expect(await redis.zcard(activeKey)).toBe(0); + await submit(restarted, 'request-00000000002'); await tick(restarted); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0)).toBeUndefined(); + expect(await restarted.get(owner, 'request-00000000002')).toMatchObject({ + state: 'failed', error: { code: 'WORKSPACE_QUARANTINED' }, + }); + } finally { clock.mockRestore(); } +}); + +test('queued durable requests retain FIFO workspace metadata across same-incarnation slot changes', async () => { + const requests = await setup(); + await submit(requests); + await submit(requests, 'request-00000000002'); + await submit(requests, 'request-00000000003', 'independent'); + await submit(requests); + await setup(2); + await tick(requests); await tick(requests); + const first = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + const independent = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 1))!; + expect(first).toBeDefined(); expect(independent).toBeDefined(); + expect(first.request).toMatchObject({ workspaceId: 'primary' }); + expect(independent.request).toMatchObject({ workspaceId: 'independent' }); + expect(await requests.get(owner, 'request-00000000002')).toMatchObject({ state: 'queued', queuePosition: 2 }); + await settle(first); await settle(independent); await tick(requests); + await bridge.confirmWorkspaceCleanup(workerId, first.assignmentId, { + protocolVersion: 1, incarnationId, generation: first.generation, leaseToken: first.leaseToken, + status: 'rejected', error: 'local cleanup confirmed', + }); + await tick(requests); + const next = (await bridge.lease(workerId, incarnationId, 0, undefined, undefined, 0))!; + expect(next).toBeDefined(); + await settle(next); await tick(requests); + expect((await requests.get(owner, 'request-00000000002'))?.state).toBe('completed'); +}); + +const readOnlyRequests: WorkspaceToolRequest[] = [ + { protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, + { protocolVersion: 1, operation: 'list_files', workspaceId: 'primary' }, + { protocolVersion: 1, operation: 'search_text', workspaceId: 'primary', query: 'needle' }, + { protocolVersion: 1, operation: 'preview_edit', workspaceId: 'primary', path: 'README.md', oldText: 'old', newText: 'new' }, +]; +const readCancellationCases = readOnlyRequests.flatMap(request => [1, 2].flatMap(slots => + ['unleased', 'claimed', 'acknowledged'].map(phase => ({ request, slots, phase })), +)); + +test.each(readCancellationCases)('durable read cancellation notifies the worker before closing (%j)', async ({ request, slots, phase }) => { + const requests = await setup(slots); + await requests.submit({ owner, requestId, workerId, requireTenantBinding: false, + request, queueWaitMs: 300_000, executionTimeoutMs: 30_000, + }); + await tick(requests); + const slot = slots === 1 ? undefined : 0; + let assignment = phase === 'unleased' ? undefined : await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slot); + if (phase === 'acknowledged') await bridge.acknowledgeLease(workerId, incarnationId, assignment!.assignmentId, + assignment!.generation, assignment!.leaseToken); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const stored = JSON.parse((await redis.hget(key, 'assignment'))!); + await requests.cancel(owner, requestId); + const cancelling = tick(new RedisWorkspaceRequests(redis, bridge)); + const marker = `codeapi:bridge:v1:assignment:${stored.assignmentId}:cancelled`; + const limit = Date.now() + 1000; + try { + while (await redis.get(marker) !== '1') { + if (Date.now() >= limit) throw new Error('Read-only worker was never notified'); + await new Promise(resolve => setTimeout(resolve, 5)); + } + expect((await requests.get(owner, requestId))?.state).toBe('admitted'); + assignment ??= await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slot); + expect(assignment).toBeDefined(); + expect(await bridge.cancelled(workerId, incarnationId, assignment!.assignmentId)).toBe(true); + await settle(assignment!, false); + await cancelling; + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'cancelled', cancelRequested: true }); + expect(await redis.zcard(activeKey)).toBe(0); + } finally { await cancelling; } +}); + +test.each([1, 2])('a durable read cancellation remains cancelled without worker settlement (slots=%s)', async slots => { + const requests = await setup(slots); await submit(requests); await tick(requests); + const key = (await redis.zrange(activeKey, 0, -1))[0]; + const stored = JSON.parse((await redis.hget(key, 'assignment'))!); + await requests.cancel(owner, requestId); + await tick(new RedisWorkspaceRequests(redis, bridge)); + expect(await redis.get(`codeapi:bridge:v1:assignment:${stored.assignmentId}:cancelled`)).toBe('1'); + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'cancelled', cancelRequested: true }); + expect((await requests.get(owner, requestId))?.error).toBeUndefined(); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0)).toBeUndefined(); + expect(await redis.zcard(activeKey)).toBe(0); + await submit(requests, 'request-00000000002'); await tick(requests); + expect(await bridge.lease(workerId, incarnationId, 0, undefined, undefined, slots === 1 ? undefined : 0)).toBeDefined(); +}, 10_000); + +test('fulfillment racing durable read cancellation keeps the winning result', async () => { + const requests = await setup(); await submit(requests); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await requests.cancel(owner, requestId); + const cancelling = tick(requests); + const marker = `codeapi:bridge:v1:assignment:${assignment.assignmentId}:cancelled`; + const limit = Date.now() + 1000; + try { + while (await redis.get(marker) !== '1') { + if (Date.now() >= limit) throw new Error('Read-only worker was never notified'); + await new Promise(resolve => setTimeout(resolve, 5)); + } + await settle(assignment); + await cancelling; + expect(await requests.get(owner, requestId)).toMatchObject({ state: 'completed', result: { content: 'hello' } }); + } finally { await cancelling; } +}); + +test('queued workspace metadata also permits a same-incarnation return to serial admission', async () => { + const requests = await setup(2); await submit(requests); await setup(); await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + expect(assignment).toBeDefined(); + expect(assignment.workspaceLeaseSlot).toBeUndefined(); + await settle(assignment); await tick(requests); + expect((await requests.get(owner, requestId))?.state).toBe('completed'); +}); + +test('unconfirmed durable mutation cancellation still reports an unknown outcome and retains its fence', async () => { + const requests = await setup(); + await requests.submit({ owner, requestId, workerId, requireTenantBinding: false, + request: { protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'sleep 30' }, + queueWaitMs: 300_000, executionTimeoutMs: 35_000, + }); + await tick(requests); + const assignment = (await bridge.lease(workerId, incarnationId, 0))!; + await bridge.acknowledgeLease(workerId, incarnationId, assignment.assignmentId, assignment.generation, assignment.leaseToken); + await requests.cancel(owner, requestId); await tick(requests); + expect(await requests.get(owner, requestId)).toMatchObject({ + state: 'failed', cancelRequested: true, error: { code: 'ASSIGNMENT_EXPIRED' }, + }); + const fence = `codeapi:bridge:v1:worker:${workerId}:workspace:${createHash('sha256').update('native-workspace:primary').digest('hex')}:quarantined`; + expect(await redis.get(fence)).toBe(assignment.assignmentId); +}, 10_000); diff --git a/service/src/workspace-tools/requests.ts b/service/src/workspace-tools/requests.ts new file mode 100644 index 00000000..2988c7cc --- /dev/null +++ b/service/src/workspace-tools/requests.ts @@ -0,0 +1,251 @@ +import { createHash, randomBytes } from 'crypto'; +import type Redis from 'ioredis'; +import type { WorkspaceToolRequest, WorkspaceToolResult } from '../../../packages/code/src/protocol'; +import type { RedisBridgeStore, RegisteredBridgeWorker, DurableWorkspaceAssignment, CodeBridgeWorkspaceSettlement } from '../bridge/store'; +import { BridgeAdmissionQueue } from '../bridge/admission'; +import { BridgeStoreError, workspaceAdmissionId } from '../bridge/store'; + +const PREFIX = 'codeapi:workspace-requests:v1'; +const ACTIVE = `${PREFIX}:active`; +const RETENTION_MS = 24 * 60 * 60_000; +const CLAIM_MS = 10_000; +export const WORKSPACE_REQUEST_ID_PATTERN = /^[A-Za-z0-9_-]{16,128}$/; + +export class WorkspaceRequestConflict extends Error {} + +export interface WorkspaceRequestOwner { + tenantId: string; + userId: string; +} + +export interface StoredWorkspaceRequest { + id: string; + key: string; + fingerprint: string; + workerId: string; + tenantId: string; + requireTenantBinding: boolean; + request: WorkspaceToolRequest; + registration: RegisteredBridgeWorker; + createdAtMs: number; + queueDeadlineAtMs: number; + executionTimeoutMs: number; + assignment?: DurableWorkspaceAssignment; + settlement?: CodeBridgeWorkspaceSettlement; + admittedAtMs?: number; + finishedAtMs?: number; + state: 'queued' | 'admitted' | 'completed' | 'failed' | 'cancelled'; + cancelRequested?: boolean; + result?: WorkspaceToolResult; + error?: { code: string; message: string }; +} + +export interface WorkspaceRequestStatus { + requestId: string; + state: StoredWorkspaceRequest['state']; + workerId: string; + workspaceId: string; + workspaceInstanceId?: string; + worktree?: string; + queuePosition?: number; + queueWaitMs: number; + executionDeadlineAt?: string; + cancelRequested: boolean; + result?: WorkspaceToolResult; + error?: StoredWorkspaceRequest['error']; +} + +function canonical(value: unknown): string { + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + if (value !== null && typeof value === 'object') { + const record = value as Record; + return `{${Object.keys(record).filter(key => record[key] !== undefined).sort() + .map(key => `${JSON.stringify(key)}:${canonical(record[key])}`).join(',')}}`; + } + return JSON.stringify(value); +} + +async function deadline(operation: Promise): Promise { + let timer: ReturnType | undefined; + try { + return await Promise.race([operation, new Promise((_, reject) => { + timer = setTimeout(() => reject(new Error('Durable workspace transition timed out')), 8_000); + timer.unref(); + })]); + } finally { clearTimeout(timer); } +} + +function requestKey(owner: WorkspaceRequestOwner, requestId: string, scope: string): string { + return `${PREFIX}:${createHash('sha256').update(canonical([scope, owner.tenantId, owner.userId, requestId])).digest('hex')}`; +} + +/** Redis owns requests; API replicas only advance bounded, fenced transitions. */ +export class RedisWorkspaceRequests { + private readonly activeKey: string; + constructor( + private readonly redis: Redis, + private readonly bridge: RedisBridgeStore, + private readonly transition?: (record: StoredWorkspaceRequest) => void, + private readonly scope = '', + ) { + this.activeKey = scope.length === 0 ? ACTIVE : `${PREFIX}:${scope}:active`; + } + + async submit(args: { + owner: WorkspaceRequestOwner; + requestId: string; + workerId: string; + requireTenantBinding: boolean; + request: WorkspaceToolRequest; + queueWaitMs: number; + executionTimeoutMs: number; + }): Promise { + if (!WORKSPACE_REQUEST_ID_PATTERN.test(args.requestId)) { + throw new BridgeStoreError('ASSIGNMENT_INVALID', 'Invalid durable workspace request ID'); + } + const key = requestKey(args.owner, args.requestId, this.scope); + const fingerprint = createHash('sha256').update(canonical({ + workerId: args.workerId, request: args.request, + queueWaitMs: args.queueWaitMs, executionTimeoutMs: args.executionTimeoutMs, + requireTenantBinding: args.requireTenantBinding, + })).digest('hex'); + const existing = await this.read(key); + if (existing != null) { + if (existing.fingerprint !== fingerprint) throw new WorkspaceRequestConflict('Request ID was already used for different work'); + return this.status(existing, args.requestId); + } + const registration = await this.bridge.prepareDurableWorkspaceTool({ + ...args, tenantId: args.owner.tenantId, + }); + const now = Date.now(); + const record: StoredWorkspaceRequest = { + id: `durable-${randomBytes(18).toString('base64url')}`, key, fingerprint, + workerId: args.workerId, tenantId: args.owner.tenantId, + requireTenantBinding: args.requireTenantBinding, + request: args.request, registration, createdAtMs: now, + queueDeadlineAtMs: now + args.queueWaitMs, + executionTimeoutMs: args.executionTimeoutMs, state: 'queued', + }; + const queue = new BridgeAdmissionQueue(this.redis); + const accepted = await queue.submit({ + workerId: record.workerId, id: record.id, deadlineAtMs: record.queueDeadlineAtMs, + workspaceId: workspaceAdmissionId(record.request.workspaceId, record.request.workspaceInstanceId, record.request.worktree), + key, activeKey: this.activeKey, fingerprint, record: JSON.stringify(record), retentionMs: RETENTION_MS, + }); + if (accepted === 'conflict') throw new WorkspaceRequestConflict('Request ID was already used for different work'); + if (accepted === 'full') throw new BridgeStoreError('WORKER_QUEUE_FULL', 'Bridge worker pending request limit reached'); + const submitted = (await this.read(key))!; + if (accepted === 'accepted') this.transition?.(submitted); + return this.status(submitted, args.requestId); + } + + async get(owner: WorkspaceRequestOwner, requestId: string): Promise { + if (!WORKSPACE_REQUEST_ID_PATTERN.test(requestId)) return undefined; + const record = await this.read(requestKey(owner, requestId, this.scope)); + return record == null ? undefined : this.status(record, requestId); + } + + async cancel(owner: WorkspaceRequestOwner, requestId: string): Promise { + if (!WORKSPACE_REQUEST_ID_PATTERN.test(requestId)) return undefined; + const key = requestKey(owner, requestId, this.scope); + await this.redis.eval([ + 'local state = redis.call(\'HGET\', KEYS[1], \'state\')', + 'if state ~= \'queued\' and state ~= \'admitted\' then return 0 end', + 'redis.call(\'HSET\', KEYS[1], \'cancelRequested\', \'1\')', + 'redis.call(\'ZADD\', KEYS[2], ARGV[1], KEYS[1])', + 'return 1', + ].join('\n'), 2, key, this.activeKey, Date.now()); + return this.get(owner, requestId); + } + + private async read(key: string): Promise { + const fields: Partial> = await this.redis.hgetall(key); + if (fields.record == null) return undefined; + return { + ...JSON.parse(fields.record) as StoredWorkspaceRequest, + ...(fields.outcome == null ? {} : JSON.parse(fields.outcome)), + state: fields.state as StoredWorkspaceRequest['state'], + assignment: fields.assignment == null ? undefined : JSON.parse(fields.assignment), + admittedAtMs: fields.admittedAtMs == null ? undefined : Number(fields.admittedAtMs), + cancelRequested: fields.cancelRequested === '1', + }; + } + + private async status(record: StoredWorkspaceRequest, requestId: string): Promise { + const rank = record.state === 'queued' + ? await new BridgeAdmissionQueue(this.redis).position(record.workerId, record.id) + : undefined; + return { + requestId, state: record.state, workerId: record.workerId, + workspaceId: record.request.workspaceId, + workspaceInstanceId: record.request.workspaceInstanceId, worktree: record.request.worktree, + queuePosition: rank, queueWaitMs: Math.max(0, (record.admittedAtMs ?? Math.min(record.finishedAtMs ?? Date.now(), record.queueDeadlineAtMs)) - record.createdAtMs), + executionDeadlineAt: record.assignment?.expiresAt, + cancelRequested: record.cancelRequested === true, + result: record.result, error: record.error, + }; + } + + async reconcile(): Promise { + const keys = await deadline(this.redis.zrangebyscore(this.activeKey, '-inf', Date.now(), 'LIMIT', 0, 32)); + const results = await Promise.allSettled(keys.map(key => deadline(this.advance(key)))); + const failure = results.find(result => result.status === 'rejected'); + if (failure?.status === 'rejected') throw failure.reason; + } + + private async advance(key: string): Promise { + const claimKey = `${key}:claim`; + const token = randomBytes(18).toString('hex'); + const claimed = await this.redis.eval([ + 'if not redis.call(\'SET\', KEYS[1], ARGV[1], \'PX\', ARGV[2], \'NX\') then return 0 end', + 'redis.call(\'ZADD\', KEYS[2], ARGV[3], KEYS[3])', + 'return 1', + ].join('\n'), 3, claimKey, this.activeKey, key, token, CLAIM_MS, Date.now() + CLAIM_MS); + if (Number(claimed) !== 1) return; + try { + let record = await this.read(key); + if (record == null) { await this.redis.zrem(this.activeKey, key); return; } + if (record.state === 'queued' || record.state === 'admitted') { + const previousState = record.state; + try { + const outcome = await deadline(this.bridge.advanceDurableWorkspaceTool(record, { key, claimKey, token })); + if (outcome != null) { + await this.redis.eval([ + 'if redis.call(\'GET\', KEYS[2]) ~= ARGV[1] then return 0 end', + 'local state = redis.call(\'HGET\', KEYS[1], \'state\')', + 'if state ~= \'queued\' and state ~= \'admitted\' then return 0 end', + 'redis.call(\'HSET\', KEYS[1], \'state\', ARGV[3], \'outcome\', ARGV[2])', + 'return 1', + ].join('\n'), 2, key, claimKey, token, JSON.stringify({ ...outcome, finishedAtMs: Date.now() }), outcome.state); + } + } catch (error) { + // Infrastructure uncertainty is reconciled, never retried as a new command. + if (!(error instanceof BridgeStoreError)) throw error; + if (error.code === 'WORKER_OFFLINE') { + // The original worker may reconnect within the remaining admission budget. + await this.redis.zadd(this.activeKey, Date.now() + 250, key); + return; + } + const current = await this.read(key); + if (current?.state === 'admitted') throw error; + await this.redis.eval([ + 'if redis.call(\'GET\', KEYS[2]) ~= ARGV[1] then return 0 end', + 'if redis.call(\'HGET\', KEYS[1], \'state\') ~= \'queued\' then return 0 end', + 'redis.call(\'HSET\', KEYS[1], \'state\', \'failed\', \'outcome\', ARGV[2])', + 'return 1', + ].join('\n'), 2, key, claimKey, token, JSON.stringify({ state: 'failed', finishedAtMs: Date.now(), error: { code: error.code, message: error.message } })); + } + record = (await this.read(key))!; + if (record.state !== previousState) this.transition?.(record); + } + if (record.state !== 'queued' && record.state !== 'admitted') { + await deadline(this.bridge.finishDurableWorkspaceTool(record)); + await this.redis.zrem(this.activeKey, key); + } else { + await this.redis.zadd(this.activeKey, Date.now() + 250, key); + } + } finally { + await this.redis.eval('if redis.call(\'GET\', KEYS[1]) == ARGV[1] then return redis.call(\'DEL\', KEYS[1]) end return 0', 1, claimKey, token); + } + } +} diff --git a/service/src/workspace-tools/router.test.ts b/service/src/workspace-tools/router.test.ts index a52b3f0b..8a753c08 100644 --- a/service/src/workspace-tools/router.test.ts +++ b/service/src/workspace-tools/router.test.ts @@ -67,13 +67,17 @@ test('binds instance admission to the authenticated tenant and user while preser expect(dispatched[1]?.workspaceInstanceId).toBeUndefined(); }); -test.each<[WorkspaceToolRequest, number, number?]>([ - [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 300_000 }, 305_000, undefined], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 6000, 1000], - [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, 600_000], -])('separates the admission deadline from execution budget for %j', async (request, expectedExecution, ceiling) => { +test.each<[WorkspaceToolRequest, number, number?, number?, number?]>([ + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined, undefined, undefined], + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, 125_000, undefined, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, undefined, undefined, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 90_000 }, 95_000, 125_000, undefined, 90_000], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready', timeoutMs: 300_000 }, 305_000, undefined, undefined, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 6000, 1000, undefined, undefined], + [{ protocolVersion: 1, operation: 'execute_command', workspaceId: 'primary', command: 'echo ready' }, 35_000, 600_000, undefined, undefined], + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, 125_000, 5000, 90_000], + [{ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }, 30_000, undefined, undefined, 300_000], +])('separates the admission deadline from execution budget for %j', async (request, expectedExecution, ceiling, queueTimeoutMs, advertisedQueueWaitMs) => { const app = express(); app.use(json()); app.use((req, _res, next) => { @@ -86,6 +90,7 @@ test.each<[WorkspaceToolRequest, number, number?]>([ app.use(createWorkspaceToolsRouter({ backend: 'remote-bridge', configuredWorkerId: 'user-worker', dynamicWorkers: false, timeoutMs: ceiling, + queueTimeoutMs, store: { async dispatchWorkspaceTool(args) { executionBudget = args.executionTimeoutMs; if (args.request.operation === 'execute_command') commandTimeout = args.request.timeoutMs; @@ -98,13 +103,53 @@ test.each<[WorkspaceToolRequest, number, number?]>([ const address = server.address(); if (address == null || typeof address === 'string') throw new Error('Missing listener'); const response = await fetch(`http://127.0.0.1:${address.port}/workspace-tools/execute`, { - method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(request), + method: 'POST', headers: { + 'Content-Type': 'application/json', + ...(advertisedQueueWaitMs === undefined ? {} : { 'X-LibreChat-Workspace-Queue-Wait-Ms': String(advertisedQueueWaitMs) }), + }, body: JSON.stringify(request), }); await response.json(); expect(executionBudget).toBe(expectedExecution); if (request.operation === 'execute_command') expect(commandTimeout).toBe(expectedExecution - 5000); - expect(queueRemaining).toBeGreaterThan(29_000); - expect(queueRemaining).toBeLessThanOrEqual(30_000); + const expectedQueueBudget = Math.min(advertisedQueueWaitMs ?? 30_000, queueTimeoutMs ?? 300_000); + expect(queueRemaining).toBeGreaterThan(expectedQueueBudget - 1000); + expect(queueRemaining).toBeLessThanOrEqual(expectedQueueBudget); +}); + +test.each([0, 300_001, Number.POSITIVE_INFINITY])('rejects an unbounded queue override (%s)', (queueTimeoutMs) => { + expect(() => createWorkspaceToolsRouter({ + backend: 'remote-bridge', + configuredWorkerId: 'user-worker', + dynamicWorkers: false, + queueTimeoutMs, + store: { async dispatchWorkspaceTool() { throw new Error('must not dispatch'); } }, + })).toThrow('Workspace queue timeout must be between 1 and 300000 milliseconds'); +}); + +test.each(['0', '-1', '300001', '1.5', '01', '1, 2', '999999999999999999999'])('rejects invalid per-request queue allowance %j before dispatch', async (queueWait) => { + const app = express(); + app.use(json()); + app.use((req, _res, next) => { + applyPrincipal(req, { userId: 'user-1', tenantId: 'tenant-1', principalSource: 'librechat_jwt', codeWorkerId: 'user-worker' }); + next(); + }); + let dispatched = false; + app.use(createWorkspaceToolsRouter({ + backend: 'remote-bridge', configuredWorkerId: 'user-worker', dynamicWorkers: false, + store: { async dispatchWorkspaceTool() { dispatched = true; throw new Error('must not dispatch'); } }, + })); + server = createServer(app); + await new Promise(resolve => server!.listen(0, '127.0.0.1', resolve)); + const address = server.address(); + if (address == null || typeof address === 'string') throw new Error('Missing listener'); + const response = await fetch(`http://127.0.0.1:${address.port}/workspace-tools/execute`, { + method: 'POST', + headers: { 'Content-Type': 'application/json', 'X-LibreChat-Workspace-Queue-Wait-Ms': queueWait }, + body: JSON.stringify({ protocolVersion: 1, operation: 'read_file', workspaceId: 'primary', path: 'README.md' }), + }); + expect(response.status).toBe(400); + expect(await response.json()).toMatchObject({ code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + expect(dispatched).toBe(false); }); test('rejects new workspace dispatches while the service is shutting down', async () => { @@ -178,6 +223,8 @@ test.each([ ['COMMAND_TIMEOUT', 504], ['COMMAND_UNAVAILABLE', 503], ['COMMAND_DISABLED', 403], + ['INVALID_PATH', 422], + ['NOT_FOUND', 422], ] as const)('maps worker %s rejections to HTTP %i', async (errorCode, expectedStatus) => { const app = express(); app.use(json()); @@ -486,6 +533,7 @@ test('logs a disconnected dispatch once without inventing HTTP 200', async () => const closed = Promise.withResolvers(); const settlementGate = Promise.withResolvers(); let dispatchAborted = false; + let queueRemaining: number | undefined; let closeConnection = (): void => { throw new Error('connection not ready'); }; app.use(json()); app.use((req, res, next) => { @@ -499,8 +547,10 @@ test('logs a disconnected dispatch once without inventing HTTP 200', async () => backend: 'remote-bridge', configuredWorkerId: 'user-worker', dynamicWorkers: true, + timeoutMs: 125_000, store: { - async dispatchWorkspaceTool({ signal }) { + async dispatchWorkspaceTool({ deadlineAtMs, signal }) { + queueRemaining = deadlineAtMs - Date.now(); started.resolve(); return await new Promise((_resolve, reject) => { signal.addEventListener( @@ -531,6 +581,8 @@ test('logs a disconnected dispatch once without inventing HTTP 200', async () => closeConnection(); await expect(response).rejects.toThrow(); await closed.promise; + expect(queueRemaining).toBeGreaterThan(29_000); + expect(queueRemaining).toBeLessThanOrEqual(30_000); expect(dispatchAborted).toBe(true); expect(logSpy).not.toHaveBeenCalled(); settlementGate.resolve(); diff --git a/service/src/workspace-tools/router.ts b/service/src/workspace-tools/router.ts index 3e1f9fee..d4d65d14 100644 --- a/service/src/workspace-tools/router.ts +++ b/service/src/workspace-tools/router.ts @@ -19,9 +19,16 @@ import { resolveBridgeWorkerSelection, } from '../bridge/selection'; import { principalWorkspaceInstanceId } from '../bridge/workspace-instance'; +import { WorkspaceRequestConflict } from './requests'; +import type { RedisWorkspaceRequests } from './requests'; + +const MAX_WORKSPACE_QUEUE_WAIT_MS = 5 * 60_000; +const DEFAULT_WORKSPACE_QUEUE_WAIT_MS = 30_000; +const WORKSPACE_QUEUE_WAIT_HEADER = 'X-LibreChat-Workspace-Queue-Wait-Ms'; interface WorkspaceToolsRouterOptions { store: Pick; + requests?: RedisWorkspaceRequests; backend: 'http' | 'lambda-microvm' | 'remote-bridge'; configuredWorkerId: string; dynamicWorkers: boolean; @@ -50,157 +57,213 @@ export function bridgeStoreStatus(error: BridgeStoreError): number { } export function createWorkspaceToolsRouter(options: WorkspaceToolsRouterOptions): Router { - const queueBudgetMs = options.queueTimeoutMs ?? 30_000; - if (!Number.isSafeInteger(queueBudgetMs) || queueBudgetMs < 1 || queueBudgetMs > 30_000) { - throw new RangeError('Workspace queue timeout must be between 1 and 30000 milliseconds'); - } if (options.timeoutMs !== undefined && ( !Number.isSafeInteger(options.timeoutMs) || options.timeoutMs < 1 )) { throw new RangeError('Workspace execution timeout must be a positive safe integer'); } + // A configured queue timeout is a ceiling. Legacy callers get 30 seconds even + // when their execution timeout is longer; only the per-request header opts in. + const queueCeilingMs = options.queueTimeoutMs ?? MAX_WORKSPACE_QUEUE_WAIT_MS; + if (!Number.isSafeInteger(queueCeilingMs) || queueCeilingMs < 1 || queueCeilingMs > MAX_WORKSPACE_QUEUE_WAIT_MS) { + throw new RangeError('Workspace queue timeout must be between 1 and 300000 milliseconds'); + } const router = Router(); - router.post( - '/workspace-tools/execute', - asyncRoute(async (req, res) => { - const outcome = getWorkspaceToolOutcome(res); - const principal = getPrincipalOrReject(req, res); - if (!principal) { - outcome.errorCode = 'UNAUTHENTICATED'; - return; - } - if ((options.isShuttingDown ?? checkServiceShutDown)()) { - outcome.errorCode = 'SERVICE_SHUTTING_DOWN'; - res.status(503).json({ error: 'Service is shutting down' }); + const execute = (durable: boolean): RequestHandler => asyncRoute(async (req, res) => { + const outcome = getWorkspaceToolOutcome(res, req.path); + const principal = getPrincipalOrReject(req, res); + if (!principal) { + outcome.errorCode = 'UNAUTHENTICATED'; + return; + } + if ((options.isShuttingDown ?? checkServiceShutDown)()) { + outcome.errorCode = 'SERVICE_SHUTTING_DOWN'; + res.status(503).json({ error: 'Service is shutting down' }); + return; + } + if (!isWorkspaceToolRequest(req.body)) { + outcome.errorCode = 'INVALID_WORKSPACE_TOOL_REQUEST'; + res.status(400).json({ + error: 'Invalid workspace tool request', + }); + return; + } + const advertisedQueueWait = req.header(WORKSPACE_QUEUE_WAIT_HEADER); + if (advertisedQueueWait !== undefined && !/^[1-9]\d*$/.test(advertisedQueueWait)) { + outcome.errorCode = 'INVALID_WORKSPACE_QUEUE_WAIT'; + res.status(400).json({ error: 'Invalid workspace queue wait', code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + return; + } + const requestedQueueWaitMs = advertisedQueueWait === undefined + ? DEFAULT_WORKSPACE_QUEUE_WAIT_MS + : Number(advertisedQueueWait); + if (!Number.isSafeInteger(requestedQueueWaitMs) || requestedQueueWaitMs > MAX_WORKSPACE_QUEUE_WAIT_MS) { + outcome.errorCode = 'INVALID_WORKSPACE_QUEUE_WAIT'; + res.status(400).json({ error: 'Invalid workspace queue wait', code: 'INVALID_WORKSPACE_QUEUE_WAIT' }); + return; + } + const queueBudgetMs = Math.min(requestedQueueWaitMs, queueCeilingMs); + outcome.operation = req.body.operation; + const principalRequest: WorkspaceToolRequest = req.body.workspaceInstanceId == null + ? req.body + : { + ...req.body, + workspaceInstanceId: principalWorkspaceInstanceId({ + instanceId: req.body.workspaceInstanceId, + tenantId: principal.tenantId, + principalId: principal.userId, + }), + }; + const request: WorkspaceToolRequest = principalRequest.operation === 'execute_command' + ? { ...principalRequest, timeoutMs: Math.min( + principalRequest.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, + options.timeoutMs ?? Number.MAX_SAFE_INTEGER, + ) } + : principalRequest; + const executionBudgetMs = request.operation === 'execute_command' + ? request.timeoutMs! + 5_000 + : Math.min(options.timeoutMs ?? 30_000, 30_000); + outcome.deadlineBudgetMs = queueBudgetMs + executionBudgetMs; + + let selection: { workerId: string; explicit: boolean } | undefined; + try { + selection = resolveBridgeWorkerSelection({ + backend: options.backend, + configuredWorkerId: options.configuredWorkerId, + dynamicWorkers: options.dynamicWorkers, + requestedWorkerId: req.header(CODEAPI_BRIDGE_WORKER_HEADER), + trustedWorkerId: principal.codeWorkerId, + }); + } catch (error) { + if (error instanceof BridgeWorkerSelectionError) { + outcome.errorCode = 'WORKER_SELECTION_REJECTED'; + res.status(error.status).json({ error: error.message }); return; } - if (!isWorkspaceToolRequest(req.body)) { - outcome.errorCode = 'INVALID_WORKSPACE_TOOL_REQUEST'; - res.status(400).json({ - error: 'Invalid workspace tool request', - }); + throw error; + } + if (selection == null) { + outcome.errorCode = 'WORKSPACE_BACKEND_UNAVAILABLE'; + res.status(503).json({ + error: 'Workspace tools require the remote-bridge backend', + }); + return; + } + outcome.workerId = selection.workerId; + + if (durable) { + if (options.requests == null) { + res.status(404).json({ error: 'Durable workspace requests are unavailable' }); return; } - outcome.operation = req.body.operation; - const principalRequest: WorkspaceToolRequest = req.body.workspaceInstanceId == null - ? req.body - : { - ...req.body, - workspaceInstanceId: principalWorkspaceInstanceId({ - instanceId: req.body.workspaceInstanceId, - tenantId: principal.tenantId, - principalId: principal.userId, - }), - }; - const request: WorkspaceToolRequest = principalRequest.operation === 'execute_command' - ? { ...principalRequest, timeoutMs: Math.min( - principalRequest.timeoutMs ?? BRIDGE_WORKSPACE_COMMAND_DEFAULT_TIMEOUT_MS, - options.timeoutMs ?? Number.MAX_SAFE_INTEGER, - ) } - : principalRequest; - const executionBudgetMs = request.operation === 'execute_command' - ? request.timeoutMs! + 5_000 - : Math.min(options.timeoutMs ?? 30_000, 30_000); - outcome.deadlineBudgetMs = queueBudgetMs + executionBudgetMs; - - let selection: { workerId: string; explicit: boolean } | undefined; try { - selection = resolveBridgeWorkerSelection({ - backend: options.backend, - configuredWorkerId: options.configuredWorkerId, - dynamicWorkers: options.dynamicWorkers, - requestedWorkerId: req.header(CODEAPI_BRIDGE_WORKER_HEADER), - trustedWorkerId: principal.codeWorkerId, + const status = await options.requests.submit({ + owner: principal, requestId: req.header('X-LibreChat-Workspace-Request-Id') ?? '', + workerId: selection.workerId, + requireTenantBinding: selection.explicit && (options.dynamicWorkers || selection.workerId !== options.configuredWorkerId), + request, queueWaitMs: queueBudgetMs, executionTimeoutMs: executionBudgetMs, }); + res.status(202).json(status); } catch (error) { - if (error instanceof BridgeWorkerSelectionError) { - outcome.errorCode = 'WORKER_SELECTION_REJECTED'; - res.status(error.status).json({ error: error.message }); - return; - } - throw error; - } - if (selection == null) { - outcome.errorCode = 'WORKSPACE_BACKEND_UNAVAILABLE'; - res.status(503).json({ - error: 'Workspace tools require the remote-bridge backend', - }); - return; + if (error instanceof WorkspaceRequestConflict) { + res.status(409).json({ error: error.message, code: 'REQUEST_CONFLICT' }); + } else if (error instanceof BridgeStoreError) { + res.status(bridgeStoreStatus(error)).json({ error: error.message, code: error.code }); + } else throw error; } - outcome.workerId = selection.workerId; + return; + } - const controller = new AbortController(); - const abort = (): void => controller.abort(); - req.once('aborted', abort); - const abortClosedResponse = (): void => { - if (!res.writableEnded) abort(); - }; - res.once('close', abortClosedResponse); - try { - outcome.dispatchPending = true; - const dispatchStartedAt = performance.now(); - const settlement = await options.store.dispatchWorkspaceTool({ - workerId: selection.workerId, - tenantId: principal.tenantId, - requireTenantBinding: + const controller = new AbortController(); + const abort = (): void => controller.abort(); + req.once('aborted', abort); + const abortClosedResponse = (): void => { + if (!res.writableEnded) abort(); + }; + res.once('close', abortClosedResponse); + try { + outcome.dispatchPending = true; + const dispatchStartedAt = performance.now(); + const settlement = await options.store.dispatchWorkspaceTool({ + workerId: selection.workerId, + tenantId: principal.tenantId, + requireTenantBinding: selection.explicit && (options.dynamicWorkers || selection.workerId !== options.configuredWorkerId), - request, - deadlineAtMs: Date.now() + queueBudgetMs, - executionTimeoutMs: executionBudgetMs, - signal: controller.signal, - }).finally(() => { - outcome.dispatchDurationMs = Math.round(performance.now() - dispatchStartedAt); - }); - if (settlement.status === 'rejected') { - outcome.errorCode = settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED'; - let status = 422; - if ( - settlement.errorCode === 'SEARCH_TIMEOUT' || + request, + deadlineAtMs: Date.now() + queueBudgetMs, + executionTimeoutMs: executionBudgetMs, + signal: controller.signal, + }).finally(() => { + outcome.dispatchDurationMs = Math.round(performance.now() - dispatchStartedAt); + }); + if (settlement.status === 'rejected') { + outcome.errorCode = settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED'; + let status = 422; + if ( + settlement.errorCode === 'SEARCH_TIMEOUT' || settlement.errorCode === 'LIST_TIMEOUT' || settlement.errorCode === 'COMMAND_TIMEOUT' - ) { - status = 504; - } - if ( - settlement.errorCode === 'SEARCH_UNAVAILABLE' || + ) { + status = 504; + } + if ( + settlement.errorCode === 'SEARCH_UNAVAILABLE' || settlement.errorCode === 'LIST_UNAVAILABLE' || settlement.errorCode === 'COMMAND_UNAVAILABLE' - ) { - status = 503; - } - if (settlement.errorCode === 'WRITE_DISABLED') status = 403; - if (settlement.errorCode === 'COMMAND_DISABLED') status = 403; - if (settlement.errorCode === 'WRITE_LIMIT_EXCEEDED') status = 413; - if (settlement.errorCode === 'WRITE_UNAVAILABLE') status = 503; - if (settlement.errorCode === 'EDIT_CONFLICT') status = 409; - res.status(status).json({ - error: settlement.error, - code: settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED', - }); - return; - } - res.status(200).json(settlement.result); - } catch (error) { - if (error instanceof BridgeStoreError) { - outcome.errorCode = error.code; - if (error.code === 'WORKSPACE_QUEUE_TIMEOUT') res.setHeader('Retry-After', '1'); - res.status(bridgeStoreStatus(error)).json({ - error: error.message, - code: error.code, - }); - return; + ) { + status = 503; } - outcome.errorCode = 'INTERNAL_ERROR'; - throw error; - } finally { - outcome.dispatchPending = false; - outcome.flush(); - req.removeListener('aborted', abort); - res.removeListener('close', abortClosedResponse); + if (settlement.errorCode === 'WRITE_DISABLED') status = 403; + if (settlement.errorCode === 'COMMAND_DISABLED') status = 403; + if (settlement.errorCode === 'WRITE_LIMIT_EXCEEDED') status = 413; + if (settlement.errorCode === 'WRITE_UNAVAILABLE') status = 503; + if (settlement.errorCode === 'EDIT_CONFLICT') status = 409; + res.status(status).json({ + error: settlement.error, + code: settlement.errorCode ?? 'WORKSPACE_TOOL_REJECTED', + }); + return; } - }), - ); + res.status(200).json(settlement.result); + } catch (error) { + if (error instanceof BridgeStoreError) { + outcome.errorCode = error.code; + if (error.code === 'WORKSPACE_QUEUE_TIMEOUT') res.setHeader('Retry-After', '1'); + res.status(bridgeStoreStatus(error)).json({ + error: error.message, + code: error.code, + }); + return; + } + outcome.errorCode = 'INTERNAL_ERROR'; + throw error; + } finally { + outcome.dispatchPending = false; + outcome.flush(); + req.removeListener('aborted', abort); + res.removeListener('close', abortClosedResponse); + } + }); + router.post('/workspace-tools/execute', execute(false)); + router.post('/workspace-tools/requests', execute(true)); + + router.get('/workspace-tools/capabilities', asyncRoute(async (req, res) => { + if (!getPrincipalOrReject(req, res)) return; + res.json({ durableWorkspaceRequests: options.requests != null ? 1 : 0 }); + })); + + const requestStatus = (cancel: boolean): RequestHandler => asyncRoute(async (req, res) => { + const principal = getPrincipalOrReject(req, res); + if (!principal) return; + const status = cancel + ? await options.requests?.cancel(principal, req.params.requestId) + : await options.requests?.get(principal, req.params.requestId); + if (status == null) { res.status(404).json({ error: 'Workspace request not found' }); return; } + res.json(status); + }); + router.get('/workspace-tools/requests/:requestId', requestStatus(false)); + router.delete('/workspace-tools/requests/:requestId', requestStatus(true)); return router; } diff --git a/test-sandbox.sh b/test-sandbox.sh old mode 100755 new mode 100644 index 27c8a877..cf6db2cd --- a/test-sandbox.sh +++ b/test-sandbox.sh @@ -286,15 +286,28 @@ expect_socket_blocked("AF_RXRPC", getattr(socket, "AF_RXRPC", 33), socket.SOCK_D expect_socket_blocked("AF_ALG", getattr(socket, "AF_ALG", 38), socket.SOCK_SEQPACKET, 0) syscalls = { - "x86_64": {"clone": 56, "clone3": 435, "vmsplice": 278}, - "amd64": {"clone": 56, "clone3": 435, "vmsplice": 278}, - "aarch64": {"clone": 220, "clone3": 435, "vmsplice": 75}, - "arm64": {"clone": 220, "clone3": 435, "vmsplice": 75}, + "x86_64": {"clone": 56, "clone3": 435, "vmsplice": 278, "futex": 202}, + "amd64": {"clone": 56, "clone3": 435, "vmsplice": 278, "futex": 202}, + "aarch64": {"clone": 220, "clone3": 435, "vmsplice": 75, "futex": 98}, + "arm64": {"clone": 220, "clone3": 435, "vmsplice": 75, "futex": 98}, } arch = platform.machine().lower() if arch not in syscalls: raise SystemExit(f"unsupported arch for syscall smoke test: {arch}") +# PI operations must be denied before the kernel sees even invalid addresses. +for command in (6, 7, 8, 11, 12, 13): + for flags in (0, 128, 256, 384): + ctypes.set_errno(0) + rc = libc.syscall(syscalls[arch]["futex"], 0, command | flags, 0, 0, 0, 0) + if rc != -1 or ctypes.get_errno() != errno.EPERM: + raise SystemExit(f"PI futex operation was not denied: {command | flags}") +word = ctypes.c_int(0) +rc = libc.syscall(syscalls[arch]["futex"], ctypes.byref(word), 1 | 128, 1, 0, 0, 0) +if rc != 0: + raise SystemExit("ordinary FUTEX_WAKE_PRIVATE was blocked") +print("futex_pi=blocked:ordinary_wake=ok") + CLONE_NEWUSER = 0x10000000 CLONE_NEWNET = 0x40000000 @@ -359,7 +372,8 @@ PY && [[ "$stdout" == *"AF_ALG=errno:1"* ]] \ && [[ "$stdout" == *"clone_namespace=rc:-1:errno:1"* ]] \ && [[ "$stdout" == *"clone3=rc:-1:errno:38"* ]] \ - && [[ "$stdout" == *"vmsplice=rc:-1:errno:1"* ]]; then + && [[ "$stdout" == *"vmsplice=rc:-1:errno:1"* ]] \ + && [[ "$stdout" == *"futex_pi=blocked:ordinary_wake=ok"* ]]; then log_success "Kernel attack-surface syscalls blocked" return 0 else diff --git a/tests/compose-bridge-config.cjs b/tests/compose-bridge-config.cjs index de14b553..bafea2fe 100644 --- a/tests/compose-bridge-config.cjs +++ b/tests/compose-bridge-config.cjs @@ -15,6 +15,13 @@ function render(overrides) { CODEAPI_BRIDGE_WORKER_ID: '', CODEAPI_BRIDGE_TOKEN: '', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '', + CODEAPI_BRIDGE_LANE_GIT: '', + CODEAPI_BRIDGE_RECOVERY_SERVER_ID: '', + CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '', + CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '', + CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE: '', + CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE: '', + CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE: '', ...overrides, }, })); @@ -28,6 +35,13 @@ for (const overrides of [ CODEAPI_BRIDGE_DYNAMIC_WORKERS: 'false', CODEAPI_BRIDGE_WORKER_ID: 'test-worker', CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS: '4', + CODEAPI_BRIDGE_LANE_GIT: 'true', + CODEAPI_BRIDGE_RECOVERY_SERVER_ID: 'https://code.example.test', + CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS: '86400', + CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS: '90', + CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE: '8', + CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE: '16', + CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE: '400', }, ]) { const config = render(overrides); @@ -37,12 +51,23 @@ for (const overrides of [ assert.equal(env.CODEAPI_BRIDGE_AUTH_MODE, 'paired'); assert.equal(env.CODEAPI_BRIDGE_TOKEN, token); assert.equal(env.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS, overrides.CODEAPI_BRIDGE_MAX_WORKSPACE_LEASE_SLOTS ?? '1'); + assert.equal(env.CODEAPI_BRIDGE_LANE_GIT, overrides.CODEAPI_BRIDGE_LANE_GIT ?? 'false'); assert.equal(env.CODEAPI_BRIDGE_DYNAMIC_WORKERS, overrides.CODEAPI_BRIDGE_DYNAMIC_WORKERS ?? 'true'); assert.equal(env.CODEAPI_BRIDGE_WORKER_ID, overrides.CODEAPI_BRIDGE_WORKER_ID ?? ''); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_SERVER_ID, overrides.CODEAPI_BRIDGE_RECOVERY_SERVER_ID ?? ''); + assert.equal(env.CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS, overrides.CODEAPI_BRIDGE_ENROLLMENT_TTL_SECONDS ?? '0'); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS, overrides.CODEAPI_BRIDGE_RECOVERY_CHALLENGE_TTL_SECONDS ?? '60'); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_CHALLENGES_PER_MINUTE ?? '12'); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_ATTEMPTS_PER_MINUTE ?? '30'); + assert.equal(env.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE, overrides.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE ?? '240'); } for (const name of ['egress_gateway', 'sandbox-runner']) { assert.equal(config.services[name].environment.CODEAPI_BRIDGE_TOKEN, undefined); + assert.equal(config.services[name].environment.CODEAPI_BRIDGE_RECOVERY_SERVER_ID, undefined); + assert.equal(config.services[name].environment.CODEAPI_BRIDGE_RECOVERY_MAX_UNTRUSTED_PER_MINUTE, undefined); } + assert.match(JSON.stringify(config.services.redis.command), /--appendonly.*yes/); + assert.ok(config.services.redis.volumes.some(volume => volume.target === '/data' && volume.type === 'volume')); } assert.equal(render({}).services.api.environment.CODEAPI_BRIDGE_TOKEN, ''); -console.log('Compose bridge configuration passed (dynamic/fixed pairing, no default secret).'); +console.log('Compose bridge configuration passed (pairing, opt-in recovery, durable Redis).'); diff --git a/tests/kvm_guest_dns.sh b/tests/kvm_guest_dns.sh old mode 100755 new mode 100644 index 0e038a12..4ea99afb --- a/tests/kvm_guest_dns.sh +++ b/tests/kvm_guest_dns.sh @@ -80,31 +80,60 @@ printf '%s\n' "$SANDBOX_RESOLV_CONF" > "$TEST_RESOLVER_OUTPUT" STUB cat > "$TEST_DIR/bin/getent" <<'STUB' #!/usr/bin/env bash -printf '192.0.2.99 stale-address\n' +[[ -z "${TEST_RUNNER_ADDRESS-192.0.2.99}" ]] || printf '%s stale-address\n' "${TEST_RUNNER_ADDRESS-192.0.2.99}" STUB -chmod +x "$TEST_DIR/bin/launcher" "$TEST_DIR/bin/getent" +# The relay stub records its endpoints; readiness comes from the fake /proc/net. +cat > "$TEST_DIR/bin/socat" <<'STUB' +#!/usr/bin/env bash +printf '%s\n' "$*" >> "$TEST_SOCAT_CALLS" +STUB +chmod +x "$TEST_DIR/bin/launcher" "$TEST_DIR/bin/getent" "$TEST_DIR/bin/socat" +# 192.0.2.99:53 bound for UDP and listening for TCP, in /proc/net byte order. +mkdir "$TEST_DIR/proc-net-bound" "$TEST_DIR/proc-net-unbound" +printf ' sl local_address rem_address st\n 0: 630200C0:0035 00000000:0000 07 00000000:00000000\n' > "$TEST_DIR/proc-net-bound/udp" +printf ' sl local_address rem_address st\n 0: 630200C0:0035 00000000:0000 0A 00000000:00000000\n' > "$TEST_DIR/proc-net-bound/tcp" +printf ' sl local_address rem_address st\n' | tee "$TEST_DIR/proc-net-unbound/udp" > "$TEST_DIR/proc-net-unbound/tcp" # libkrun appends every guest environment entry to the kernel command line, # which linux-loader limits to single-line printable ASCII. Docker's generated # resolv.conf is multi-line with comments, so the entrypoint must forward only # the resolver directives, joined by the separator guest-dns.sh expands. run_entrypoint() { - local resolv_conf="$1" + local resolv_conf="$1" proc_net="${2:-$TEST_DIR/proc-net-bound}" sed -e "s|/usr/local/bin/launcher|$TEST_DIR/bin/launcher|g" \ -e "s|/etc/resolv.conf|$resolv_conf|g" \ + -e "s|/proc/net/|$proc_net/|g" \ + -e 's|RESOLVER_RELAY_READY_ATTEMPTS=50|RESOLVER_RELAY_READY_ATTEMPTS=3|' \ "$ROOT/launcher/entrypoint.sh" > "$TEST_DIR/entrypoint.sh" - rm -f "$TEST_DIR/forwarded" + rm -f "$TEST_DIR/forwarded" "$TEST_DIR/socat-calls" PATH="$TEST_DIR/bin:$PATH" \ EGRESS_GATEWAY_URL='https://egress_gateway:3190/base' \ FILE_SERVER_URL='http://[::1]:3000/base' \ SANDBOX_FORWARD_TARGET='tool_call_server:3033' \ LAUNCHER_FILTER_VSOCK_ENOTCONN=false \ TEST_RESOLVER_OUTPUT="$TEST_DIR/forwarded" \ + TEST_SOCAT_CALLS="$TEST_DIR/socat-calls" \ bash "$TEST_DIR/entrypoint.sh" } # Independent reference for the expected handoff: directives only, whitespace -# collapsed, joined by the separator. +# collapsed, joined by the separator. The first loopback nameserver becomes the +# relay address; other loopback nameservers are unreachable and dropped. encoded_reference() { - LC_ALL=C awk '/^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]]/ { sub(/\r$/, ""); $1 = $1; print }' "$1" | paste -sd '|' - + LC_ALL=C awk -v relay=192.0.2.99 '/^[[:space:]]*(nameserver|search|domain|options|sortlist)[[:space:]]/ { + sub(/\r$/, ""); $1 = $1 + address = tolower($2) + if ($1 == "nameserver" && (address ~ /^127\./ || address ~ /^[0:]*:0*1$/ || address ~ /^::ffff:127\./)) { if (loop == "") loop = $2; if ($2 != loop) next; $2 = relay } + print + }' "$1" | paste -sd '|' - +} +# The relays start in the background, so wait for both before comparing. Both +# bound idle children; the TCP listener also caps concurrent children. +assert_relayed() { + local udp_target="$1" tcp_target="$2" attempt + for ((attempt = 0; attempt < 50; attempt++)); do + [[ -e "$TEST_DIR/socat-calls" && "$(wc -l < "$TEST_DIR/socat-calls")" -ge 2 ]] && break + sleep 0.1 + done + [[ "$(LC_ALL=C sort "$TEST_DIR/socat-calls")" == "$(printf -- '-T 30 TCP4-LISTEN:53,bind=192.0.2.99,reuseaddr,fork,max-children=64 %s\n-T 5 UDP4-RECVFROM:53,bind=192.0.2.99,fork %s' "$tcp_target" "$udp_target")" ]] } cat > "$TEST_DIR/docker-resolv.conf" <<'RESOLV' # Generated by Docker Engine. @@ -119,24 +148,92 @@ options ndots:0 # Overrides: [] # Option ndots from: internal RESOLV +# The guest keeps loopback traffic local, so Docker's embedded resolver is +# relayed from the runner address and the guest gets that address instead. run_entrypoint "$TEST_DIR/docker-resolv.conf" -[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 127.0.0.11|options ndots:0' ]] +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|options ndots:0' ]] [[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/docker-resolv.conf")" ]] +assert_relayed UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 [[ "$(wc -l < "$TEST_DIR/forwarded")" == 1 ]] if LC_ALL=C grep -q '[^ -~]' "$TEST_DIR/forwarded"; then echo 'forwarded resolver contains bytes the kernel command line rejects' >&2; exit 1 fi -# The guest restores exactly the directives Docker supplied. +# The guest restores the Docker directives with the relayed nameserver. rm -rf "$TEST_DIR/guest/run/codeapi-resolver" SANDBOX_RESOLV_CONF="$(cat "$TEST_DIR/forwarded")" configure_guest_dns "$TEST_DIR/guest" -printf 'nameserver 127.0.0.11\noptions ndots:0\n' > "$TEST_DIR/expected" +printf 'nameserver 192.0.2.99\noptions ndots:0\n' > "$TEST_DIR/expected" cmp "$TEST_DIR/expected" "$TEST_DIR/guest/etc/resolv.conf" +# Only one loopback resolver can be relayed; other loopback entries would time +# out in the guest, while routable nameservers keep their order. +printf 'nameserver 127.0.0.53\r\nnameserver 10.0.0.2\nnameserver\t127.0.0.11\nsearch example.internal\n' > "$TEST_DIR/multi-loopback-resolv.conf" +run_entrypoint "$TEST_DIR/multi-loopback-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|nameserver 10.0.0.2|search example.internal' ]] +[[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/multi-loopback-resolv.conf")" ]] +assert_relayed UDP4-SENDTO:127.0.0.53:53 TCP4:127.0.0.53:53 + +# An IPv6 loopback resolver is just as unreachable from the guest; the IPv4 +# relay forwards to it over IPv6. +printf 'nameserver ::1\noptions ndots:0\n' > "$TEST_DIR/ipv6-loopback-resolv.conf" +run_entrypoint "$TEST_DIR/ipv6-loopback-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|options ndots:0' ]] +[[ "$(cat "$TEST_DIR/forwarded")" == "$(encoded_reference "$TEST_DIR/ipv6-loopback-resolv.conf")" ]] +assert_relayed 'UDP6-SENDTO:[::1]:53' 'TCP6:[::1]:53' +printf 'nameserver 127.0.0.11\nnameserver ::1\n' > "$TEST_DIR/mixed-loopback-resolv.conf" +run_entrypoint "$TEST_DIR/mixed-loopback-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99' ]] +assert_relayed UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +# Every spelling glibc accepts for a loopback resolver is relayed to its +# canonical address: compressed or zero-padded ::1, IPv4-mapped 127/8 in any +# IPv6 form, and inet_aton(3) IPv4 forms. +while read -r spelling udp_target tcp_target; do + printf 'nameserver %s\nnameserver 10.0.0.2\n' "$spelling" > "$TEST_DIR/spelled-loopback-resolv.conf" + run_entrypoint "$TEST_DIR/spelled-loopback-resolv.conf" + [[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|nameserver 10.0.0.2' ]] + assert_relayed "$udp_target" "$tcp_target" +done <<'SPELLINGS' +0:0:0:0:0:0:0:1 UDP6-SENDTO:[::1]:53 TCP6:[::1]:53 +0000::0001 UDP6-SENDTO:[::1]:53 TCP6:[::1]:53 +::1%lo UDP6-SENDTO:[::1]:53 TCP6:[::1]:53 +::FFFF:127.0.0.11 UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +0:0:0:0:0:ffff:127.0.0.11 UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +::ffff:7f00:b UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +127.1 UDP4-SENDTO:127.0.0.1:53 TCP4:127.0.0.1:53 +2130706443 UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +0x7f.0.0.0xb UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +0177.0.0.013 UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +SPELLINGS +# Spellings of the same loopback resolver collapse into one relayed entry. +printf 'nameserver 127.0.0.11\nnameserver ::ffff:7f00:b\nnameserver 127.0.0.53\n' > "$TEST_DIR/same-loopback-resolv.conf" +run_entrypoint "$TEST_DIR/same-loopback-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 192.0.2.99|nameserver 192.0.2.99' ]] +assert_relayed UDP4-SENDTO:127.0.0.11:53 TCP4:127.0.0.11:53 +# Routable resolvers and addresses glibc rejects are forwarded unchanged, without +# a relay: 0127 is octal 87, and ::7f00:1 is not IPv4-mapped. +printf 'nameserver 2001:db8::1\nnameserver fd00::11\nnameserver ::ffff:10.0.0.2\nnameserver ::7f00:1\nnameserver 0127.0.0.1\nnameserver 127.0.0.256\nnameserver 1:2:3:4:5:6:7:8:\nnameserver 127..1\n' > "$TEST_DIR/routable-resolv.conf" +run_entrypoint "$TEST_DIR/routable-resolv.conf" +[[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 2001:db8::1|nameserver fd00::11|nameserver ::ffff:10.0.0.2|nameserver ::7f00:1|nameserver 0127.0.0.1|nameserver 127.0.0.256|nameserver 1:2:3:4:5:6:7:8:|nameserver 127..1' ]] +[[ ! -e "$TEST_DIR/socat-calls" ]] + +# A relay that never listens, or a runner without a routable address, fails +# startup instead of booting a guest that cannot resolve service names. +if run_entrypoint "$TEST_DIR/docker-resolv.conf" "$TEST_DIR/proc-net-unbound" 2> "$TEST_DIR/entrypoint-error"; then + echo 'started launcher without a listening resolver relay' >&2; exit 1 +fi +grep -q 'guest DNS relay could not listen on 192.0.2.99:53' "$TEST_DIR/entrypoint-error" +[[ ! -e "$TEST_DIR/forwarded" ]] +if TEST_RUNNER_ADDRESS='' run_entrypoint "$TEST_DIR/docker-resolv.conf" 2> "$TEST_DIR/entrypoint-error"; then + echo 'started launcher without a resolver relay address' >&2; exit 1 +fi +grep -q 'no non-loopback IPv4 address to relay loopback resolver 127.0.0.11' "$TEST_DIR/entrypoint-error" +[[ ! -e "$TEST_DIR/forwarded" && ! -e "$TEST_DIR/socat-calls" ]] + # Kubernetes resolvers: tabs, CRLF, trailing spaces, and unknown keywords are # normalized away; search domains and options survive intact. printf 'nameserver\t10.96.0.10 \r\nsearch tenant.svc.cluster.local svc.cluster.local cluster.local\n; resolver comment\nlookup file bind\noptions ndots:5\n' > "$TEST_DIR/k8s-resolv.conf" # leak-check:allow run_entrypoint "$TEST_DIR/k8s-resolv.conf" [[ "$(cat "$TEST_DIR/forwarded")" == 'nameserver 10.96.0.10|search tenant.svc.cluster.local svc.cluster.local cluster.local|options ndots:5' ]] # leak-check:allow +[[ ! -e "$TEST_DIR/socat-calls" ]] # The runner's own resolver must round-trip through the same reference. run_entrypoint /etc/resolv.conf diff --git a/tests/nsjail_unix_gc.py b/tests/nsjail_unix_gc.py new file mode 100644 index 00000000..08c7cf8d --- /dev/null +++ b/tests/nsjail_unix_gc.py @@ -0,0 +1,152 @@ +#!/usr/bin/env python3 +"""Exercise the rendered policy with the pinned Kafel on Linux. + +Usage: python3 tests/nsjail_unix_gc.py POLICY /nsjail/kafel/libkafel.so +Run in the nsjail-builder image. No exploit or cyclic descriptor graph is used. +The filter stays installed until this test process and its children exit. +""" +import array +import ctypes as c +import errno +import multiprocessing as mp +import os +import socket +import struct +import sys + + +class SockFilter(c.Structure): + _fields_ = [("code", c.c_ushort), ("jt", c.c_ubyte), + ("jf", c.c_ubyte), ("k", c.c_uint)] + + +class SockFprog(c.Structure): + _fields_ = [("len", c.c_ushort), ("filter", c.POINTER(SockFilter))] + + +class Iovec(c.Structure): + _fields_ = [("base", c.c_void_p), ("length", c.c_size_t)] + + +class Msghdr(c.Structure): + _fields_ = [("name", c.c_void_p), ("namelen", c.c_uint), + ("iov", c.POINTER(Iovec)), ("iovlen", c.c_size_t), + ("control", c.c_void_p), ("controllen", c.c_size_t), + ("flags", c.c_int)] + + +class Mmsghdr(c.Structure): + _fields_ = [("header", Msghdr), ("length", c.c_uint)] + + +def expect_eperm(call): + try: + call() + except OSError as error: + assert error.errno == errno.EPERM, error + else: + raise AssertionError("descriptor-passing send was allowed") + + +def square(value): + return value * value + + +def queue_child(queue): + queue.put("queue works") + + +def test_sends(libc, pair): + left, right = pair + # A regular-file descriptor avoids creating any socket reference cycle, + # even if a regressed policy allows the send and the assertion fails. + descriptor = os.open("/dev/null", os.O_RDONLY) + rights = [(socket.SOL_SOCKET, socket.SCM_RIGHTS, + array.array("i", [descriptor]))] + expect_eperm(lambda: left.sendmsg([b"x"], rights)) + expect_eperm(lambda: left.sendmsg([b"x"])) + # Valid batch: seccomp must deny even messages without ancillary data. + data = c.create_string_buffer(b"x") + iov = Iovec(c.cast(data, c.c_void_p), 1) + message = Mmsghdr() + message.header.iov = c.pointer(iov) + message.header.iovlen = 1 + c.set_errno(0) + assert libc.sendmmsg(left.fileno(), c.byref(message), 1, 0) == -1 + assert c.get_errno() == errno.EPERM + control = c.create_string_buffer(socket.CMSG_SPACE(c.sizeof(c.c_int))) + header = struct.pack("@Nii", socket.CMSG_LEN(c.sizeof(c.c_int)), + socket.SOL_SOCKET, socket.SCM_RIGHTS) + control[:len(header)] = header + offset = socket.CMSG_LEN(0) + control[offset:offset + c.sizeof(c.c_int)] = struct.pack("@i", descriptor) + message.header.control = c.cast(control, c.c_void_p) + message.header.controllen = socket.CMSG_SPACE(c.sizeof(c.c_int)) + assert libc.sendmmsg(left.fileno(), c.byref(message), 1, 0) == -1 + assert c.get_errno() == errno.EPERM + os.close(descriptor) + left.sendall(b"ordinary IPC") + assert right.recv(12) == b"ordinary IPC" + left.close() + right.close() + + +def main(): + # Open before installation to verify the rule also covers inherited FDs. + inherited = socket.socketpair() + libc = c.CDLL(None, use_errno=True) + libc.sendmmsg.argtypes = [c.c_int, c.POINTER(Mmsghdr), c.c_uint, c.c_int] + kafel = c.CDLL(sys.argv[2]) + kafel.kafel_compile_string.argtypes = [c.c_char_p, c.POINTER(SockFprog)] + program = SockFprog() + with open(sys.argv[1], "rb") as source: + assert kafel.kafel_compile_string(source.read(), c.byref(program)) == 0 + # PR_SET_NO_NEW_PRIVS, followed by PR_SET_SECCOMP / SECCOMP_MODE_FILTER. + assert libc.prctl(38, 1, 0, 0, 0) == 0 + assert libc.prctl(22, 2, c.byref(program), 0, 0) == 0, c.get_errno() + # These numbers are shared on x86_64 and arm64. No ring may bypass sends. + for syscall in (425, 426, 427): + assert libc.syscall(syscall, 0, 0, 0, 0, 0, 0) == -1 + assert c.get_errno() == errno.EPERM + test_sends(libc, inherited) + expect_eperm(lambda: socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM)) + for domain in (socket.AF_UNIX, socket.AF_INET, socket.AF_INET6, socket.AF_NETLINK): + expect_eperm(lambda: socket.socket(domain, socket.SOCK_STREAM)) + expect_eperm(lambda: socket.socketpair(socket.AF_UNIX, socket.SOCK_DGRAM)) + expect_eperm(lambda: socket.socketpair(socket.AF_UNIX, socket.SOCK_STREAM, 6)) + import asyncio + from python_pipe_runtime import check_asyncio, duplex_child + asyncio.run(check_asyncio()) + for method in ("fork", "spawn"): + context = mp.get_context(method) + left, right = context.Pipe() + process = context.Process(target=duplex_child, args=(right,)) + process.start() + right.close() + left.send_bytes(b'x' * 100_000) + assert left.poll(10) + assert left.recv_bytes() == b'x' * 100_000 + process.join(10) + assert process.exitcode == 0 + try: + left.recv_bytes() + except EOFError: + pass + else: + raise AssertionError('duplex peer EOF missing') + left.close() + queue = context.Queue() + process = context.Process(target=queue_child, args=(queue,)) + process.start() + assert queue.get(timeout=10) == "queue works" + process.join(10) + assert process.exitcode == 0 + queue.close() + queue.join_thread() + with context.Pool(2) as pool: + assert pool.map(square, [2, 3, 4]) == [4, 9, 16] + print("PASS: descriptor sends denied; socket creation denied; socketpairs denied; pipe-based asyncio, duplex IPC, queues and pools work") + + +if __name__ == "__main__": + main() diff --git a/tests/package_init_runtime_rollout.sh b/tests/package_init_runtime_rollout.sh new file mode 100644 index 00000000..3bb8cf6c --- /dev/null +++ b/tests/package_init_runtime_rollout.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d)" +trap 'rm -r "$TMP_DIR"' EXIT +mkdir "$TMP_DIR/chart" +cp "$ROOT/helm/codeapi/values.yaml" "$TMP_DIR/chart/values.yaml" +cp -R "$ROOT/helm/codeapi/templates" "$TMP_DIR/chart/templates" +awk '/^dependencies:/{exit} {print}' "$ROOT/helm/codeapi/Chart.yaml" > "$TMP_DIR/chart/Chart.yaml" +render() { + helm template runtime-rollout "$TMP_DIR/chart" \ + --show-only templates/package-init-job.yaml \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test \ + --set workerSandbox.enabled=true \ + --set workerSandbox.packages.source=pvc "$@" +} +# Default mutable image must refresh on every pre-install/pre-upgrade hook. +render | grep -q 'imagePullPolicy: Always' +# Reused historical values must not bypass the runtime migration. +render --set workerSandbox.packages.initJob.image.pullPolicy=IfNotPresent \ + | grep -q 'imagePullPolicy: Always' +# Immutable release images retain their explicitly configured cache policy. +render --set workerSandbox.packages.initJob.image.tag=release-123 \ + --set workerSandbox.packages.initJob.image.pullPolicy=IfNotPresent \ + | grep -q 'imagePullPolicy: IfNotPresent' +# Local kind/minikube builds are deliberately loaded without a registry. +render -f "$ROOT/helm/codeapi/values-local.yaml" | grep -q 'imagePullPolicy: Never' +render --set workerSandbox.packages.initJob.image.pullPolicy=Never \ + | grep -q 'imagePullPolicy: Never' +echo 'PASS: package-init refreshes cached latest and preserves offline images' diff --git a/tests/pipe_relay_backpressure.py b/tests/pipe_relay_backpressure.py new file mode 100644 index 00000000..fb18803c --- /dev/null +++ b/tests/pipe_relay_backpressure.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""A closed request writer must not spin the trusted relay under backpressure.""" +import os +import signal +import subprocess +import sys +import tempfile +import time + +request_reader, request_writer = os.pipe() +response_reader, response_writer = os.pipe() +os.set_blocking(request_writer, False) +try: + while True: + os.write(request_writer, b'x' * 4096) +except BlockingIOError: + pass # The upstream deliberately does not drain requests. + +with tempfile.TemporaryDirectory() as directory: + marker = os.path.join(directory, 'ready') + job = os.path.join(directory, 'job.py') + with open(job, 'w') as output: + output.write(f"import os,time; os.write(3,b'x'*4096); os.close(3); open({marker!r},'w').close(); time.sleep(30)") + bootstrap = (f"import os; os.dup2({request_writer},3); os.dup2({response_reader},4); " + f"os.execv('/usr/local/bin/tool-call-pipe-bridge'," + f"['tool-call-pipe-bridge','--job-pipes',{sys.executable!r},{job!r}])") + relay = subprocess.Popen([sys.executable, '-c', bootstrap], + pass_fds=[request_writer, response_reader]) + try: + deadline = time.monotonic() + 5 + while not os.path.exists(marker) and time.monotonic() < deadline: + assert relay.poll() is None + time.sleep(0.01) + assert os.path.exists(marker), 'job did not close its request writer' + def cpu_seconds(): + with open(f'/proc/{relay.pid}/stat') as source: + fields = source.read().rsplit(')', 1)[1].split() + return (int(fields[11]) + int(fields[12])) / os.sysconf('SC_CLK_TCK') + before = cpu_seconds() + time.sleep(0.5) + used = cpu_seconds() - before + assert used < 0.1, f'relay spun on pipe HUP under backpressure: {used:.3f}s CPU' + assert relay.poll() is None + # Closing the broker response writer still terminates the invocation. + os.close(response_writer) + response_writer = -1 + assert relay.wait(timeout=2) == 137 + print('PASS: blocked relay stays idle after request EOF and terminates on broker EOF') + finally: + if relay.poll() is None: + relay.send_signal(signal.SIGKILL) + relay.wait(timeout=2) +for fd in [request_reader, request_writer, response_reader, response_writer]: + if fd >= 0: + os.close(fd) diff --git a/tests/pipe_runtime_image.cjs b/tests/pipe_runtime_image.cjs new file mode 100644 index 00000000..8b2d1008 --- /dev/null +++ b/tests/pipe_runtime_image.cjs @@ -0,0 +1,61 @@ +#!/usr/bin/env node +// Packaging regression: run in each guest rootfs. Uses the image's own broker, +// Node, bridge, NsJail and guard, with namespaces disabled for this smoke test. +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const http = require('node:http'); +const { spawn } = require('node:child_process'); +const { once } = require('node:events'); + +if (process.argv[2] === '--client') { + assert(fs.fstatSync(3).isFIFO()); + assert(fs.fstatSync(4).isFIFO()); + const frame = Buffer.from(JSON.stringify({ id: 1, headers: { + 'x-execution-id': 'image-smoke', 'x-callback-token': 'image-token', 'x-tool-call-id': 'image-call', + }, body: '{"tool_name":"echo","input":{}}' })); + const prefix = Buffer.alloc(4); prefix.writeUInt32BE(frame.length); + const bytes = Buffer.concat([prefix, frame]); + for (let offset = 0; offset < bytes.length;) offset += fs.writeSync(3, bytes, offset, bytes.length - offset); + function readExact(size) { + const bytes = Buffer.alloc(size); + for (let offset = 0; offset < size;) { + const count = fs.readSync(4, bytes, offset, size - offset); + assert(count > 0, 'broker response ended prematurely'); offset += count; + } + return bytes; + } + const reply = JSON.parse(readExact(readExact(4).readUInt32BE()).toString()); + assert.deepEqual(reply, { id: 1, status: 200, body: 'packaged runtime works' }); + console.log('PASS: image Node/broker/bridge/NsJail/guard pipe round trip'); +} else { + (async () => { + const upstream = http.createServer(async (req, res) => { + assert.equal(req.url, '/tool-call'); + assert.equal(req.headers['x-callback-token'], 'image-token'); + for await (const bytes of req) {} + res.end('packaged runtime works'); + }); + upstream.listen(0, '127.0.0.1'); await once(upstream, 'listening'); + let child; + let timer; + try { + child = spawn('/usr/local/bin/tool-call-pipe-bridge', [ + '--broker', '/usr/local/bin/node', '/sandbox_api/.build/tool-call-pipe-broker.cjs', + '/usr/sbin/nsjail', '--mode', 'e', '--disable_clone_newnet', '--disable_clone_newuser', + '--disable_clone_newns', '--disable_clone_newpid', '--disable_clone_newipc', + '--disable_clone_newuts', '--disable_clone_newcgroup', '--disable_proc', '--disable_rlimits', + '--pass_fd', '3', '--pass_fd', '4', '--', '/usr/local/bin/spec-guard', + '--tool-call-pipes', '/usr/local/bin/node', '/tests/pipe_runtime_image.cjs', '--client', + ], { stdio: ['ignore', 'inherit', 'inherit'], env: { + ...process.env, SANDBOX_FORWARD_TARGET: `http://127.0.0.1:${upstream.address().port}`, + }}); + timer = setTimeout(() => child.kill('SIGKILL'), 5000); + const [code, signal] = await once(child, 'exit'); + assert.equal(code, 0, `image pipe runtime failed (${signal})`); + } finally { + clearTimeout(timer); + if (child && child.exitCode === null && child.signalCode === null) child.kill('SIGKILL'); + upstream.closeAllConnections(); upstream.close(); + } + })().catch(error => { console.error(error); process.exitCode = 1; }); +} diff --git a/tests/python_pipe_runtime.py b/tests/python_pipe_runtime.py new file mode 100644 index 00000000..52a48095 --- /dev/null +++ b/tests/python_pipe_runtime.py @@ -0,0 +1,47 @@ +"""Runtime transition checks also run under the installed seccomp filter.""" +import asyncio +import os +import signal +import stat +import sys + + +def duplex_child(connection): + try: + assert stat.S_ISFIFO(os.fstat(connection.fileno()).st_mode) + connection.send_bytes(connection.recv_bytes()) + finally: + connection.close() + + +async def check_asyncio(): + loop = asyncio.get_running_loop() + assert stat.S_ISFIFO(os.fstat(loop._ssock.fileno()).st_mode) + assert stat.S_ISFIFO(os.fstat(loop._csock.fileno()).st_mode) + assert not os.get_inheritable(loop._ssock.fileno()) + assert not os.get_inheritable(loop._csock.fileno()) + # Executor completion crosses from a worker thread into the event loop. + values = await asyncio.wait_for(asyncio.gather(*[ + asyncio.to_thread(lambda n=n: n * n) for n in range(64) + ]), 10) + assert values == [n * n for n in range(64)] + awakened = asyncio.Event() + loop.add_signal_handler(signal.SIGUSR1, awakened.set) + try: + os.kill(os.getpid(), signal.SIGUSR1) + await asyncio.wait_for(awakened.wait(), 2) + finally: + loop.remove_signal_handler(signal.SIGUSR1) + process = await asyncio.create_subprocess_exec( + sys.executable, '-c', 'print("child works")', + stdout=asyncio.subprocess.PIPE) + output, _ = await asyncio.wait_for(process.communicate(), 10) + assert output == b'child works\n' and process.returncode == 0 + pending = asyncio.create_task(asyncio.sleep(30)) + pending.cancel() + try: + await pending + except asyncio.CancelledError: + pass + else: + raise AssertionError('cancellation lost') diff --git a/tests/sandbox_runner_metrics.sh b/tests/sandbox_runner_metrics.sh new file mode 100644 index 00000000..df7c9ff3 --- /dev/null +++ b/tests/sandbox_runner_metrics.sh @@ -0,0 +1,147 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +TMP_DIR="$(mktemp -d)" +trap 'rm -rf "$TMP_DIR"' EXIT +command -v helm >/dev/null + +# Like block_root_package_delivery.sh, render only this chart's templates so +# tests do not download Redis or MinIO dependencies or contact a cluster. +mkdir "$TMP_DIR/chart" +cp "$ROOT/helm/codeapi/values.yaml" "$TMP_DIR/chart/values.yaml" +cp -R "$ROOT/helm/codeapi/templates" "$TMP_DIR/chart/templates" +awk '/^dependencies:/{exit} {print}' "$ROOT/helm/codeapi/Chart.yaml" > "$TMP_DIR/chart/Chart.yaml" + +render() { + helm template telemetry "$TMP_DIR/chart" \ + --set executionManifest.privateKey=test \ + --set executionManifest.publicKey=test "$@" +} + +resource() { + local file="$1" kind="$2" name="$3" + awk -v kind="$kind" -v name="$name" ' + function emit() { if (foundKind && foundName) printf "%s", document } + /^---$/ { emit(); document=""; foundKind=0; foundName=0; next } + { document=document $0 "\n" } + $0 == "kind: " kind { foundKind=1 } + $0 == " name: " name { foundName=1 } + END { emit() } + ' "$file" +} + +contains() { + if ! grep -Fq -- "$2" "$1"; then + echo "Missing '$2' in $(basename "$1")" >&2 + exit 1 + fi +} + +check_monitor() { + local rendered="$1" component="$2" port="$3" + local monitor="$TMP_DIR/$component-monitor.yaml" + resource "$rendered" PodMonitor "telemetry-codeapi-$component" > "$monitor" + contains "$monitor" ' - path: /metrics' + contains "$monitor" " port: $port" + contains "$monitor" ' interval: 17s' + contains "$monitor" ' scrapeTimeout: 7s' + # Exact selectors must match the corresponding Deployment's pod labels, + # including this release, and not accidentally select the service worker. + sed -n '/^ selector:/,$p' "$monitor" > "$TMP_DIR/selector.yaml" + contains "$TMP_DIR/selector.yaml" ' app.kubernetes.io/name: codeapi' + contains "$TMP_DIR/selector.yaml" ' app.kubernetes.io/instance: telemetry' + contains "$TMP_DIR/selector.yaml" " app.kubernetes.io/component: $component" + resource "$rendered" Deployment "telemetry-codeapi-$component" > "$TMP_DIR/deployment.yaml" + contains "$TMP_DIR/deployment.yaml" " app.kubernetes.io/component: $component" + contains "$TMP_DIR/deployment.yaml" " - name: $port" +} + +render > "$TMP_DIR/default.yaml" +if grep -q '^kind: PodMonitor$' "$TMP_DIR/default.yaml"; then + echo 'metrics.enabled=false must not create PodMonitors' >&2 + exit 1 +fi + +render --set metrics.enabled=true --set metrics.interval=17s --set metrics.scrapeTimeout=7s \ + --set workerSandbox.sandbox.port=2345 > "$TMP_DIR/enabled.yaml" +check_monitor "$TMP_DIR/enabled.yaml" sandbox-runner sandbox +check_monitor "$TMP_DIR/enabled.yaml" service-worker health +contains "$TMP_DIR/deployment.yaml" ' app.kubernetes.io/instance: telemetry' +resource "$TMP_DIR/enabled.yaml" Deployment telemetry-codeapi-sandbox-runner > "$TMP_DIR/runner.yaml" +contains "$TMP_DIR/runner.yaml" ' containerPort: 2345' + +# Enabling metrics must not silently weaken sandbox isolation. +resource "$TMP_DIR/default.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/default-policy.yaml" +resource "$TMP_DIR/enabled.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/enabled-policy.yaml" +# Use the same configured runner port when comparing policies. +sed 's/port: 2345/port: 2000/' "$TMP_DIR/enabled-policy.yaml" > "$TMP_DIR/normalized-policy.yaml" +cmp "$TMP_DIR/default-policy.yaml" "$TMP_DIR/normalized-policy.yaml" + +cat > "$TMP_DIR/scraper.yaml" <<'YAML' +metrics: + sandboxRunner: + ingressFrom: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: metrics-test + podSelector: + matchLabels: + app.kubernetes.io/name: prometheus-test +YAML +render --set metrics.enabled=true --set workerSandbox.sandbox.port=2345 \ + -f "$TMP_DIR/scraper.yaml" > "$TMP_DIR/scraper-render.yaml" +resource "$TMP_DIR/scraper-render.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/scraper-policy.yaml" +contains "$TMP_DIR/scraper-policy.yaml" 'kubernetes.io/metadata.name: metrics-test' +contains "$TMP_DIR/scraper-policy.yaml" 'app.kubernetes.io/name: prometheus-test' +contains "$TMP_DIR/scraper-policy.yaml" 'app.kubernetes.io/component: service-worker' +# The namespace and pod selectors must be ANDed in ONE peer, not two ORed peers. +contains "$TMP_DIR/scraper-policy.yaml" ' - namespaceSelector:' +contains "$TMP_DIR/scraper-policy.yaml" ' podSelector:' +[[ "$(grep -Fc ' port: 2345' "$TMP_DIR/scraper-policy.yaml")" == 2 ]] +# Scraping changes only ingress, never the sandbox egress boundary. +sed -n '/^ egress:/,$p' "$TMP_DIR/scraper-policy.yaml" > "$TMP_DIR/scraper-egress.yaml" +sed -n '/^ egress:/,$p' "$TMP_DIR/default-policy.yaml" > "$TMP_DIR/default-egress.yaml" +cmp "$TMP_DIR/default-egress.yaml" "$TMP_DIR/scraper-egress.yaml" + +render -f "$TMP_DIR/scraper.yaml" > "$TMP_DIR/disabled-scraper.yaml" +resource "$TMP_DIR/disabled-scraper.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/disabled-scraper-policy.yaml" +cmp "$TMP_DIR/default-policy.yaml" "$TMP_DIR/disabled-scraper-policy.yaml" + +render --set metrics.enabled=true --set workerSandbox.enabled=false \ + -f "$TMP_DIR/scraper.yaml" > "$TMP_DIR/no-workers.yaml" +[[ -z "$(resource "$TMP_DIR/no-workers.yaml" PodMonitor telemetry-codeapi-sandbox-runner)" ]] +[[ -z "$(resource "$TMP_DIR/no-workers.yaml" PodMonitor telemetry-codeapi-service-worker)" ]] +resource "$TMP_DIR/no-workers.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/no-workers-policy.yaml" +if grep -q prometheus-test "$TMP_DIR/no-workers-policy.yaml"; then + echo 'disabled workerSandbox must not enable scrape ingress' >&2 + exit 1 +fi + +# Older values may omit the new subtree; this must remain worker-only ingress. +render --set metrics.enabled=true --set metrics.sandboxRunner=null > "$TMP_DIR/old-values.yaml" +resource "$TMP_DIR/old-values.yaml" NetworkPolicy telemetry-codeapi-sandbox-runner > "$TMP_DIR/old-values-policy.yaml" +cmp "$TMP_DIR/default-policy.yaml" "$TMP_DIR/old-values-policy.yaml" + +render --set metrics.enabled=true --set api.enabled=false > "$TMP_DIR/no-api.yaml" +[[ -n "$(resource "$TMP_DIR/no-api.yaml" PodMonitor telemetry-codeapi-sandbox-runner)" ]] +[[ -z "$(resource "$TMP_DIR/no-api.yaml" PodMonitor telemetry-codeapi-api)" ]] + +render --set metrics.enabled=true --set metrics.interval= --set metrics.scrapeTimeout= > "$TMP_DIR/no-timing.yaml" +resource "$TMP_DIR/no-timing.yaml" PodMonitor telemetry-codeapi-sandbox-runner > "$TMP_DIR/no-timing-monitor.yaml" +if grep -Eq 'interval:|scrapeTimeout:' "$TMP_DIR/no-timing-monitor.yaml"; then + echo 'empty scrape timing overrides must defer to Prometheus defaults' >&2 + exit 1 +fi + +render --set metrics.enabled=true --set networkPolicy.enabled=false \ + -f "$TMP_DIR/scraper.yaml" > "$TMP_DIR/no-policy.yaml" +[[ -n "$(resource "$TMP_DIR/no-policy.yaml" PodMonitor telemetry-codeapi-sandbox-runner)" ]] +if grep -q '^kind: NetworkPolicy$' "$TMP_DIR/no-policy.yaml"; then + echo 'networkPolicy.enabled=false must not create NetworkPolicies' >&2 + exit 1 +fi + +helm lint "$TMP_DIR/chart" --set executionManifest.privateKey=test --set executionManifest.publicKey=test \ + --set metrics.enabled=true -f "$TMP_DIR/scraper.yaml" +echo 'Sandbox runner metrics chart regressions passed' diff --git a/tests/specguard_tool_call_pipes.py b/tests/specguard_tool_call_pipes.py new file mode 100644 index 00000000..549b1fa9 --- /dev/null +++ b/tests/specguard_tool_call_pipes.py @@ -0,0 +1,45 @@ +#!/usr/bin/env python3 +"""Verify the native guard preserves only validated pipe capabilities.""" +import fcntl +import os + + +def run_guard(pipes, preserve, reversed_ends=False): + request = os.pipe() + response = os.pipe() + ordinary = os.open('/dev/null', os.O_RDWR) + original = [request[0 if reversed_ends else 1] if pipes else ordinary, + response[1 if reversed_ends else 0] if pipes else ordinary, ordinary] + sources = [fcntl.fcntl(fd, fcntl.F_DUPFD_CLOEXEC, 10) for fd in original] + child = os.fork() + if child == 0: + for source, destination in zip(sources, [3, 4, 9]): + os.dup2(source, destination, inheritable=True) + code = ''' +import os, stat +for fd in ([9] if PRESERVE else [3,4,9]): + try: + os.fstat(fd) + except OSError: + pass + else: + raise AssertionError("unrelated FD survived guard: " + str(fd)) +if PRESERVE: + assert stat.S_ISFIFO(os.fstat(3).st_mode) + assert stat.S_ISFIFO(os.fstat(4).st_mode) +'''.replace('PRESERVE', repr(preserve)) + args = [os.environ.get('SPEC_GUARD_BINARY', '/usr/local/bin/spec-guard')] + if preserve: + args.append('--tool-call-pipes') + os.execv(args[0], args + ['/usr/bin/python3', '-c', code]) + for fd in sources + list(request) + list(response) + [ordinary]: + os.close(fd) + _, status = os.waitpid(child, 0) + return os.waitstatus_to_exitcode(status) + + +assert run_guard(True, True) == 0 +assert run_guard(True, False) == 0 +assert run_guard(False, True) == 1 +assert run_guard(True, True, reversed_ends=True) == 1 +print('PASS: guard preserves only explicit FIFO endpoints and rejects regular files') diff --git a/tests/supervisor_dns.py b/tests/supervisor_dns.py new file mode 100644 index 00000000..f2a9a291 --- /dev/null +++ b/tests/supervisor_dns.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Run with Docker --dns 127.0.0.1; no external DNS/network dependency.""" +import json +import os +import socket +import struct +import subprocess +import threading + +server = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +server.bind(('127.0.0.1', 53)) +queries = [] +def serve(): + while True: + packet, peer = server.recvfrom(4096) + offset = 12 + labels = [] + while packet[offset]: + length = packet[offset] + labels.append(packet[offset + 1:offset + length + 1]) + offset += length + 1 + offset += 1 + kind, cls = struct.unpack('!HH', packet[offset:offset + 4]) + offset += 4 + queries.append((b'.'.join(labels), kind)) + address = socket.inet_pton(socket.AF_INET6 if kind == 28 else socket.AF_INET, + '::1' if kind == 28 else '127.0.0.1') + reply = packet[:2] + struct.pack('!HHHHH', 0x8180, 1, 1, 0, 0) + packet[12:offset] + reply += b'\xc0\x0c' + struct.pack('!HHIH', kind, cls, 60, len(address)) + address + server.sendto(reply, peer) +threading.Thread(target=serve, daemon=True).start() +program = """ +const assert=require('node:assert/strict'); +assert(process.env.RES_OPTIONS.includes('timeout:1')); +assert(process.env.RES_OPTIONS.includes('single-request')); +require('node:dns').lookup('supervisor-test.invalid.',{all:true},(error,addresses)=>{ + if(error) throw error; + assert(addresses.length>0); + assert(addresses.every(a=>a.address==='127.0.0.1'||a.address==='::1')); + console.log('PASS: hostname resolution under supervisor filter'); +}); +""" +for runtime in ['/usr/local/bin/node', '/usr/local/bin/bun']: + result = subprocess.run(['/usr/local/bin/sandbox-supervisor-policy', runtime, '-e', program], + env={**os.environ, 'RES_OPTIONS': 'timeout:1 attempts:1'}, + capture_output=True, text=True, timeout=5) + assert result.returncode == 0, (runtime, result.stdout, result.stderr) + print(os.path.basename(runtime) + ': ' + result.stdout.strip()) +assert queries and all(name == b'supervisor-test.invalid' for name, _ in queries), queries diff --git a/tests/supervisor_hosted_network.ts b/tests/supervisor_hosted_network.ts new file mode 100644 index 00000000..e581576b --- /dev/null +++ b/tests/supervisor_hosted_network.ts @@ -0,0 +1,42 @@ +// Run as root with NET_ADMIN in the packaged guest, under supervisor-policy. +import { installHostedAppNetworkGuard } from '/work/api/src/hosted-app'; +import { spawnPipeProcess } from '/work/api/src/pipe-process'; +import { once } from 'node:events'; +import * as net from 'node:net'; +import assert from 'node:assert/strict'; + +const uid = 200123; +await installHostedAppNetworkGuard(uid); +const upstream = net.createServer(); +upstream.listen(0, '127.0.0.1'); +await once(upstream, 'listening'); +const port = (upstream.address() as net.AddressInfo).port; +const child = spawnPipeProcess(process.execPath, ['-e', ` +import net from 'node:net'; +import {once} from 'node:events'; +import assert from 'node:assert/strict'; +process.setgid(${uid}); process.setuid(${uid}); +const outbound = net.connect(${port}, '127.0.0.1'); +await assert.rejects(once(outbound, 'connect')); +outbound.destroy(); +const server = net.createServer(client => client.end('preview reply')); +server.listen(0, '127.0.0.1'); await once(server, 'listening'); +console.log(server.address().port); +await once(server, 'connection'); server.close(); +`]); +child.stdin.end(); +child.stderr.pipe(process.stderr); +const closed = once(child, 'close'); +let timer: ReturnType | undefined; +try { + timer = setTimeout(() => child.kill('SIGKILL'), 5000); + const [bytes] = await once(child.stdout, 'data'); + const preview = net.connect(Number(bytes.toString().trim()), '127.0.0.1'); + let reply = ''; + for await (const chunk of preview) reply += chunk.toString(); + assert.equal(reply, 'preview reply'); + assert.equal((await closed)[0], 0); + console.log('PASS: hosted-app egress is blocked and inbound preview replies work under supervisor policy'); +} finally { + clearTimeout(timer); child.kill('SIGKILL'); upstream.close(); +} diff --git a/tests/supervisor_policy.py b/tests/supervisor_policy.py new file mode 100644 index 00000000..caab12c0 --- /dev/null +++ b/tests/supervisor_policy.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Exercise the installed supervisor filter, including inherited Unix sockets.""" +import ctypes +import errno +import os +import platform +import socket +import subprocess +import sys + +POLICY = '/usr/local/bin/sandbox-supervisor-policy' +if len(sys.argv) > 1 and sys.argv[1] == '--filtered': + def denied(action): + try: + action() + except OSError as error: + assert error.errno == errno.EPERM, error + else: + raise AssertionError('supervisor syscall allowed') + denied(lambda: socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)) + denied(socket.socketpair) + inherited = socket.socket(fileno=int(sys.argv[2])) + denied(lambda: inherited.sendmsg([b'x'], [(socket.SOL_SOCKET, socket.SCM_RIGHTS, b'\0' * 4)])) + libc = ctypes.CDLL(None, use_errno=True) + calls = {'x86_64': [307, 425, 426, 427], 'aarch64': [269, 425, 426, 427]}[platform.machine()] + for number in calls: # sendmmsg and all io_uring entry points + assert libc.syscall(number, -1, 0, 0, 0, 0, 0) == -1 + assert ctypes.get_errno() == errno.EPERM + # TCP remains usable for the API listener and broker upstream. + listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + listener.bind(('127.0.0.1', 0)); listener.listen() + client = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + client.connect(listener.getsockname()) + peer, _ = listener.accept() + client.sendall(b'tcp'); assert peer.recv(3) == b'tcp' + print('PASS: supervisor denies Unix sockets, inherited descriptor sends and io_uring; TCP works') +else: + left, right = socket.socketpair() + result = subprocess.run([POLICY, sys.executable, __file__, '--filtered', str(left.fileno())], + pass_fds=[left.fileno()], timeout=5) + assert result.returncode == 0, result.returncode + # The pipe-based NsJail handshake must handle child setup/exec failure, + # as well as the successful normal path tested by the runner canary. + command = [POLICY, '/nsjail/nsjail', '--mode', 'o', '--disable_clone_newnet', + '--disable_clone_newuser', '--disable_clone_newns', '--disable_clone_newpid', + '--disable_clone_newipc', '--disable_clone_newuts', '--disable_clone_newcgroup', + '--disable_proc', '--disable_rlimits', '--', '/missing-codeapi-executable'] + result = subprocess.run(command, capture_output=True, timeout=5) + assert result.returncode != 0 + # The parent can kill the failed child before its final log is flushed. + # Its error-pipe diagnostic proves the startup handshake reported failure. + assert b'Received error message from the child process before it has been executed' in result.stderr, result.stderr + print('PASS: anonymous NsJail startup handshake reports child exec failure') diff --git a/tests/tool_call_pipes.py b/tests/tool_call_pipes.py new file mode 100644 index 00000000..c9388530 --- /dev/null +++ b/tests/tool_call_pipes.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +"""End-to-end broker -> anonymous pipes -> NsJail -> guard -> Python client. + +Runs in the native nsjail-ipc-test image with Docker's outer filter disabled. +Arguments: rendered policy, built Node broker, generated Python preamble. +""" +import ctypes +import http.server +import json +import os +import subprocess +import sys +import tempfile +import threading +import time + + +class Upstream(http.server.BaseHTTPRequestHandler): + calls = [] + + def log_message(self, *args): + pass + + def do_POST(self): + assert self.path == '/tool-call' + assert self.headers['X-Execution-ID'] == 'pipe-test-execution' + assert self.headers['X-Callback-Token'] == 'pipe-test-token' + data = json.loads(self.rfile.read(int(self.headers['Content-Length']))) + value = data['input']['n'] + self.calls.append(value) + time.sleep((8 - value) * 0.01) # Responses arrive out of order. + body = json.dumps({'success': True, 'result': value}).encode() + self.send_response(200) + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + + +def main(): + policy, broker, preamble = sys.argv[1:] + server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), Upstream) + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + with tempfile.TemporaryDirectory() as directory: + program = os.path.join(directory, 'client.py') + with open(preamble) as source: + client = source.read() + client += ''' +import errno +import socket +import subprocess +assert _stat.S_ISFIFO(os.fstat(3).st_mode) +assert _stat.S_ISFIFO(os.fstat(4).st_mode) +try: + os.fstat(9) +except OSError: + pass +else: + raise AssertionError("unrelated inherited descriptor survived spec-guard") +for domain in (socket.AF_UNIX, socket.AF_INET, socket.AF_INET6): + try: + socket.socket(domain, socket.SOCK_STREAM) + except OSError as error: + assert error.errno == errno.EPERM + else: + raise AssertionError("socket creation was allowed") +async def check(): + results = await asyncio.gather(*[_execute_tool_internal_async("echo", {"n": n}) for n in range(8)]) + assert results == list(range(8)), results +asyncio.run(check()) +# The capability must not leak through ordinary subprocess exec. +assert subprocess.check_output([sys.executable, "-c", "import os; print(os.path.exists('/proc/self/fd/3'))"]).strip() == b'False' +print("PASS: real pipes, restricted filter, guard cleanup and concurrent Python tool calls") +''' + with open(program, 'w') as output: + output.write(client) + env = {**os.environ, 'SANDBOX_FORWARD_TARGET': f'http://127.0.0.1:{server.server_port}', + 'TCS_PIPE_BRIDGE': '/usr/local/bin/tool-call-pipe-bridge'} + command = ['/usr/local/bin/tool-call-pipe-bridge', '--broker', '/usr/local/bin/node', broker, '/nsjail/nsjail', '--mode', 'e', + '--disable_clone_newnet', '--disable_clone_newuser', '--disable_clone_newns', + '--disable_clone_newpid', '--disable_clone_newipc', '--disable_clone_newuts', + '--disable_clone_newcgroup', '--disable_proc', '--disable_rlimits', + '--pass_fd', '3', '--pass_fd', '4', + '--seccomp_policy', policy, '--', '/usr/local/bin/spec-guard', + '--tool-call-pipes', '/usr/bin/python3', program] + result = subprocess.run(command, env=env, capture_output=True, text=True, timeout=30) + assert result.returncode == 0, (result.returncode, result.stdout, result.stderr) + assert sorted(Upstream.calls) == list(range(8)), Upstream.calls + print(result.stdout.strip()) + # Losing the response reader must terminate even with no buffered reply. + with tempfile.TemporaryDirectory() as directory: + closer = os.path.join(directory, 'close-response.py') + with open(closer, 'w') as output: + output.write("import os,time; os.close(4); time.sleep(30)") + interrupted = subprocess.run(command[:-1] + [closer], env=env, + capture_output=True, text=True, timeout=2) + assert interrupted.returncode == 137, (interrupted.returncode, interrupted.stdout, interrupted.stderr) + print('PASS: closing the response reader promptly terminates the invocation') + # A guest that drains rejection replies must still lose the capability + # once malformed-claim frames exhaust the broker's admission budget. + with tempfile.TemporaryDirectory() as directory: + flooder = os.path.join(directory, 'rejected-frames.py') + with open(flooder, 'w') as output: + output.write("""import json,os,struct,threading,time +threading.Thread(target=lambda: [os.read(4, 4096) for _ in range(1000)], daemon=True).start() +frames = [] +for id in range(1, 258): + frame = json.dumps({'id': id, 'headers': {}, 'body': ''}).encode() + frames.append(struct.pack('!I', len(frame)) + frame) +data = b''.join(frames) +while data: + data = data[os.write(3, data):] +time.sleep(30) +""") + interrupted = subprocess.run(command[:-1] + [flooder], env=env, + capture_output=True, text=True, timeout=2) + assert interrupted.returncode == 137, (interrupted.returncode, interrupted.stdout, interrupted.stderr) + print('PASS: drained rejection-frame flood promptly terminates the invocation') + # A killed API/controller must not orphan its broker, relay or job. Become a + # subreaper so this test can reap both adopted descendants in Docker. + assert ctypes.CDLL(None).prctl(36, 1, 0, 0, 0) == 0 + with tempfile.TemporaryDirectory() as directory: + marker = os.path.join(directory, 'job.pid') + sleeper = os.path.join(directory, 'sleep.py') + with open(sleeper, 'w') as output: + output.write(f"import os,time; open({marker!r},'w').write(str(os.getpid())); time.sleep(300)") + command[-1] = sleeper + controller = os.path.join(directory, 'controller.js') + broker_marker = os.path.join(directory, 'broker.pid') + with open(controller, 'w') as output: + output.write("import {spawn} from 'node:child_process'; import fs from 'node:fs';" + + f"const args={json.dumps(command)};" + + "const child=spawn(args[0], args.slice(1), {stdio:['pipe','inherit','inherit']}); child.stdin.end();" + + f"fs.writeFileSync({json.dumps(broker_marker)},String(child.pid));" + + "setInterval(()=>{},1000);") + running = subprocess.Popen(['/usr/local/bin/bun', controller], env=env, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + try: + deadline = time.monotonic() + 5 + while not os.path.exists(marker) and time.monotonic() < deadline: + time.sleep(0.01) + assert os.path.exists(marker), 'job did not start' + with open(marker) as source: + job_pid = int(source.read()) + with open(f'/proc/{job_pid}/status') as source: + relay_pid = next(int(line.split()[1]) for line in source if line.startswith('PPid:')) + with open(broker_marker) as source: + broker_pid = int(source.read()) + running.kill() + running.communicate(timeout=5) + for pid in [job_pid, relay_pid, broker_pid]: + deadline = time.monotonic() + 5 + while time.monotonic() < deadline: + reaped, status = os.waitpid(pid, os.WNOHANG) + if reaped: + assert os.waitstatus_to_exitcode(status) == -9 + break + time.sleep(0.01) + else: + os.kill(pid, 9) + raise AssertionError('orphan survived broker death') + finally: + if running.poll() is None: + running.kill() + running.communicate(timeout=5) + print('PASS: API death kills and reaps broker, relay and invocation') + server.shutdown() + server.server_close() + + +if __name__ == '__main__': + main() diff --git a/tests/tool_call_runner.cjs b/tests/tool_call_runner.cjs new file mode 100644 index 00000000..61ddcb9e --- /dev/null +++ b/tests/tool_call_runner.cjs @@ -0,0 +1,131 @@ +#!/usr/bin/env node +// Run inside the sandbox-build image with its Mac/Docker capabilities and a +// read-only /pkgs mount. Arguments: generated pipe preamble, Python version. +// Exercises the signed API route and the normal NsJail namespace/mount setup. +const assert = require('node:assert/strict'); +const crypto = require('node:crypto'); +const fs = require('node:fs'); +const http = require('node:http'); +const { spawn } = require('node:child_process'); +const { once } = require('node:events'); +const sleep = ms => new Promise(resolve => setTimeout(resolve, ms)); +function canonical(value) { + if (Array.isArray(value)) return `[${value.map(canonical).join(',')}]`; + if (value && typeof value === 'object') return `{${Object.keys(value).sort().map(key => `${JSON.stringify(key)}:${canonical(value[key])}`).join(',')}}`; + return JSON.stringify(value); +} +async function main() { + const [preamblePath, version = '3.14.4'] = process.argv.slice(2); + const calls = []; + const upstream = http.createServer(async (req, res) => { + try { + assert.equal(req.url, '/tool-call'); + assert.equal(req.headers['x-execution-id'], 'pipe-test-execution'); + assert.equal(req.headers['x-callback-token'], 'pipe-test-token'); + let bytes = ''; + for await (const part of req) bytes += part; + const { n } = JSON.parse(bytes).input; + calls.push(n); + await sleep((8 - n) * 10); + res.setHeader('Content-Type', 'application/json'); + res.end(JSON.stringify({ success: true, result: n })); + } catch (error) { res.writeHead(500).end(JSON.stringify({ success: false, error: String(error) })); } + }); + upstream.listen(0, '127.0.0.1'); + await once(upstream, 'listening'); + const port = upstream.address().port; + const { privateKey, publicKey } = crypto.generateKeyPairSync('ed25519'); + const api = spawn('/sandbox_api/entrypoint.sh', [], { cwd: '/sandbox_api', stdio: ['ignore', 'inherit', 'inherit'], env: { + ...process.env, PORT: '2000', SANDBOX_USE_CGROUPV2: 'false', + SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP: 'false', SANDBOX_REQUIRE_EGRESS_MANIFEST: 'true', + SANDBOX_EXECUTION_MANIFEST_PUBLIC_KEY: publicKey.export({ type: 'spki', format: 'pem' }), + SANDBOX_ALLOWED_LOCAL_NETWORK_PORT: String(port), SANDBOX_FORWARD_TARGET: `http://127.0.0.1:${port}`, + SANDBOX_OUTPUT_MAX_SIZE: '65536', SANDBOX_LOG_LEVEL: 'info', + }}); + try { + for (let n = 0; n < 200; n++) { + if (api.exitCode !== null) throw new Error(`API exited: ${api.exitCode}`); + try { if ((await fetch('http://127.0.0.1:2000/')).ok) break; } catch {} + if (n === 199) throw new Error('API did not start'); + await sleep(50); + } + const runtimes = await (await fetch('http://127.0.0.1:2000/api/v2/runtimes')).json(); + assert(runtimes.some(runtime => runtime.language === 'python' && runtime.version === version), JSON.stringify(runtimes)); + const outerNamespaces = Object.fromEntries(['net', 'pid', 'mnt', 'user', 'ipc', 'uts', 'cgroup'].map(name => [name, fs.readlinkSync(`/proc/self/ns/${name}`)])); + const client = fs.readFileSync(preamblePath, 'utf8') + ` +import errno, socket, subprocess, sys, multiprocessing as mp +def _mp_echo(sender): + try: + _do_request('POST', '/tool-call', '{}') + except RuntimeError as error: + assert 'primary process' in str(error) + else: + raise AssertionError('child acquired tool-call capability') + sender.send('child IPC works') + sender.close() +if __name__ == '__main__': + assert sys.version.split()[0] == ${JSON.stringify(version)} + assert os.getuid() == 65534 + assert all(_stat.S_ISFIFO(os.fstat(fd).st_mode) for fd in range(5)) + for _namespace, _outside in ${JSON.stringify(outerNamespaces)}.items(): + assert os.readlink('/proc/self/ns/' + _namespace) != _outside, _namespace + for domain in (socket.AF_UNIX, socket.AF_INET, socket.AF_INET6): + try: socket.socket(domain, socket.SOCK_STREAM) + except OSError as error: assert error.errno == errno.EPERM + else: raise AssertionError('socket creation allowed') + try: socket.socketpair() + except OSError as error: assert error.errno == errno.EPERM + else: raise AssertionError('socketpair allowed') + assert subprocess.check_output([sys.executable, '-c', "import os; print(os.path.exists('/proc/self/fd/3'))"]).strip() == b'False' + async def check(): + results = await asyncio.gather(*[_execute_tool_internal_async('echo', {'n': n}) for n in range(8)]) + assert results == list(range(8)), results + asyncio.run(check()) + # Pools need /dev/shm and forkserver needs a named AF_UNIX listener. + # Test the supported pipe-only spawn/fork multiprocessing paths. + for method in ('spawn', 'fork'): + ctx = mp.get_context(method) + receiver, sender = ctx.Pipe() + assert _stat.S_ISFIFO(os.fstat(receiver.fileno()).st_mode) + child = ctx.Process(target=_mp_echo, args=(sender,)) + child.start() + sender.close() + assert receiver.poll(5), method + assert receiver.recv() == 'child IPC works' + receiver.close() + child.join(5) + assert child.exitcode == 0, (method, child.exitcode) + print('PASS: packaged Python, signed API, NsJail namespaces, pipes, concurrent tools, subprocess and multiprocessing') +`; + async function execute(content, pipes, language = 'python', runtimeVersion = version) { + const body = { language, version: runtimeVersion, session_id: `pipe-canary-${crypto.randomUUID()}`, files: [{ name: language === 'bun-js' ? 'main.js' : language === 'bun-ts' ? 'main.ts' : 'main.py', content }], tool_call_socket: pipes, run_timeout: 30000 }; + const now = Math.floor(Date.now() / 1000); + const claims = { v: 1, exec_id: 'pipe-test-execution', tenant_id: 'canary', user_id: 'canary', session_key: 'canary:user:canary', input_files: [], read_sessions: [], output_session_id: body.session_id, max_upload_bytes: 52428800, max_output_files: 50, max_requests: 1000, iat: now, exp: now + 300, tool_call_socket: pipes, execute_body_sha256: crypto.createHash('sha256').update(canonical(body)).digest('base64url') }; + const payload = canonical(claims); + body.execution_manifest = `${Buffer.from(payload).toString('base64url')}.${crypto.sign(null, Buffer.from(payload), privateKey).toString('base64url')}`; + const response = await fetch('http://127.0.0.1:2000/api/v2/execute', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); + const result = await response.json(); + assert.equal(response.status, 200, JSON.stringify(result)); + assert.equal(result.run.code, 0, JSON.stringify(result)); + return result; + } + const result = await execute(client, true); + assert.equal(calls.length, 8); + assert.deepEqual([...calls].sort((a, b) => a - b), [0,1,2,3,4,5,6,7]); + assert(result.run.stdout.includes('PASS:'), JSON.stringify(result)); + console.log(result.run.stdout.trim()); + // Ordinary jobs still work and do not receive the tool-call descriptors. + const plain = await execute("import os, asyncio; assert not os.path.exists('/proc/self/fd/3'); asyncio.run(asyncio.to_thread(lambda: 1)); print('PASS: ordinary asyncio execution without pipe capability')", false); + console.log(plain.run.stdout.trim()); + for (const language of ['bun-js', 'bun-ts']) { + const js = await execute("let denied = false; try { Bun.spawn([process.execPath, '-e', '0'], {stdout:'pipe', stderr:'pipe'}); } catch (error) { if (error.code !== 'EPERM' || error.syscall !== 'socketpair') throw error; denied = true; } if (!denied) throw new Error('socketpair spawn allowed'); console.log('PASS: Bun ' + Bun.version + ', socketpair-based spawn denied')", false, language, '1.4.2'); + assert(js.run.stdout.includes('PASS: Bun 1.4.2'), JSON.stringify(js)); + console.log(language + ': ' + js.run.stdout.trim()); + } + } finally { + api.kill('SIGTERM'); + await once(api, 'exit'); + upstream.closeAllConnections(); upstream.close(); + } +} +main().catch(error => { console.error(error); process.exitCode = 1; }); diff --git a/tests/worktree-metadata-phases.mjs b/tests/worktree-metadata-phases.mjs new file mode 100644 index 00000000..089251b9 --- /dev/null +++ b/tests/worktree-metadata-phases.mjs @@ -0,0 +1,23 @@ +import assert from 'node:assert/strict'; +import { mkdtemp,writeFile,mkdir,rename,rm } from 'node:fs/promises'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +const exec=promisify(execFile),root=await mkdtemp(join(tmpdir(),'metadata-phases-')); +try { + const git=(args,cwd=root)=>exec('git',args,{cwd,env:{...process.env,GIT_CONFIG_NOSYSTEM:'1'}}); + await git(['init','-q','-b','main']); await writeFile(join(root,'file'),'tracked\n'); await git(['add','file']); + await git(['-c','user.name=test','-c','user.email=test@example.test','-c','commit.gpgsign=false','commit','-qm','seed']); + const commit=(await git(['rev-parse','HEAD'])).stdout.trim(); await mkdir(join(root,'.worktrees')); + const lane=join(root,'.worktrees','review'); + await git(['worktree','add','--no-checkout','--detach',lane,commit]); + await git(['-c','core.hooksPath=/dev/null','checkout','--detach',commit],lane); + assert.equal((await git(['status','--porcelain'],lane)).stdout,''); + const tombstone=join(root,'.retired-review'); await rename(lane,tombstone); + await git(['worktree','remove',lane]); + assert.doesNotMatch((await git(['worktree','list','--porcelain'])).stdout,/review/); + // Registration removal did not delete the detached bulk directory. + await rm(tombstone,{recursive:true}); + console.log('Split-phase registration/checkout and rename/detach/delete: passed'); +} finally { await rm(root,{recursive:true,force:true}); } From 98d99131386362613be1ca81a370c0d850b911f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B3n=20Levy?= Date: Mon, 12 Oct 2026 16:45:49 +0000 Subject: [PATCH 2/9] Sync upstream main: remove superseded path --- api/src/tool-call-socket-process.test.ts | 119 ----------------------- 1 file changed, 119 deletions(-) delete mode 100644 api/src/tool-call-socket-process.test.ts diff --git a/api/src/tool-call-socket-process.test.ts b/api/src/tool-call-socket-process.test.ts deleted file mode 100644 index 3efe2a4b..00000000 --- a/api/src/tool-call-socket-process.test.ts +++ /dev/null @@ -1,119 +0,0 @@ -import { afterEach, describe, expect, test } from 'bun:test'; -import * as fsp from 'fs/promises'; -import * as os from 'os'; -import * as path from 'path'; -import { config } from './config'; -import { - ensureToolCallSocketProxyReady, - stopToolCallSocketProxy, -} from './tool-call-socket-process'; - -const originalPort = config.allowed_local_network_port; -const originalEnv = { - target: process.env.SANDBOX_FORWARD_TARGET, - socket: process.env.TCS_SOCKET, - bundle: process.env.TCS_PROXY_BUNDLE, - node: process.env.TCS_NODE_BINARY, - marker: process.env.TCS_TEST_MARKER, -}; -let tempDir: string | undefined; - -function restoreEnv(name: string, value: string | undefined): void { - if (value === undefined) delete process.env[name]; - else process.env[name] = value; -} - -afterEach(async () => { - await stopToolCallSocketProxy(); - config.allowed_local_network_port = originalPort; - restoreEnv('SANDBOX_FORWARD_TARGET', originalEnv.target); - restoreEnv('TCS_SOCKET', originalEnv.socket); - restoreEnv('TCS_PROXY_BUNDLE', originalEnv.bundle); - restoreEnv('TCS_NODE_BINARY', originalEnv.node); - restoreEnv('TCS_TEST_MARKER', originalEnv.marker); - if (tempDir) await fsp.rm(tempDir, { recursive: true, force: true }); - tempDir = undefined; -}); - -describe('lazy tool-call socket process', () => { - test('starts once on demand, awaits readiness, and stops cleanly', async () => { - tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'lazy-tcs-')); - const socketPath = path.join(tempDir, 'proxy.sock'); - const markerPath = path.join(tempDir, 'launches.txt'); - const bundlePath = path.join(tempDir, 'fake-proxy.cjs'); - await fsp.writeFile(bundlePath, ` - const fs = require('fs'); - const net = require('net'); - try { fs.unlinkSync(process.env.TCS_SOCKET); } catch {} - fs.appendFileSync(process.env.TCS_TEST_MARKER, 'launch\\n'); - const server = net.createServer(socket => socket.end()); - server.listen(process.env.TCS_SOCKET); - const stop = () => server.close(() => process.exit(0)); - process.on('SIGTERM', stop); - process.on('SIGINT', stop); - `); - - config.allowed_local_network_port = 443; - process.env.SANDBOX_FORWARD_TARGET = 'https://gateway.example'; - process.env.TCS_SOCKET = socketPath; - process.env.TCS_PROXY_BUNDLE = bundlePath; - process.env.TCS_TEST_MARKER = markerPath; - - await Promise.all([ - ensureToolCallSocketProxyReady(), - ensureToolCallSocketProxyReady(), - ensureToolCallSocketProxyReady(), - ]); - - expect((await fsp.lstat(socketPath)).isSocket()).toBe(true); - expect((await fsp.readFile(markerPath, 'utf8')).trim().split('\n')).toHaveLength(1); - - /* A live child can lose its socket pathname (for example after an external - * cleanup). A previously-resolved launch promise must not mask that loss. */ - await fsp.unlink(socketPath); - await Promise.all([ - ensureToolCallSocketProxyReady(), - ensureToolCallSocketProxyReady(), - ensureToolCallSocketProxyReady(), - ]); - expect((await fsp.lstat(socketPath)).isSocket()).toBe(true); - expect((await fsp.readFile(markerPath, 'utf8')).trim().split('\n')).toHaveLength(2); - - await stopToolCallSocketProxy(); - expect(await fsp.lstat(socketPath).catch(() => null)).toBeNull(); - }); - - test('fails closed when forwarding is not configured', async () => { - config.allowed_local_network_port = 0; - process.env.SANDBOX_FORWARD_TARGET = ''; - await expect(ensureToolCallSocketProxyReady()).rejects.toThrow('not configured'); - }); - - test('reports a spawn failure without crashing and permits a later retry', async () => { - tempDir = await fsp.mkdtemp(path.join(os.tmpdir(), 'lazy-tcs-spawn-')); - const socketPath = path.join(tempDir, 'proxy.sock'); - const bundlePath = path.join(tempDir, 'fake-proxy.cjs'); - await fsp.writeFile(bundlePath, ` - const fs = require('fs'); - const net = require('net'); - try { fs.unlinkSync(process.env.TCS_SOCKET); } catch {} - const server = net.createServer(socket => socket.end()); - server.listen(process.env.TCS_SOCKET); - process.on('SIGTERM', () => server.close(() => process.exit(0))); - `); - - config.allowed_local_network_port = 443; - process.env.SANDBOX_FORWARD_TARGET = 'https://gateway.example'; - process.env.TCS_SOCKET = socketPath; - process.env.TCS_PROXY_BUNDLE = bundlePath; - process.env.TCS_NODE_BINARY = path.join(tempDir, 'missing-node'); - - await expect(ensureToolCallSocketProxyReady()).rejects.toThrow( - 'tool-call socket proxy failed to spawn', - ); - - process.env.TCS_NODE_BINARY = process.execPath; - await ensureToolCallSocketProxyReady(); - expect((await fsp.lstat(socketPath)).isSocket()).toBe(true); - }); -}); From f06f7310d9336728c7dece95f40757821b381671 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B3n=20Levy?= Date: Mon, 12 Oct 2026 16:45:53 +0000 Subject: [PATCH 3/9] Sync upstream main: remove superseded path --- api/src/tool-call-socket-process.ts | 191 ---------------------------- 1 file changed, 191 deletions(-) delete mode 100644 api/src/tool-call-socket-process.ts diff --git a/api/src/tool-call-socket-process.ts b/api/src/tool-call-socket-process.ts deleted file mode 100644 index 4694c07f..00000000 --- a/api/src/tool-call-socket-process.ts +++ /dev/null @@ -1,191 +0,0 @@ -import { spawn, type ChildProcess } from 'child_process'; -import * as net from 'net'; -import * as path from 'path'; -import { logger } from './logger'; -import { config } from './config'; - -const START_TIMEOUT_MS = 10_000; -const STOP_TIMEOUT_MS = 3_000; - -let child: ChildProcess | undefined; -let starting: Promise | undefined; - -function proxyBundlePath(): string { - return process.env.TCS_PROXY_BUNDLE - ?? path.resolve(__dirname, '..', '.build', 'tool-call-socket-proxy.cjs'); -} - -function proxySocketPath(): string { - return process.env.TCS_SOCKET || '/tmp/tcs.sock'; -} - -function proxyOwner(): { uid: string; gid: string } { - const runningAsRoot = typeof process.getuid === 'function' && process.getuid() === 0; - return { - uid: String(runningAsRoot ? 65534 : (process.getuid?.() ?? 65534)), - gid: String(runningAsRoot ? 65534 : (process.getgid?.() ?? 65534)), - }; -} - -function socketAcceptsConnections(socketPath: string): Promise { - return new Promise((resolve) => { - const socket = net.createConnection(socketPath); - let settled = false; - const finish = (ready: boolean): void => { - if (settled) return; - settled = true; - socket.destroy(); - resolve(ready); - }; - socket.once('connect', () => finish(true)); - socket.once('error', () => finish(false)); - socket.setTimeout(100, () => finish(false)); - }); -} - -async function waitUntilReady( - started: ChildProcess, - socketPath: string, - timeoutMs: number, -): Promise { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - if (started.exitCode !== null || started.signalCode !== null) { - throw new Error( - `tool-call socket proxy exited before readiness` - + ` (code=${String(started.exitCode)}, signal=${String(started.signalCode)})`, - ); - } - if (await socketAcceptsConnections(socketPath)) return; - await new Promise(resolve => setTimeout(resolve, 25)); - } - throw new Error(`tool-call socket proxy did not become ready within ${timeoutMs}ms`); -} - -async function launch(): Promise { - const rawTarget = process.env.SANDBOX_FORWARD_TARGET?.trim() ?? ''; - if (config.allowed_local_network_port <= 0 || !rawTarget) { - throw new Error('tool-call socket proxy is not configured'); - } - - const socketPath = proxySocketPath(); - const owner = proxyOwner(); - const started = spawn(process.env.TCS_NODE_BINARY || 'node', [proxyBundlePath()], { - env: { - ...process.env, - TCS_SOCKET: socketPath, - TCS_SOCKET_UID: owner.uid, - TCS_SOCKET_GID: owner.gid, - SANDBOX_FORWARD_TARGET: rawTarget, - }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - child = started; - let rejectSpawnFailure: (error: Error) => void; - const spawnFailure = new Promise((_resolve, reject) => { - rejectSpawnFailure = reject; - }); - /* A failed spawn emits `error`; it does not reliably set exitCode or - * signalCode. Without an immediate listener Node treats ENOENT/EACCES as an - * uncaught event and crashes the runner instead of failing this execution. */ - started.on('error', (error) => { - if (child === started) child = undefined; - const launchError = new Error( - `tool-call socket proxy failed to spawn: ${error.message}`, - { cause: error }, - ); - logger.error({ error }, 'tool-call socket proxy spawn failed'); - rejectSpawnFailure(launchError); - }); - started.stdout?.on('data', (chunk: Buffer) => { - logger.debug({ proxy: chunk.toString().trim() }, 'tool-call socket proxy'); - }); - started.stderr?.on('data', (chunk: Buffer) => { - logger.warn({ proxy: chunk.toString().trim() }, 'tool-call socket proxy stderr'); - }); - started.once('exit', (code, signal) => { - if (child === started) { - child = undefined; - } - if (code !== 0 && signal !== 'SIGTERM') { - logger.error({ code, signal }, 'tool-call socket proxy exited unexpectedly'); - } - }); - - try { - await Promise.race([ - waitUntilReady(started, socketPath, START_TIMEOUT_MS), - spawnFailure, - ]); - logger.info( - { socketPath, target: rawTarget }, - 'Tool-call socket proxy started after MicroVM restore', - ); - } catch (error) { - started.kill('SIGKILL'); - if (child === started) child = undefined; - throw error; - } -} - -/** - * Starts Node only when an authenticated execute actually receives the - * tool-call socket capability. Lambda MicroVM image creation snapshots the - * already-running container; starting Node in entrypoint would therefore clone - * its embedded OpenSSL process state into every VM. Lazy initialization keeps - * Node entirely on the post-restore side of that boundary. - */ -async function ensureToolCallSocketProxyReadyOnce(): Promise { - const socketPath = proxySocketPath(); - const running = child; - if (running && running.exitCode === null && running.signalCode === null) { - if (await socketAcceptsConnections(socketPath)) return; - running.kill('SIGKILL'); - if (child === running) child = undefined; - } - await launch(); -} - -export async function ensureToolCallSocketProxyReady(): Promise { - /* Serialize the whole probe/relaunch transaction, not only spawn readiness. - * Otherwise concurrent recovery callers can both await a failed socket probe, - * then each kill/relaunch through the shared `child` slot. Publishing this - * promise before the first probe await makes every follower join one attempt. */ - if (starting) { - await starting; - return; - } - const attempt = ensureToolCallSocketProxyReadyOnce(); - starting = attempt; - try { - await attempt; - } finally { - /* A successful launch must not leave a resolved promise cached forever: - * if the socket later disappears while the process is still alive, the - * next caller has to kill/relaunch rather than return false readiness. */ - if (starting === attempt) starting = undefined; - } -} - -export async function stopToolCallSocketProxy(): Promise { - const running = child; - child = undefined; - starting = undefined; - if (!running || running.exitCode !== null || running.signalCode !== null) return; - - await new Promise((resolve) => { - let settled = false; - const finish = (): void => { - if (settled) return; - settled = true; - clearTimeout(timer); - resolve(); - }; - const timer = setTimeout(() => { - running.kill('SIGKILL'); - finish(); - }, STOP_TIMEOUT_MS); - running.once('exit', finish); - running.kill('SIGTERM'); - }); -} From 971c10355cd220a8152f2acd0b1b6ab38fad8e73 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B3n=20Levy?= Date: Mon, 12 Oct 2026 16:45:58 +0000 Subject: [PATCH 4/9] Sync upstream main: remove superseded path --- .../tool-call-socket-proxy-runtime.test.ts | 905 ------------------ 1 file changed, 905 deletions(-) delete mode 100644 api/src/tool-call-socket-proxy-runtime.test.ts diff --git a/api/src/tool-call-socket-proxy-runtime.test.ts b/api/src/tool-call-socket-proxy-runtime.test.ts deleted file mode 100644 index 496f017b..00000000 --- a/api/src/tool-call-socket-proxy-runtime.test.ts +++ /dev/null @@ -1,905 +0,0 @@ -/** - * Runtime regression tests for the tool-call socket proxy. - * - * The tests in `tool-call-socket-proxy.test.ts` instantiate the proxy - * in-process via `startToolCallSocketProxy()` under bun:test. That is fine - * for verifying request-handler logic, but it CANNOT catch the audit DoS - * because Bun's node:http compat layer never fires `'connection'` events - * and Bun.serve's idleTimeout doesn't close silent unix-socket connects — - * so `activeConnections()`, `maxConnections`, and `idleSocketTimeoutMs` - * silently no-op when the proxy is hosted by Bun. The in-process tests - * pass vacuously (counters stay 0) while the production runtime is - * unprotected. - * - * In production the proxy runs under Node (see api/Dockerfile + the - * entrypoint.sh edit). These tests spawn the BUILT proxy bundle as a - * Node subprocess so the runtime under test matches production exactly. - * - * Skipped automatically when: - * - `node` isn't on PATH (CI without Node) - * - `.build/tool-call-socket-proxy.cjs` doesn't exist (test invoked - * before `bun run build`) - * - we're on Windows (unix sockets in temp dirs hit AppData EACCES - * quirks; production is Linux) - */ -import { afterEach, beforeAll, beforeEach, describe, expect, test } from 'bun:test'; -import { spawn, spawnSync, type ChildProcess } from 'node:child_process'; -import * as fs from 'node:fs'; -import * as fsp from 'node:fs/promises'; -import * as http from 'node:http'; -import * as net from 'node:net'; -import * as os from 'node:os'; -import * as path from 'node:path'; - -const PROXY_BUNDLE = path.resolve(__dirname, '..', '.build', 'tool-call-socket-proxy.cjs'); -const NODE_AVAILABLE = (() => { - try { return spawnSync('node', ['--version'], { stdio: 'ignore' }).status === 0; } - catch { return false; } -})(); -const SHOULD_RUN = NODE_AVAILABLE && fs.existsSync(PROXY_BUNDLE) && process.platform !== 'win32'; - -const describeIfRuntime = SHOULD_RUN ? describe : describe.skip; - -interface UpstreamHandle { - server: http.Server; - url: string; - calls: () => number; - close: () => Promise; -} - -interface ProxyHandle { - socketPath: string; - proc: ChildProcess; - stop: () => Promise; -} - -let workdir: string; - -beforeAll(() => { - if (!NODE_AVAILABLE) { - console.warn('[tcs-proxy-runtime] skipping: `node` not on PATH'); - } else if (!fs.existsSync(PROXY_BUNDLE)) { - console.warn(`[tcs-proxy-runtime] skipping: ${PROXY_BUNDLE} missing — run \`bun run build\` first`); - } else if (process.platform === 'win32') { - console.warn('[tcs-proxy-runtime] skipping: Windows (unix-socket EACCES quirk); production is Linux'); - } -}); - -beforeEach(async () => { - workdir = await fsp.mkdtemp(path.join(os.tmpdir(), 'tcs-proxy-runtime-')); -}); - -afterEach(async () => { - if (workdir) await fsp.rm(workdir, { recursive: true, force: true }); -}); - -async function startUpstream(handler?: http.RequestListener): Promise { - let calls = 0; - const server = http.createServer((req, res) => { - calls++; - if (handler) return handler(req, res); - let total = 0; - req.on('data', chunk => { total += chunk.length; }); - req.on('end', () => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ ok: true, received: total })); - }); - }); - await new Promise((resolve, reject) => { - server.once('error', reject); - server.listen(0, '127.0.0.1', resolve); - }); - const addr = server.address(); - if (!addr || typeof addr === 'string') throw new Error('upstream listen failed'); - return { - server, - url: `http://127.0.0.1:${addr.port}`, - calls: () => calls, - close: () => new Promise(r => server.close(() => r())), - }; -} - -async function spawnProxy(opts: { upstreamUrl: string; env?: Record }): Promise { - const socketPath = path.join(workdir, `tcs-${Math.random().toString(36).slice(2)}.sock`); - const proc = spawn('node', [PROXY_BUNDLE], { - env: { - ...process.env, - TCS_SOCKET: socketPath, - SANDBOX_FORWARD_TARGET: opts.upstreamUrl, - ...(opts.env || {}), - }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - - /* Wait for the proxy to be connectable. fs.existsSync on a unix socket - * is unreliable on some platforms; try-connect-then-disconnect is the - * authoritative readiness probe. */ - let ready = false; - for (let i = 0; i < 50; i++) { - try { - const probe = net.createConnection(socketPath); - await new Promise((res, rej) => { - probe.once('connect', () => { probe.destroy(); res(); }); - probe.once('error', rej); - }); - ready = true; - break; - } catch { /* not yet */ } - await new Promise(r => setTimeout(r, 100)); - } - if (!ready) { - proc.kill('SIGKILL'); - throw new Error('proxy never became connectable'); - } - - return { - socketPath, - proc, - stop: async () => { - proc.kill('SIGTERM'); - await new Promise(r => { - const killTimer = setTimeout(() => { proc.kill('SIGKILL'); r(); }, 1000); - proc.once('exit', () => { - clearTimeout(killTimer); - r(); - }); - }); - }, - }; -} - -function rawConnect(socketPath: string): Promise { - return new Promise((resolve, reject) => { - const sock = net.createConnection(socketPath); - sock.once('connect', () => resolve(sock)); - sock.once('error', reject); - }); -} - -function slowHeaderConnect( - socketPath: string, - dripEveryMs: number, -): Promise<{ socket: net.Socket; close: () => void }> { - return new Promise((resolve, reject) => { - const sock = net.createConnection(socketPath); - let drip: ReturnType | undefined; - const clearDrip = (): void => { - if (drip) clearInterval(drip); - drip = undefined; - }; - const onInitialError = (error: Error): void => { - clearDrip(); - reject(error); - }; - - sock.once('error', onInitialError); - sock.once('connect', () => { - sock.off('error', onInitialError); - sock.on('error', () => { /* connection was destroyed by the proxy */ }); - sock.once('close', clearDrip); - sock.write('POST /tool-call HTTP/1.1\r\nHost: x\r\nX-Drip: '); - drip = setInterval(() => { - if (sock.destroyed) { - clearDrip(); - return; - } - try { sock.write('x'); } catch { clearDrip(); } - }, dripEveryMs); - resolve({ - socket: sock, - close: () => { - clearDrip(); - sock.destroy(); - }, - }); - }); - }); -} - -/* The proxy's PTC-presence filter rejects /tool-call requests that lack - * the SDK-supplied headers (canonical 404 — see proxy source for why). - * Tests that want to exercise the forwarding path supply matching shape - * here. Cryptographic validation is the upstream's job; presence is the - * proxy's. */ -const PTC_HEADERS = { - 'X-Execution-ID': 'test-exec-id', - 'X-Tool-Call-ID': 'test-call-001', - 'X-Callback-Token': 'test-callback-token', -} as const; - -function postToolCall( - socketPath: string, - body: string, - extraHeaders: Record = {}, -): Promise<{ status: number; body: string }> { - return new Promise((resolve, reject) => { - const req = http.request( - { - socketPath, - method: 'POST', - path: '/tool-call', - headers: { - 'Content-Type': 'application/json', - 'Content-Length': Buffer.byteLength(body), - ...PTC_HEADERS, - ...extraHeaders, - }, - }, - res => { - const chunks: Buffer[] = []; - res.on('data', c => chunks.push(c)); - res.on('end', () => resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString() })); - }, - ); - req.on('error', reject); - req.write(body); - req.end(); - }); -} - -describeIfRuntime('tool-call-socket-proxy under Node runtime (production parity)', () => { - test('forwards a normal POST /tool-call', async () => { - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const res = await postToolCall(proxy.socketPath, '{"x":1}'); - expect(res.status).toBe(200); - expect(JSON.parse(res.body)).toEqual({ ok: true, received: 7 }); - expect(upstream.calls()).toBe(1); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('SILENT-CONNECTION DoS: idle accept-then-do-nothing connections close before they pin FDs', async () => { - /* This is the audit's actual attack: - * for i in range(N): socket.connect('/tmp/tcs.sock') - * Open many connections without writing anything. Without the Node - * runtime + working `'connection'` events + working socket.setTimeout, - * each accepted FD sits in the proxy until the OS limit. With them - * all working, every FD must be reclaimed within a small multiple of - * idleSocketTimeoutMs. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { /* defaults: idle 2s, max 64. Test against the production defaults. */ }, - }); - try { - const sockets: net.Socket[] = []; - for (let i = 0; i < 32; i++) { - try { sockets.push(await rawConnect(proxy.socketPath)); } - catch { /* over-cap rejection is fine — also "didn't pin a FD" */ } - } - /* Wait for idle timer (default 2000ms) + slack. */ - const start = Date.now(); - const closeTimes = await Promise.all(sockets.map(s => new Promise(resolve => { - if (s.destroyed) { resolve(0); return; } - s.once('close', () => resolve(Date.now() - start)); - setTimeout(() => resolve(-1), 8000); - }))); - const closed = closeTimes.filter(t => t >= 0).length; - const stuck = closeTimes.filter(t => t < 0).length; - expect(stuck).toBe(0); - /* All idle FDs must be reclaimed within a small multiple of the - * default idleSocketTimeoutMs (2s). Slack is generous. */ - const maxClose = Math.max(...closeTimes); - expect(maxClose).toBeLessThan(7000); - expect(closed).toBeGreaterThan(0); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('legitimate POST still succeeds after a silent-connection storm clears', async () => { - /* Regression guard for the actual user-facing failure: while - * malicious code spams idle connections, real tool calls from - * sibling sandboxes must still succeed. With the default cap (64) - * and idleSocketTimeoutMs=2s, we open the storm, wait ~2.5s for - * the proxy to reclaim those FDs, then verify a real POST works. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - const stormSockets: net.Socket[] = []; - try { - for (let i = 0; i < 32; i++) { - try { stormSockets.push(await rawConnect(proxy.socketPath)); } - catch { /* over-cap rejection is fine */ } - } - /* Let the proxy's idleSocketTimeoutMs (default 2s) reclaim FDs. */ - await new Promise(r => setTimeout(r, 2500)); - const res = await postToolCall(proxy.socketPath, '{"during":"storm"}'); - expect(res.status).toBe(200); - expect(JSON.parse(res.body)).toEqual({ ok: true, received: 18 }); - } finally { - stormSockets.forEach(s => { try { s.destroy(); } catch { /* ignore */ } }); - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: slow incomplete headers cannot starve all connection slots', async () => { - /* Header-drip clients are more subtle than silent sockets: they send - * bytes under the idle timeout, so socket.setTimeout never fires. The - * proxy must still reclaim each accepted socket on an absolute - * accept-to-headers deadline before all maxConnections slots can be held. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { - TCS_MAX_CONNECTIONS: '4', - TCS_HEADER_TIMEOUT_MS: '500', - TCS_CONNECTION_RATE_BURST: '32', - TCS_CONNECTION_RATE_REFILL_PER_SEC: '200', - }, - }); - const drips: Array<{ socket: net.Socket; close: () => void }> = []; - - try { - for (let i = 0; i < 4; i++) { - drips.push(await slowHeaderConnect(proxy.socketPath, 100)); - } - - await new Promise(r => setTimeout(r, 900)); - - const res = await postToolCall(proxy.socketPath, '{"x":1}'); - expect(res.status).toBe(200); - expect(JSON.parse(res.body)).toEqual({ ok: true, received: 7 }); - expect(upstream.calls()).toBe(1); - } finally { - drips.forEach(drip => drip.close()); - await proxy.stop(); - await upstream.close(); - } - }, { timeout: 10_000 }); - - test('CIRCUMVENTION: HTTP request smuggling (Content-Length + Transfer-Encoding) does not reach upstream', async () => { - /* The classic CL.TE / TE.CL smuggling primitive: send both headers - * with conflicting interpretations. If the proxy and upstream - * disagree on body boundary, the attacker can prepend bytes to a - * synthesized "second request" that bypasses the path filter and - * could reach a different upstream route. The defense per RFC 7230 - * §3.3.3 is to reject the request rather than try to reconcile. - * Whether node:http rejects in its own parser or our isSmugglingShaped - * gate fires first, the security property is the same: upstream - * must never see the smuggled bytes. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const sock = await rawConnect(proxy.socketPath); - const smuggled = - 'POST /tool-call HTTP/1.1\r\n' + - 'Host: x\r\n' + - 'Content-Length: 13\r\n' + - 'Transfer-Encoding: chunked\r\n' + - 'Connection: close\r\n' + - '\r\n' + - '0\r\n\r\n' + - 'POST /admin HTTP/1.1\r\nHost: x\r\nContent-Length: 0\r\n\r\n'; - sock.write(smuggled); - await new Promise(resolve => { - sock.once('close', () => resolve()); - setTimeout(resolve, 3000); - }); - /* Critical security property: upstream MUST NOT have been called. */ - expect(upstream.calls()).toBe(0); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: hop-by-hop headers (Connection, Upgrade, TE, Trailer, Transfer-Encoding) do not reach upstream', async () => { - /* RFC 7230 §6.1: a proxy MUST NOT forward hop-by-hop headers. If we - * leak them through, an upstream that treats Upgrade specially could - * be coerced into an unexpected protocol switch, and Transfer- - * Encoding pollution feeds smuggling chains at the next hop. */ - let receivedHeaders: http.IncomingHttpHeaders = {}; - const upstream = await startUpstream((req, res) => { - receivedHeaders = req.headers; - let total = 0; - req.on('data', c => { total += c.length; }); - req.on('end', () => { - res.writeHead(200, { 'Content-Type': 'application/json', 'Connection': 'keep-alive' }); - res.end(JSON.stringify({ ok: true, received: total })); - }); - }); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const body = '{"x":1}'; - const reply = await new Promise<{ status: number; body: string; headers: http.IncomingHttpHeaders }>((resolve, reject) => { - const req = http.request({ - socketPath: proxy.socketPath, - method: 'POST', - path: '/tool-call', - headers: { - 'Content-Type': 'application/json', - 'Content-Length': Buffer.byteLength(body), - /* PTC presence required to forward (proxy filter). */ - ...PTC_HEADERS, - /* Hop-by-hop headers we expect the proxy to strip: */ - 'Connection': 'keep-alive, Upgrade-Insecure-Requests', - 'Keep-Alive': 'timeout=300, max=1000', - 'Upgrade': 'websocket', - 'TE': 'trailers', - 'Trailer': 'X-Foo', - 'Proxy-Authorization': 'Basic c3RlYWw=', - 'Proxy-Connection': 'close', - 'Host': 'attacker-controlled.example', - }, - }, res => { - const chunks: Buffer[] = []; - res.on('data', c => chunks.push(c)); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString(), - headers: res.headers, - })); - }); - req.on('error', reject); - req.write(body); - req.end(); - }); - expect(reply.status).toBe(200); - /* Upstream must not have seen the attacker's hop-by-hop values. - * The Connection header is special: node:http always sets one - * automatically (we tell it `connection: close`), so what we - * verify there is "the proxy's choice, not the attacker's". */ - expect(receivedHeaders.connection).toBe('close'); - expect(receivedHeaders.connection).not.toContain('Upgrade'); - for (const stripped of [ - 'keep-alive', 'upgrade', 'te', 'trailer', - 'transfer-encoding', 'proxy-authorization', 'proxy-connection', - ]) { - expect(receivedHeaders[stripped]).toBeUndefined(); - } - /* Host header must be the upstream's, not the attacker's value. */ - expect(receivedHeaders.host).not.toBe('attacker-controlled.example'); - /* And the upstream's hop-by-hop response headers must not reach - * the sandbox client (Connection: keep-alive from upstream gets - * stripped, our Connection: close wins). */ - expect(reply.headers.connection).toBe('close'); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: slow-loris body upload is bounded by requestBodyTimeoutMs (~5s default)', async () => { - /* Drip body bytes one at a time, each within the per-byte idle - * window. Without an absolute server.requestTimeout, this would - * stretch up to activeRequestTimeoutMs (default 30s) and let an - * attacker monopolize a slot. With server.requestTimeout = - * requestBodyTimeoutMs, the proxy must close us within ~5s. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const sock = await rawConnect(proxy.socketPath); - sock.write( - 'POST /tool-call HTTP/1.1\r\n' + - 'Host: x\r\n' + - 'Content-Type: application/json\r\n' + - 'Content-Length: 100\r\n' + - 'Connection: close\r\n' + - /* PTC headers required so the proxy forwards (presence-only check; - * lets us reach the bodyUploadDeadline path the test exercises). */ - 'X-Execution-ID: test-exec-id\r\n' + - 'X-Tool-Call-ID: test-call-001\r\n' + - 'X-Callback-Token: test-callback-token\r\n' + - '\r\n', - ); - /* Drip 1 byte every 800ms. After 5s we've sent ~6 bytes — far short - * of the declared 100 — and the proxy must have closed us. */ - const start = Date.now(); - const dripInterval = setInterval(() => { - if (sock.destroyed) { clearInterval(dripInterval); return; } - try { sock.write('x'); } catch { clearInterval(dripInterval); } - }, 800); - const closedAt = await new Promise(resolve => { - sock.once('close', () => resolve(Date.now() - start)); - setTimeout(() => resolve(-1), 15000); - }); - clearInterval(dripInterval); - expect(closedAt).toBeGreaterThanOrEqual(0); - /* Bounded close — proves slow-loris is mitigated. Generous upper - * bound to absorb scheduler jitter; the point is "<<30s". */ - expect(closedAt).toBeLessThan(12000); - expect(upstream.calls()).toBe(0); - } finally { - await proxy.stop(); - await upstream.close(); - } - }, { timeout: 15_000 }); - - test('CIRCUMVENTION: HTTP/2 binary framing on a /tmp/tcs.sock connection fails clean (no crash)', async () => { - /* If the proxy crashed on unexpected protocol bytes, an attacker - * could brick it. Send raw HTTP/2 connection-preface bytes; the - * node:http parser must close the socket with clientError without - * affecting subsequent legitimate clients. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - /* HTTP/2 connection preface (24 bytes) per RFC 7540. */ - const h2Preface = Buffer.from('PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n'); - for (let i = 0; i < 5; i++) { - const sock = await rawConnect(proxy.socketPath); - sock.write(h2Preface); - await new Promise(r => sock.once('close', () => r())); - } - /* Proxy must still serve a normal POST. */ - const res = await postToolCall(proxy.socketPath, '{"x":1}'); - expect(res.status).toBe(200); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: oversize request body is rejected with 413 before reaching upstream', async () => { - /* maxBodyBytes (default 1 MiB) bounds memory pressure. The - * Content-Length precheck closes the loophole where streaming the - * body to upstream would let the first chunks of an honestly- - * declared oversize request leak through. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { /* default maxBodyBytes */ }, - }); - try { - const big = 'A'.repeat(2 * 1024 * 1024); // 2 MiB body - const reply = await new Promise<{ status: number; body: string }>(resolve => { - const req = http.request({ - socketPath: proxy.socketPath, - method: 'POST', - path: '/tool-call', - headers: { - 'Content-Length': Buffer.byteLength(big), - 'Connection': 'close', - /* PTC presence so the proxy actually attempts to forward, - * letting the maxBodyBytes / Content-Length precheck fire. */ - ...PTC_HEADERS, - }, - }, res => { - const chunks: Buffer[] = []; - res.on('data', c => chunks.push(c)); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString(), - })); - }); - req.on('error', err => resolve({ status: 0, body: err.message })); - req.write(big); - req.end(); - }); - /* Either 413 (CL precheck or runtime byte counter) or transport - * error (proxy closed mid-stream). The security property is the - * same: NO bytes of the oversize body reach the upstream. */ - expect([413, 0]).toContain(reply.status); - expect(upstream.calls()).toBe(0); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('PTC presence filter: duplicate empty header lines do not bypass the gate', async () => { - /* Codex caught this: Node joins duplicate request headers with - * `, ` (per RFC 7230 §3.2.2). A naive `!headers[name]` check on - * `X-Execution-ID:` sent twice with empty values produces the - * truthy string `", "`, slips past the gate, and lets the - * attacker reach the upstream — whose structured auth-error - * response defeats the route-opacity goal. The proxy must - * normalise (strip commas + whitespace) before checking. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const sock = await rawConnect(proxy.socketPath); - /* Each required PTC header sent TWICE with empty values. Without - * the fix, Node's join produces `, ` for each name and the - * proxy forwards. With the fix, the values normalise to '' and - * the proxy returns its canonical 404. */ - sock.write( - 'POST /tool-call HTTP/1.1\r\n' + - 'Host: x\r\n' + - 'Content-Length: 0\r\n' + - 'Connection: close\r\n' + - 'X-Execution-ID:\r\n' + - 'X-Execution-ID:\r\n' + - 'X-Tool-Call-ID:\r\n' + - 'X-Tool-Call-ID:\r\n' + - 'X-Callback-Token:\r\n' + - 'X-Callback-Token:\r\n' + - '\r\n', - ); - const reply = await new Promise(resolve => { - const chunks: Buffer[] = []; - sock.on('data', c => chunks.push(c)); - sock.on('close', () => resolve(Buffer.concat(chunks).toString())); - setTimeout(() => resolve(Buffer.concat(chunks).toString()), 3000); - }); - /* Critical: upstream MUST NOT have been called. Status MUST be - * 404. Body MUST be the canonical "not found". */ - expect(reply).toContain('404'); - expect(reply).toContain('not found'); - expect(reply).not.toMatch(/X-Request-ID|ETag/i); - expect(upstream.calls()).toBe(0); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('PTC presence filter: missing headers get canonical 404, never reach upstream', async () => { - /* The proxy filters on PTC-header presence before forwarding so a - * sandbox attacker probing /tool-call without SDK headers cannot - * fingerprint the route via the upstream's structured "missing PTC - * headers" response. The `bogus_headers` case proves cryptographic - * validation is still upstream's job — bogus values DO forward and - * upstream's structured error is preserved (so the SDK can parse it). */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - // No PTC headers → proxy 404, upstream NOT called - const noHeaders = await new Promise<{ status: number; body: string; ct?: string }>(resolve => { - const r = http.request({ socketPath: proxy.socketPath, method: 'POST', path: '/tool-call' }, res => { - const chunks: Buffer[] = []; - res.on('data', c => chunks.push(c)); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString(), - ct: res.headers['content-type'] as string | undefined, - })); - }); - r.on('error', err => resolve({ status: 0, body: err.message })); - r.end(); - }); - expect(noHeaders.status).toBe(404); - expect(noHeaders.body).toBe('not found'); - expect(noHeaders.ct).toContain('text/plain'); - const callsAfterNoHeaders = upstream.calls(); - expect(callsAfterNoHeaders).toBe(0); - - // With PTC headers (any values) → proxy forwards - const withHeaders = await postToolCall(proxy.socketPath, '{"x":1}'); - expect(withHeaders.status).toBe(200); - expect(upstream.calls()).toBe(1); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: pipelined requests on a single socket only target /tool-call and respect active-request budget', async () => { - /* HTTP/1.1 pipelining lets a client batch requests on one socket. - * Both pipelined requests still go through the proxy's path filter - * and active-request budget — they cannot reach a different upstream - * route. node:http may parse and dispatch the second request - * before the first response's Connection: close fires; that is fine - * as long as the second is ALSO subject to all the proxy's defenses. */ - let receivedPaths: string[] = []; - const upstream = await startUpstream((req, res) => { - receivedPaths.push(req.url ?? ''); - let n = 0; req.on('data', c => { n += c.length; }); - req.on('end', () => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ ok: true, received: n })); - }); - }); - const proxy = await spawnProxy({ upstreamUrl: upstream.url }); - try { - const sock = await rawConnect(proxy.socketPath); - /* Second request tries to hit /admin (a path the proxy's filter - * MUST reject). If pipelining could bypass the filter, we'd see - * /admin in receivedPaths. */ - /* PTC headers on the first request so it actually forwards; the - * second request deliberately omits them AND targets /admin to - * verify both the path filter AND the presence filter hold even - * across pipelined dispatch. */ - const pipelined = - 'POST /tool-call HTTP/1.1\r\nHost: x\r\nContent-Length: 7\r\n' + - 'X-Execution-ID: t\r\nX-Tool-Call-ID: t\r\nX-Callback-Token: t\r\n\r\n{"x":1}' + - 'POST /admin HTTP/1.1\r\nHost: x\r\nContent-Length: 7\r\n\r\n{"y":2}'; - sock.write(pipelined); - await new Promise(r => sock.once('close', () => r())); - /* Whatever node:http chose to dispatch, the path filter must hold: - * upstream must NEVER see /admin. */ - expect(receivedPaths.every(p => p === '/tool-call')).toBe(true); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('ACTIVE-FLOOD DoS: excess concurrent /tool-call requests get 429 and do not reach upstream', async () => { - /* Hold the upstream so requests pile up at the proxy's active cap. - * With maxActiveRequests=2, only 2 requests should reach upstream; - * the others should get 429 + Retry-After. */ - let pendingUpstreams = 0; - let release: (() => void) | undefined; - const releaseAll = new Promise(r => { release = r; }); - const upstream = await startUpstream(async (req, res) => { - pendingUpstreams++; - req.resume(); - await new Promise(r => req.on('end', () => r())); - await releaseAll; - res.writeHead(200).end('{"ok":true}'); - }); - - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { - SANDBOX_MAX_CONCURRENT_JOBS: '2', - }, - }); - - try { - const inflight = Array.from({ length: 32 }, () => - postToolCall(proxy.socketPath, '{"x":1}').catch(err => ({ status: -1, body: String(err) })), - ); - /* Give the proxy time to admit the first batch and 429 the rest. */ - await new Promise(r => setTimeout(r, 500)); - release!(); - const results = await Promise.all(inflight); - const ok = results.filter(r => r.status === 200).length; - const tooMany = results.filter(r => r.status === 429).length; - /* Active cap default-derives from SANDBOX_MAX_CONCURRENT_JOBS but is - * floored at 16. Launch above the floor so this test actually exercises - * the 429 branch instead of merely proving that 16 requests succeed. */ - expect(ok).toBeGreaterThan(0); - expect(ok).toBeLessThanOrEqual(16); - expect(tooMany).toBeGreaterThan(0); - expect(upstream.calls()).toBeLessThanOrEqual(16); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CIRCUMVENTION: SIGTERM during proxy init exits cleanly (no TypeError on undefined handle)', async () => { - /* Codex flagged that `shutdown()` could fall through to `handle.close()` - * when a signal arrives before `startToolCallSocketProxy()` resolves — - * which would crash with `TypeError: Cannot read properties of undefined`. - * Hold the socket path with a sentinel server so the real proxy's - * listen() blocks indefinitely (EADDRINUSE on a unix socket retries - * via Node's listen-with-existing-socket dance, but with another - * server actively listening the bind never resolves), then signal it. - * The proxy must exit code 0 (signal handler ran cleanly) — never - * exit on the SIGTERM signal directly (which would mean the handler - * never ran) and never exit non-zero (which would mean the handler - * ran but threw). */ - const socketPath = path.join(workdir, 'sigterm-init-race.sock'); - /* Pre-bind the socket path so the proxy's listen() hangs in EADDRINUSE - * retry loops, keeping `handle` undefined long enough to test the race. */ - const blocker = http.createServer(() => { /* noop */ }); - await new Promise(r => blocker.listen(socketPath, () => r())); - - const proc = spawn('node', [PROXY_BUNDLE], { - env: { - ...process.env, - TCS_SOCKET: socketPath, - SANDBOX_FORWARD_TARGET: 'http://127.0.0.1:1', - }, - stdio: ['ignore', 'pipe', 'pipe'], - }); - /* Wait long enough for Node to install the SIGTERM handler but - * NOT long enough for startToolCallSocketProxy to have resolved - * (it can't — the socket is bound by `blocker`). 100ms is well - * above the ~10ms handler-install window observed in probes. */ - await new Promise(r => setTimeout(r, 100)); - proc.kill('SIGTERM'); - - const exit = await new Promise<{ code: number | null; signal: NodeJS.Signals | null }>(resolve => { - const t = setTimeout(() => { - proc.kill('SIGKILL'); - resolve({ code: -1, signal: null }); - }, 3000); - proc.once('exit', (code, signal) => { - clearTimeout(t); - resolve({ code, signal }); - }); - }); - - /* Critical assertion: handler ran AND exited cleanly. Exit code 0 is - * the only acceptable outcome — anything else (signal:'SIGTERM' meaning - * default behavior, code 1 meaning thrown error, code -1 meaning hung - * past 3s) indicates the race window is broken. */ - expect(exit.code).toBe(0); - expect(exit.signal).toBeNull(); - - await new Promise(r => blocker.close(() => r())); - }); - - test('CONNECT-FLOOD DoS: connection-rate limiter drops fast-burst accepts that would pressure kernel slab', async () => { - /* This is the SPECIFIC audit scenario PR #1648 follow-up was opened - * for: 500 AF_UNIX connect()s in <50 ms. Without the rate limiter, - * every accept allocates kernel unix_sock + dentry + skb queue slab - * entries; the proxy's existing maxConnections only bounds the - * application-level activeSockets set, not the allocation rate. - * Observation from the audit: in-VM kernel slab pressure surfaces - * as EMFILE in subsequent loader open() calls even though the fd - * table itself sits at 0.1% utilization. - * - * Test verifies the wire-level effect: with burst=8, refill=0 (frozen - * for determinism), only the first 8 of 100 concurrent /tool-call - * requests reach the upstream. The remaining 92 hit the rate limiter - * and are destroyed before the HTTP request layer sees them — they - * surface as either ECONNRESET / 'socket hang up' or EPIPE on the - * client side. */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { - /* burst=8 — much smaller than maxConnections so the RATE limit - * bites first. refill=1/sec is effectively frozen over the - * test's ~100ms window. */ - TCS_CONNECTION_RATE_BURST: '8', - TCS_CONNECTION_RATE_REFILL_PER_SEC: '1', - }, - }); - - try { - const results = await Promise.allSettled( - Array.from({ length: 100 }, () => - postToolCall(proxy.socketPath, '{"x":1}'), - ), - ); - - const ok = results.filter(r => r.status === 'fulfilled' && r.value.status === 200).length; - const dropped = results.length - ok; - - /* Burst=8 with a near-frozen refill — somewhere in [8, 12] should - * succeed (burst + occasional refill during test window). The - * key assertion is that the count is bounded WELL below 100. */ - expect(ok).toBeGreaterThanOrEqual(1); - expect(ok).toBeLessThanOrEqual(20); - expect(dropped).toBeGreaterThanOrEqual(80); - /* And critically: upstream sees only the rate-limit-allowed ones, - * NOT 100. This bounds kernel slab pressure: the proxy never - * creates the in-VM unix_sock entries for the dropped batch. */ - expect(upstream.calls()).toBeLessThanOrEqual(ok); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); - - test('CONNECT-FLOOD recovery: bucket refills so legitimate traffic resumes after a flood', async () => { - /* Follow-up to the test above: confirm the limiter doesn't permanently - * starve a connection. After draining the burst bucket, wait long - * enough for the refill to top it back up, then verify a new - * /tool-call succeeds. This guards against a bug where dropped - * connections might leak the rate-limit credit (e.g. if the bucket - * was decremented BEFORE the maxConnections check and never - * compensated when both gates were hit by the same connection). */ - const upstream = await startUpstream(); - const proxy = await spawnProxy({ - upstreamUrl: upstream.url, - env: { - TCS_CONNECTION_RATE_BURST: '4', - /* 20/sec refill — burst recovers in ~250ms. */ - TCS_CONNECTION_RATE_REFILL_PER_SEC: '20', - }, - }); - - try { - /* Drain the burst with a parallel flood. */ - const drain = await Promise.allSettled( - Array.from({ length: 30 }, () => postToolCall(proxy.socketPath, '{"x":1}')), - ); - const okDuringDrain = drain.filter(r => r.status === 'fulfilled' && r.value.status === 200).length; - expect(okDuringDrain).toBeGreaterThanOrEqual(1); - expect(okDuringDrain).toBeLessThanOrEqual(15); - - /* Wait for the bucket to refill past 1 token (50ms refills 1 token - * at 20/sec; 500ms gives us plenty of margin). */ - await new Promise(r => setTimeout(r, 500)); - - /* A single legitimate request after the wait must succeed. */ - const post = await postToolCall(proxy.socketPath, '{"x":2}'); - expect(post.status).toBe(200); - } finally { - await proxy.stop(); - await upstream.close(); - } - }); -}); From 72f45a0b15c32c010a78a98f699373eba153e59f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B3n=20Levy?= Date: Mon, 12 Oct 2026 16:46:01 +0000 Subject: [PATCH 5/9] Sync upstream main: remove superseded path --- api/src/tool-call-socket-proxy.test.ts | 448 ------------------------- 1 file changed, 448 deletions(-) delete mode 100644 api/src/tool-call-socket-proxy.test.ts diff --git a/api/src/tool-call-socket-proxy.test.ts b/api/src/tool-call-socket-proxy.test.ts deleted file mode 100644 index c15cde60..00000000 --- a/api/src/tool-call-socket-proxy.test.ts +++ /dev/null @@ -1,448 +0,0 @@ -import { afterEach, describe, expect, test } from 'bun:test'; -import fs from 'node:fs'; -import http from 'node:http'; -import net from 'node:net'; -import os from 'node:os'; -import path from 'node:path'; -import { - createTokenBucket, - defaultConnectionRateBurst, - defaultConnectionRateRefillPerSec, - startToolCallSocketProxy, - type ToolCallSocketProxyHandle, -} from './tool-call-socket-proxy'; - -/* PTC-presence headers required by the proxy's filter so legitimate - * /tool-call requests forward instead of getting the canonical 404. The - * proxy only checks PRESENCE; cryptographic validation happens upstream. */ -const PTC_HEADERS = { - 'X-Execution-ID': 'test-exec-id', - 'X-Tool-Call-ID': 'test-call-001', - 'X-Callback-Token': 'test-callback-token', -} as const; - -const handles: ToolCallSocketProxyHandle[] = []; -const tempDirs: string[] = []; - -function makeSocketPath(): string { - const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'tcs-proxy-test-')); - tempDirs.push(dir); - return path.join(dir, 'tcs.sock'); -} - -function wait(ms: number): Promise { - return new Promise(resolve => setTimeout(resolve, ms)); -} - -function createRawSocket(socketPath: string): Promise { - return new Promise((resolve, reject) => { - const socket = net.createConnection(socketPath); - socket.once('connect', () => resolve(socket)); - socket.once('error', reject); - }); -} - -async function startUpstream(): Promise<{ url: string; close: () => Promise }> { - const server = http.createServer((req, res) => { - if (req.method !== 'POST' || req.url !== '/tool-call') { - res.writeHead(404).end(); - return; - } - const chunks: Buffer[] = []; - req.on('data', chunk => chunks.push(Buffer.from(chunk))); - req.on('end', () => { - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ - success: true, - body: Buffer.concat(chunks).toString('utf8'), - })); - }); - }); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const address = server.address(); - if (address == null || typeof address === 'string') { - throw new Error('unexpected upstream address'); - } - return { - url: `http://127.0.0.1:${address.port}`, - close: () => new Promise(resolve => server.close(() => resolve())), - }; -} - -afterEach(async () => { - while (handles.length > 0) { - await handles.pop()!.close(); - } - for (const dir of tempDirs.splice(0)) { - fs.rmSync(dir, { recursive: true, force: true }); - } -}); - -describe('tool-call socket proxy', () => { - test('forwards valid tool-call requests with Connection: close', async () => { - const upstream = await startUpstream(); - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: upstream.url, - idleSocketTimeoutMs: 100, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const response = await new Promise<{ status: number; body: string; closeHeader: unknown }>((resolve, reject) => { - const req = http.request({ - socketPath, - method: 'POST', - path: '/tool-call', - headers: { - 'Content-Type': 'application/json', - Connection: 'keep-alive', - ...PTC_HEADERS, - }, - }, res => { - const chunks: Buffer[] = []; - res.on('data', chunk => chunks.push(Buffer.from(chunk))); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString('utf8'), - closeHeader: res.headers.connection, - })); - }); - req.on('error', reject); - req.end('{"tool_name":"safe","input":{}}'); - }); - - expect(response.status).toBe(200); - expect(response.closeHeader).toBe('close'); - expect(JSON.parse(response.body).body).toBe('{"tool_name":"safe","input":{}}'); - await upstream.close(); - }); - - test('destroys idle raw connections so socket probes cannot pin FDs', async () => { - const upstream = await startUpstream(); - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: upstream.url, - idleSocketTimeoutMs: 50, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const sockets = await Promise.all( - Array.from({ length: 8 }, () => createRawSocket(socketPath)), - ); - - await wait(250); - - expect(proxy.activeConnections()).toBe(0); - for (const socket of sockets) { - socket.destroy(); - } - await upstream.close(); - }); - - test('keeps socket connectable for per-job UID sandboxes', async () => { - const upstream = await startUpstream(); - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: upstream.url, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - expect(fs.statSync(socketPath).mode & 0o777).toBe(0o666); - await upstream.close(); - }); - - test('caps active connections and drops excess clients', async () => { - const upstream = await startUpstream(); - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: upstream.url, - maxConnections: 2, - idleSocketTimeoutMs: 500, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const sockets = await Promise.allSettled( - Array.from({ length: 6 }, () => createRawSocket(socketPath)), - ); - - await wait(50); - - expect(sockets.filter(result => result.status === 'fulfilled').length).toBeGreaterThanOrEqual(2); - expect(proxy.activeConnections()).toBeLessThanOrEqual(2); - for (const result of sockets) { - if (result.status === 'fulfilled') result.value.destroy(); - } - await upstream.close(); - }); - - test('caps active tool-call requests separately from raw socket connections', async () => { - let releaseUpstream!: () => void; - let upstreamCalls = 0; - const upstreamReleased = new Promise(resolve => { - releaseUpstream = resolve; - }); - const server = http.createServer(async (req, res) => { - upstreamCalls += 1; - req.resume(); - await new Promise(resolve => req.on('end', resolve)); - await upstreamReleased; - res.writeHead(200, { 'Content-Type': 'application/json' }); - res.end(JSON.stringify({ success: true })); - }); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const address = server.address(); - if (address == null || typeof address === 'string') throw new Error('bad address'); - - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: `http://127.0.0.1:${address.port}`, - maxConnections: 8, - maxActiveRequests: 1, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const first = new Promise((resolve, reject) => { - const req = http.request({ socketPath, method: 'POST', path: '/tool-call', headers: PTC_HEADERS }, res => { - res.on('end', () => resolve(res.statusCode ?? 0)); - res.resume(); - }); - req.on('error', reject); - req.end('{"tool_name":"slow","input":{}}'); - }); - - while (proxy.activeRequests() < 1) { - await wait(5); - } - - const second = await new Promise<{ status: number; body: string; retryAfter: unknown }>((resolve, reject) => { - const req = http.request({ socketPath, method: 'POST', path: '/tool-call', headers: PTC_HEADERS }, res => { - const chunks: Buffer[] = []; - res.on('data', chunk => chunks.push(Buffer.from(chunk))); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString('utf8'), - retryAfter: res.headers['retry-after'], - })); - }); - req.on('error', reject); - req.end('{"tool_name":"second","input":{}}'); - }); - - expect(second.status).toBe(429); - expect(second.retryAfter).toBe('1'); - expect(JSON.parse(second.body)).toEqual({ - success: false, - error: 'Too many concurrent tool-call requests', - }); - expect(upstreamCalls).toBe(1); - - releaseUpstream(); - expect(await first).toBe(200); - await new Promise(resolve => server.close(() => resolve())); - }); - - test('expires active tool-call requests instead of holding proxy FDs indefinitely', async () => { - let upstreamCalls = 0; - let resolveUpstreamSeen!: () => void; - const upstreamSeen = new Promise(resolve => { - resolveUpstreamSeen = resolve; - }); - const server = http.createServer((req, res) => { - upstreamCalls += 1; - req.resume(); - req.on('end', resolveUpstreamSeen); - // Intentionally never respond; the proxy must time out and release its - // active request/upstream budgets. - void res; - }); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const address = server.address(); - if (address == null || typeof address === 'string') throw new Error('bad address'); - - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: `http://127.0.0.1:${address.port}`, - maxActiveRequests: 1, - activeRequestTimeoutMs: 50, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const client = await createRawSocket(socketPath); - const body = '{"tool_name":"slow","input":{}}'; - client.end([ - 'POST /tool-call HTTP/1.1', - 'Host: localhost', - 'Content-Type: application/json', - `Content-Length: ${Buffer.byteLength(body)}`, - 'Connection: close', - 'X-Execution-ID: t', - 'X-Tool-Call-ID: t', - 'X-Callback-Token: t', - '', - body, - ].join('\r\n')); - - await upstreamSeen; - expect(upstreamCalls).toBe(1); - expect(proxy.activeRequests()).toBe(1); - expect(proxy.activeUpstreams()).toBe(1); - - const deadline = Date.now() + 500; - while (proxy.activeUpstreams() > 0 && Date.now() < deadline) { - await wait(5); - } - - expect(proxy.activeRequests()).toBe(0); - expect(proxy.activeUpstreams()).toBe(0); - client.destroy(); - await new Promise(resolve => server.close(() => resolve())); - }); - - test('rejects oversized request bodies before proxying', async () => { - let upstreamCalls = 0; - const server = http.createServer((_req, res) => { - upstreamCalls += 1; - res.writeHead(200).end('unexpected'); - }); - await new Promise(resolve => server.listen(0, '127.0.0.1', resolve)); - const address = server.address(); - if (address == null || typeof address === 'string') throw new Error('bad address'); - - const socketPath = makeSocketPath(); - const proxy = await startToolCallSocketProxy({ - socketPath, - rawTarget: `http://127.0.0.1:${address.port}`, - maxBodyBytes: 4, - log: { log() {}, warn() {}, error() {} }, - }); - handles.push(proxy); - - const response = await new Promise<{ status: number; body: string }>((resolve, reject) => { - const req = http.request({ socketPath, method: 'POST', path: '/tool-call', headers: PTC_HEADERS }, res => { - const chunks: Buffer[] = []; - res.on('data', chunk => chunks.push(Buffer.from(chunk))); - res.on('end', () => resolve({ - status: res.statusCode ?? 0, - body: Buffer.concat(chunks).toString('utf8'), - })); - }); - req.on('error', reject); - req.end('too large'); - }); - - expect(response.status).toBe(413); - expect(response.body).toBe('request body too large'); - expect(upstreamCalls).toBe(0); - await new Promise(resolve => server.close(() => resolve())); - }); - - /* Integration tests for the connection-rate limiter live in - * tool-call-socket-proxy-runtime.test.ts. Bun's node:http compat layer - * does not fire `'connection'` events, so the in-process proxy here - * never invokes the rate-limiter code path under `bun test` — the - * checks would pass vacuously. The runtime tests spawn the proxy as a - * real Node subprocess, matching the production runtime. The pure - * createTokenBucket logic is fully unit-tested below and runs fine - * under either runtime. */ -}); - -describe('createTokenBucket', () => { - test('starts full at burst capacity', () => { - const b = createTokenBucket({ burst: 10, refillPerSec: 1 }); - expect(b.tokens()).toBeCloseTo(10); - }); - - test('returns true until the bucket is drained, then false', () => { - const b = createTokenBucket({ burst: 3, refillPerSec: 0, now: () => 0 }); - expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(false); - expect(b.tryConsume()).toBe(false); - }); - - test('refills at refillPerSec without exceeding the burst cap', () => { - let t = 1000; - const b = createTokenBucket({ burst: 10, refillPerSec: 100, now: () => t }); - /* Drain. */ - for (let i = 0; i < 10; i++) expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(false); - /* Advance 50ms — 100/sec * 0.050 = 5 tokens. */ - t += 50; - for (let i = 0; i < 5; i++) expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(false); - /* Advance way more than burst's worth — capacity caps at 10, not - * unbounded growth. */ - t += 10_000; - expect(b.tokens()).toBeCloseTo(10); - }); - - test('refillPerSec=0 freezes the bucket once drained (test-determinism guarantee)', () => { - const b = createTokenBucket({ burst: 2, refillPerSec: 0, now: () => 0 }); - expect(b.tryConsume()).toBe(true); - expect(b.tryConsume()).toBe(true); - /* No matter how much time we'd advance with a real clock, refill of - * 0/sec adds zero tokens. */ - expect(b.tryConsume()).toBe(false); - expect(b.tokens()).toBe(0); - }); -}); - -describe('connection-rate defaults scale with SANDBOX_MAX_CONCURRENT_JOBS', () => { - /* Codex P2 follow-up on PR #1652: the previous fixed 20-accept/sec - * refill would throttle legitimate sustained traffic on deployments - * where maxConcurrentJobs is scaled up (e.g. 64), because the proxy's - * other defaults (maxConnections, maxActiveRequests) already scale - * from SANDBOX_MAX_CONCURRENT_JOBS but the rate limiter did not. */ - const originalEnv = process.env.SANDBOX_MAX_CONCURRENT_JOBS; - - afterEach(() => { - if (originalEnv === undefined) { - delete process.env.SANDBOX_MAX_CONCURRENT_JOBS; - } else { - process.env.SANDBOX_MAX_CONCURRENT_JOBS = originalEnv; - } - }); - - test('unset env keeps the fixed 64/20 defaults', () => { - delete process.env.SANDBOX_MAX_CONCURRENT_JOBS; - expect(defaultConnectionRateBurst()).toBe(64); - expect(defaultConnectionRateRefillPerSec()).toBe(20); - }); - - test('small concurrency floors at the baseline (never tighter than the fixed defaults)', () => { - process.env.SANDBOX_MAX_CONCURRENT_JOBS = '8'; - /* 8 * 4 = 32, but floor at 64 baseline. */ - expect(defaultConnectionRateBurst()).toBe(64); - /* 8 * 2 = 16, but floor at 20 baseline. */ - expect(defaultConnectionRateRefillPerSec()).toBe(20); - }); - - test('mid concurrency scales burst proportionally', () => { - process.env.SANDBOX_MAX_CONCURRENT_JOBS = '32'; - /* 32 * 4 = 128 (above 64 floor, below 256 ceiling). */ - expect(defaultConnectionRateBurst()).toBe(128); - /* 32 * 2 = 64 (above 20 floor, below 200 ceiling). */ - expect(defaultConnectionRateRefillPerSec()).toBe(64); - }); - - test('high concurrency caps at the ceiling — bounded resource use even with extreme config', () => { - process.env.SANDBOX_MAX_CONCURRENT_JOBS = '256'; - /* 256 * 4 = 1024, capped at 256 ceiling. */ - expect(defaultConnectionRateBurst()).toBe(256); - /* 256 * 2 = 512, capped at 200 ceiling. */ - expect(defaultConnectionRateRefillPerSec()).toBe(200); - }); -}); From 7f28db1cefd046a1ad0ce9cccc7e590c48f7c044 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B3n=20Levy?= Date: Mon, 12 Oct 2026 16:46:04 +0000 Subject: [PATCH 6/9] Sync upstream main: remove superseded path --- api/src/tool-call-socket-proxy.ts | 669 ------------------------------ 1 file changed, 669 deletions(-) delete mode 100644 api/src/tool-call-socket-proxy.ts diff --git a/api/src/tool-call-socket-proxy.ts b/api/src/tool-call-socket-proxy.ts deleted file mode 100644 index 78e689b7..00000000 --- a/api/src/tool-call-socket-proxy.ts +++ /dev/null @@ -1,669 +0,0 @@ -import fs from 'node:fs'; -import http from 'node:http'; -import https from 'node:https'; -import type net from 'node:net'; - -export interface ToolCallSocketProxyOptions { - socketPath: string; - rawTarget: string; - maxConnections?: number; - maxActiveRequests?: number; - idleSocketTimeoutMs?: number; - headerTimeoutMs?: number; - requestBodyTimeoutMs?: number; - activeRequestTimeoutMs?: number; - maxBodyBytes?: number; - listenBacklog?: number; - /** Token-bucket capacity for the connection-rate limiter. The bucket - * starts full and refills at `connectionRateRefillPerSec`. Burst of - * legitimate concurrent tool calls (up to maxActiveRequests=16 by - * default) easily fits; a 500-connect()-in-50ms flood drains the - * bucket and subsequent connections are dropped at the application - * layer instead of allocating kernel unix_sock slab entries. */ - connectionRateBurst?: number; - /** Sustained connection-acceptance rate in connections/second after the - * burst bucket is drained. The SDK opens one connection per tool call; - * even an aggressive agent rarely exceeds a few /sec. */ - connectionRateRefillPerSec?: number; - socketUid?: number; - socketGid?: number; - socketMode?: number; - log?: Pick; -} - -export interface ToolCallSocketProxyHandle { - server: http.Server; - close: () => Promise; - activeConnections: () => number; - activeRequests: () => number; - activeUpstreams: () => number; - /** For tests: current available tokens in the connection-rate bucket. */ - connectionRateTokens: () => number; - /** For tests + observability: cumulative count of connections dropped - * by the connection-rate limiter since proxy start. */ - connectionRateDropped: () => number; -} - -const DEFAULT_MAX_CONNECTIONS = 64; -const DEFAULT_MAX_ACTIVE_REQUESTS = 16; -const DEFAULT_IDLE_SOCKET_TIMEOUT_MS = 2_000; -const DEFAULT_REQUEST_BODY_TIMEOUT_MS = 5_000; -const DEFAULT_ACTIVE_REQUEST_TIMEOUT_MS = 30_000; -const DEFAULT_MAX_BODY_BYTES = 1_048_576; -const DEFAULT_LISTEN_BACKLOG = 16; -const MAX_DEFAULT_CONNECTIONS = 256; -const MAX_DEFAULT_ACTIVE_REQUESTS = 64; -const ACTIVE_REQUEST_TIMEOUT_GRACE_MS = 5_000; -const MAX_DEFAULT_ACTIVE_REQUEST_TIMEOUT_MS = 60_000; -const DEFAULT_CONNECTION_RATE_BURST = 64; -const DEFAULT_CONNECTION_RATE_REFILL_PER_SEC = 20; -const MAX_DEFAULT_CONNECTION_RATE_BURST = 256; -const MAX_DEFAULT_CONNECTION_RATE_REFILL_PER_SEC = 200; - -/** Monotonic-ish token bucket. Tokens refill continuously at a configured - * rate up to a burst cap. `tryConsume()` returns false instead of waiting - * when empty — callers drop the connection synchronously, which is the - * right behavior for a SOCK_STREAM accept handler (queuing in JS would - * still hold the kernel socket struct alive, defeating the point). */ -export interface TokenBucket { - tryConsume(): boolean; - tokens(): number; -} - -export function createTokenBucket(opts: { - burst: number; - refillPerSec: number; - now?: () => number; -}): TokenBucket { - const now = opts.now ?? Date.now; - const refillPerMs = opts.refillPerSec / 1000; - let tokens = opts.burst; - let last = now(); - - function refill(): void { - const t = now(); - const elapsed = t - last; - if (elapsed <= 0) return; - tokens = Math.min(opts.burst, tokens + elapsed * refillPerMs); - last = t; - } - - return { - tryConsume(): boolean { - refill(); - if (tokens >= 1) { - tokens -= 1; - return true; - } - return false; - }, - tokens(): number { - refill(); - return tokens; - }, - }; -} - -function normalizeTarget(rawTarget: string): URL { - if (!rawTarget) { - throw new Error('SANDBOX_FORWARD_TARGET is required'); - } - return new URL(rawTarget.includes('://') ? rawTarget : `http://${rawTarget}`); -} - -function destroySocket(socket: net.Socket): void { - socket.destroy(); -} - -/* RFC 7230 §6.1 hop-by-hop headers. A proxy MUST NOT forward these to the - * upstream — they describe the proxy<->client connection, not the request. - * `host` is rewritten separately to point at the upstream. Forwarding the - * client's `host` would let a malicious sandbox steer the upstream's - * routing if it ever virtual-hosts. */ -const HOP_BY_HOP_HEADERS: ReadonlySet = new Set([ - 'connection', - 'keep-alive', - 'proxy-authenticate', - 'proxy-authorization', - 'proxy-connection', - 'te', - 'trailer', - 'transfer-encoding', - 'upgrade', - 'host', -]); - -function buildForwardedHeaders( - reqHeaders: http.IncomingHttpHeaders, - upstreamHost: string, -): http.OutgoingHttpHeaders { - const out: http.OutgoingHttpHeaders = {}; - for (const [k, v] of Object.entries(reqHeaders)) { - if (HOP_BY_HOP_HEADERS.has(k.toLowerCase())) continue; - if (v == null) continue; - out[k] = v; - } - /* Set our own connection-management headers; the proxy never keep-alives - * to upstream because each request is one-shot. */ - out.host = upstreamHost; - out.connection = 'close'; - return out; -} - -/* True only when the header has at least one non-empty, non-whitespace - * character. Naive `!headers[name]` is bypassable: Node joins duplicate - * request headers with `, ` (per RFC 7230 §3.2.2), so two empty - * `X-Foo:` lines arrive as the truthy string `", "`. An attacker who - * sends each PTC header twice with empty values would slip past the - * presence filter and reach the upstream — defeating the route-opacity - * goal. Stripping commas + whitespace before the empty check closes - * that gap. */ -function hasPtcHeaderValue(value: string | string[] | undefined): boolean { - if (value == null) return false; - const joined = Array.isArray(value) ? value.join(',') : value; - /* Strip every comma (Node's separator for joined duplicates) and every - * whitespace character. If anything is left, the header carries real - * content. */ - return joined.replace(/[,\s]/g, '') !== ''; -} - -/* Request smuggling defense (CVE class). When a request carries BOTH - * Transfer-Encoding and Content-Length, an upstream may pick the - * different one than the proxy did, letting an attacker prepend bytes - * to a synthesized "second request" that bypasses the proxy's path - * filter. The fix per RFC 7230 §3.3.3 is to reject the request rather - * than try to reconcile. */ -function isSmugglingShaped(headers: http.IncomingHttpHeaders): boolean { - const hasCL = headers['content-length'] != null; - const hasTE = headers['transfer-encoding'] != null; - return hasCL && hasTE; -} - -function parsePositiveInt(raw: string | undefined): number | undefined { - if (raw == null) return undefined; - const n = Number(raw); - if (!Number.isFinite(n) || n < 1) return undefined; - return Math.floor(n); -} - -function defaultMaxConnections(): number { - const maxConcurrentJobs = parsePositiveInt(process.env.SANDBOX_MAX_CONCURRENT_JOBS); - if (maxConcurrentJobs == null) return DEFAULT_MAX_CONNECTIONS; - return Math.min(Math.max(maxConcurrentJobs * 4, DEFAULT_MAX_CONNECTIONS), MAX_DEFAULT_CONNECTIONS); -} - -function defaultMaxActiveRequests(): number { - const maxConcurrentJobs = parsePositiveInt(process.env.SANDBOX_MAX_CONCURRENT_JOBS); - if (maxConcurrentJobs == null) return DEFAULT_MAX_ACTIVE_REQUESTS; - return Math.min(Math.max(maxConcurrentJobs, DEFAULT_MAX_ACTIVE_REQUESTS), MAX_DEFAULT_ACTIVE_REQUESTS); -} - -function defaultActiveRequestTimeoutMs(): number { - const runTimeoutMs = parsePositiveInt(process.env.SANDBOX_RUN_TIMEOUT); - if (runTimeoutMs == null) return DEFAULT_ACTIVE_REQUEST_TIMEOUT_MS; - return Math.min( - Math.max(runTimeoutMs + ACTIVE_REQUEST_TIMEOUT_GRACE_MS, DEFAULT_REQUEST_BODY_TIMEOUT_MS), - MAX_DEFAULT_ACTIVE_REQUEST_TIMEOUT_MS, - ); -} - -/* Rate-limit defaults scale with SANDBOX_MAX_CONCURRENT_JOBS in the same - * shape as defaultMaxConnections / defaultMaxActiveRequests above. A - * deployment with maxConcurrentJobs=64 is sized to run ~64 tool calls - * in parallel; capping the connection-acceptance rate at the fixed 20/sec - * baseline would throttle legitimate sustained traffic from that - * workload — connections that would have been 429'd by maxActiveRequests - * instead get destroyed at the rate limiter, which is the wrong shape - * (clients see ECONNRESET instead of a structured Retry-After response). */ -export function defaultConnectionRateBurst(): number { - const maxConcurrentJobs = parsePositiveInt(process.env.SANDBOX_MAX_CONCURRENT_JOBS); - if (maxConcurrentJobs == null) return DEFAULT_CONNECTION_RATE_BURST; - return Math.min( - Math.max(maxConcurrentJobs * 4, DEFAULT_CONNECTION_RATE_BURST), - MAX_DEFAULT_CONNECTION_RATE_BURST, - ); -} - -export function defaultConnectionRateRefillPerSec(): number { - const maxConcurrentJobs = parsePositiveInt(process.env.SANDBOX_MAX_CONCURRENT_JOBS); - if (maxConcurrentJobs == null) return DEFAULT_CONNECTION_RATE_REFILL_PER_SEC; - /* 2x maxConcurrentJobs gives headroom over the steady-state rate - * (~1 connection per active job per second for typical tool-call - * workloads). Floor at 20/sec so small deployments don't get a - * tighter limit than the fixed baseline. */ - return Math.min( - Math.max(maxConcurrentJobs * 2, DEFAULT_CONNECTION_RATE_REFILL_PER_SEC), - MAX_DEFAULT_CONNECTION_RATE_REFILL_PER_SEC, - ); -} - -export async function startToolCallSocketProxy( - opts: ToolCallSocketProxyOptions, -): Promise { - const log = opts.log ?? console; - const socketPath = opts.socketPath; - const target = normalizeTarget(opts.rawTarget); - const transport = target.protocol === 'https:' ? https : http; - const maxConnections = opts.maxConnections ?? defaultMaxConnections(); - const maxActiveRequests = opts.maxActiveRequests ?? defaultMaxActiveRequests(); - const idleSocketTimeoutMs = opts.idleSocketTimeoutMs ?? DEFAULT_IDLE_SOCKET_TIMEOUT_MS; - const headerTimeoutMs = opts.headerTimeoutMs ?? idleSocketTimeoutMs; - const requestBodyTimeoutMs = opts.requestBodyTimeoutMs ?? DEFAULT_REQUEST_BODY_TIMEOUT_MS; - const activeRequestTimeoutMs = opts.activeRequestTimeoutMs ?? defaultActiveRequestTimeoutMs(); - const maxBodyBytes = opts.maxBodyBytes ?? DEFAULT_MAX_BODY_BYTES; - const listenBacklog = opts.listenBacklog ?? DEFAULT_LISTEN_BACKLOG; - const connectionRateBurst = opts.connectionRateBurst ?? defaultConnectionRateBurst(); - const connectionRateRefillPerSec = opts.connectionRateRefillPerSec ?? defaultConnectionRateRefillPerSec(); - // The socket is intentionally connectable by sandbox jobs even when - // SANDBOX_PER_JOB_UIDS maps each job to a distinct outside UID. Abuse is - // bounded by the proxy's connection caps/timeouts, not by inode ownership. - const socketMode = opts.socketMode ?? 0o666; - - const activeSockets = new Set(); - /* Absolute accept-to-headers deadlines cover slow-header clients. Node's - * headersTimeout and socket idle timers are not a short per-socket deadline - * for AF_UNIX header drip attacks, so track the accepted socket ourselves - * until node:http has parsed enough bytes to dispatch a request. */ - const headerReceiveDeadlines = new WeakMap>(); - let activeRequests = 0; - let activeUpstreams = 0; - - const clearHeaderReceiveDeadline = (socket: net.Socket): void => { - const deadline = headerReceiveDeadlines.get(socket); - if (deadline) clearTimeout(deadline); - headerReceiveDeadlines.delete(socket); - }; - - /* Connection-rate limiter — counters the audit pattern of 500 AF_UNIX - * connect()s in <50 ms. Without this, even though the proxy itself - * stays bounded (maxConnections destroys excess accepts), each accept - * still allocates a unix_sock + dentry + skb queue in the in-VM - * kernel slab. On a memory-tight microVM, 500+ allocations in a - * single tick pressures the slab allocator hard enough that - * subsequent legitimate openat() / mmap() calls (e.g. ld.so loading - * shared libraries) sporadically fail with EMFILE — even though the - * fd table itself is at <1% utilization. Capping the *rate* of - * accepts at the application layer bounds how fast new slab entries - * can be created, giving the kernel time to reclaim. */ - const connectionRateLimiter = createTokenBucket({ - burst: connectionRateBurst, - refillPerSec: connectionRateRefillPerSec, - }); - let connectionRateDroppedCount = 0; - - const server = http.createServer((req, res) => { - const socket = req.socket; - clearHeaderReceiveDeadline(socket); - socket.setTimeout(requestBodyTimeoutMs, () => destroySocket(socket)); - req.setTimeout(requestBodyTimeoutMs, () => destroySocket(socket)); - res.setHeader('Connection', 'close'); - - res.on('finish', () => { - socket.end(); - }); - - if (req.method !== 'POST' || req.url !== '/tool-call') { - req.resume(); - res.writeHead(404, { 'Content-Type': 'text/plain', Connection: 'close' }); - res.end('not found'); - return; - } - - /* PTC-header presence filter at the proxy boundary. Three reasons: - * - * (1) Route opacity. If a sandbox attacker probes /tool-call without - * the SDK-supplied PTC headers, the upstream's 404 leaks Express's - * framing (ETag/X-Request-ID/charset/content-length) which is - * visibly different from the proxy's own 404 for unknown paths. - * Filtering at the proxy means missing-header probes never reach - * the upstream and the response is byte-identical to /any/unknown. - * - * (2) Cheap rejection. Don't burn an active-request slot or open an - * upstream connection for a request that's going to be rejected. - * - * (3) Preserves legitimate 404s. The previous shape blanket-masked - * ALL upstream 404 responses, which broke "Session not found" and - * other structured upstream errors that the SDK's preamble parses - * as JSON (json.loads("not found") raises). With the filter - * here, only missing-header probes are masked; a genuine 404 with - * a JSON body for an authenticated request still reaches the - * SDK intact. - * - * The proxy only checks PRESENCE — actual cryptographic validation - * happens at the upstream. An attacker supplying garbage values for - * these headers gets forwarded and the upstream rejects with its - * structured response, but they've already revealed sandbox-side - * activity by sending non-empty headers, so route opacity isn't the - * goal there. */ - if ( - !hasPtcHeaderValue(req.headers['x-execution-id']) - || !hasPtcHeaderValue(req.headers['x-tool-call-id']) - || !hasPtcHeaderValue(req.headers['x-callback-token']) - ) { - req.resume(); - res.writeHead(404, { 'Content-Type': 'text/plain', Connection: 'close' }); - res.end('not found'); - return; - } - - /* Smuggling defense — block before we do anything else with the - * request so neither our active-request budget nor any upstream - * connection is consumed. */ - if (isSmugglingShaped(req.headers)) { - req.resume(); - res.writeHead(400, { 'Content-Type': 'text/plain', Connection: 'close' }); - res.end('ambiguous Content-Length and Transfer-Encoding'); - return; - } - - /* Content-Length precheck — if the client honestly declares a body - * larger than maxBodyBytes, reject BEFORE opening an upstream and - * BEFORE leaking any of those bytes through req.pipe(upstream). The - * runtime byte counter below is the second line of defense for - * chunked uploads and dishonest CL values, but this gate handles - * the common case cleanly. */ - const declaredCL = Number(req.headers['content-length'] ?? '0'); - if (Number.isFinite(declaredCL) && declaredCL > maxBodyBytes) { - req.resume(); - res.writeHead(413, { 'Content-Type': 'text/plain', Connection: 'close' }); - res.end('request body too large'); - return; - } - - if (activeRequests >= maxActiveRequests) { - req.resume(); - res.writeHead(429, { - 'Content-Type': 'application/json', - Connection: 'close', - 'Retry-After': '1', - }); - res.end(JSON.stringify({ - success: false, - error: 'Too many concurrent tool-call requests', - })); - return; - } - - activeRequests += 1; - let releasedActiveRequest = false; - const releaseActiveRequest = (): void => { - if (releasedActiveRequest) return; - releasedActiveRequest = true; - activeRequests = Math.max(0, activeRequests - 1); - }; - res.on('finish', releaseActiveRequest); - res.on('close', releaseActiveRequest); - - let bodyBytes = 0; - let rejected = false; - let upstreamClosed = false; - - /* Absolute body-upload deadline. socket.setTimeout / req.setTimeout - * are idle timers and reset on every byte, so a malicious client - * dripping bytes just under the idle threshold (slow-loris) bypasses - * them. Node's `server.requestTimeout` was meant to bound this but - * empirically does NOT fire mid-body for unix-socket clients in - * Node 22 (probed; drip kept flowing past 3x the configured value). - * This explicit setTimeout fires unconditionally `requestBodyTimeoutMs` - * after the request handler runs and is cleared once the body is - * fully received. */ - const bodyUploadDeadline = setTimeout(() => { - if (req.complete || rejected) return; - destroySocket(socket); - }, requestBodyTimeoutMs); - const clearBodyUploadDeadline = (): void => clearTimeout(bodyUploadDeadline); - req.on('end', clearBodyUploadDeadline); - req.on('aborted', clearBodyUploadDeadline); - res.on('finish', clearBodyUploadDeadline); - res.on('close', clearBodyUploadDeadline); - - req.on('end', () => { - socket.setTimeout(activeRequestTimeoutMs, () => destroySocket(socket)); - }); - - /* BUFFER then forward — do NOT open upstream or pipe bytes until the - * full body is received. Streaming would let a slow-loris that drips - * body bytes pin an UPSTREAM connection slot for the entire body- - * upload window, even though the proxy itself bounds its own socket - * lifetime. Tool-call payloads are small JSON (capped at maxBodyBytes, - * default 1 MiB), so buffering is cheap and the security property - * "upstream never sees a partial request" is much stronger. */ - const bodyChunks: Buffer[] = []; - let upstream: http.ClientRequest | undefined; - const abortUpstream = (): void => { - if (!upstream || upstreamClosed || res.writableEnded) return; - upstream.destroy(new Error('tool-call client disconnected')); - }; - req.on('aborted', abortUpstream); - res.on('close', abortUpstream); - socket.on('close', abortUpstream); - - req.on('data', chunk => { - if (rejected) return; - bodyBytes += chunk.length; - if (bodyBytes > maxBodyBytes) { - rejected = true; - res.writeHead(413, { 'Content-Type': 'text/plain', Connection: 'close' }); - res.end('request body too large'); - destroySocket(socket); - return; - } - bodyChunks.push(Buffer.from(chunk)); - }); - - req.on('end', () => { - if (rejected) return; - const body = bodyChunks.length === 1 ? bodyChunks[0] : Buffer.concat(bodyChunks); - - /* Strip ALL hop-by-hop headers before forwarding (RFC 7230 §6.1). - * Previously we only stripped `proxy-connection`, which left - * Upgrade, TE, Trailer, and friends as a vector for protocol - * confusion at the upstream. */ - const headers = buildForwardedHeaders(req.headers, target.host); - /* Override declared content-length with the actual buffered size - * — if the client lied, what we send is what we have. */ - headers['content-length'] = body.length; - - upstream = transport.request({ - protocol: target.protocol, - hostname: target.hostname, - port: target.port || undefined, - method: 'POST', - path: '/tool-call', - headers, - }, upstreamRes => { - if (rejected) { - upstreamRes.resume(); - return; - } - /* Strip hop-by-hop on the response path. Upstream's Connection / - * Keep-Alive / Upgrade headers describe the proxy <-> upstream - * link, not what we should tell the sandbox client. */ - const respHeaders: http.OutgoingHttpHeaders = {}; - for (const [k, v] of Object.entries(upstreamRes.headers)) { - if (HOP_BY_HOP_HEADERS.has(k.toLowerCase())) continue; - if (v == null) continue; - respHeaders[k] = v; - } - respHeaders.Connection = 'close'; - res.writeHead(upstreamRes.statusCode || 502, respHeaders); - upstreamRes.pipe(res); - }); - activeUpstreams += 1; - - upstream.on('close', () => { - upstreamClosed = true; - activeUpstreams = Math.max(0, activeUpstreams - 1); - releaseActiveRequest(); - }); - - upstream.setTimeout(activeRequestTimeoutMs, () => { - upstream?.destroy(new Error('tool-call upstream timeout')); - }); - - upstream.on('error', error => { - if (rejected) return; - log.error('tool-call socket proxy upstream error', error); - if (!res.headersSent) { - res.writeHead(502, { 'Content-Type': 'text/plain', Connection: 'close' }); - } - res.end('bad gateway'); - }); - - upstream.end(body); - }); - }); - - server.maxConnections = maxConnections; - server.headersTimeout = headerTimeoutMs; - server.keepAliveTimeout = 1; - /* Bound the entire request reception (first byte -> last body byte) to - * the body-upload window. Prevents slow-loris drip attacks: socket-idle - * timers reset on every byte, so a malicious client could otherwise - * stretch body upload to activeRequestTimeoutMs (default 30s) by - * sending one byte just under the idle threshold. */ - server.requestTimeout = requestBodyTimeoutMs; - server.timeout = activeRequestTimeoutMs; - - server.on('connection', socket => { - /* Rate-limit BEFORE the concurrency check. The concurrency check - * (`activeSockets.size`) tells us how many connections are alive - * RIGHT NOW; the rate limiter tells us how fast new ones are being - * created. The audit's flood passes the concurrency check trivially - * (most connections accepted, processed, and destroyed faster than - * the count grows) but creates thousands of socket structs in the - * process. Rate-limiting at connect-time is what bounds the kernel - * slab pressure that surfaces as EMFILE in subsequent loader open() - * calls — see createTokenBucket comment for the full mechanism. */ - if (!connectionRateLimiter.tryConsume()) { - connectionRateDroppedCount += 1; - destroySocket(socket); - return; - } - if (activeSockets.size >= maxConnections) { - log.warn('tool-call socket proxy connection limit reached; dropping connection'); - destroySocket(socket); - return; - } - - activeSockets.add(socket); - const headerReceiveDeadline = setTimeout(() => { - headerReceiveDeadlines.delete(socket); - destroySocket(socket); - }, headerTimeoutMs); - headerReceiveDeadlines.set(socket, headerReceiveDeadline); - - socket.setTimeout(idleSocketTimeoutMs, () => destroySocket(socket)); - socket.on('close', () => { - clearHeaderReceiveDeadline(socket); - activeSockets.delete(socket); - }); - socket.on('error', () => { - clearHeaderReceiveDeadline(socket); - activeSockets.delete(socket); - }); - }); - - server.on('clientError', (_error, socket) => { - destroySocket(socket as net.Socket); - }); - - try { - fs.unlinkSync(socketPath); - } catch { - // Missing stale socket is fine. - } - - await new Promise((resolve, reject) => { - const onError = (error: Error): void => { - server.off('listening', onListening); - reject(error); - }; - const onListening = (): void => { - server.off('error', onError); - if (opts.socketUid != null && opts.socketGid != null) { - fs.chownSync(socketPath, opts.socketUid, opts.socketGid); - } - fs.chmodSync(socketPath, socketMode); - log.log(`tool-call socket proxy listening on ${socketPath}`); - resolve(); - }; - server.once('error', onError); - server.once('listening', onListening); - server.listen(socketPath, listenBacklog); - }); - - return { - server, - close: async () => { - for (const socket of activeSockets) { - destroySocket(socket); - } - await new Promise((resolve) => { - server.close(() => { - try { - fs.unlinkSync(socketPath); - } catch { - // Socket was already removed. - } - resolve(); - }); - }); - }, - activeConnections: () => activeSockets.size, - activeRequests: () => activeRequests, - activeUpstreams: () => activeUpstreams, - connectionRateTokens: () => connectionRateLimiter.tokens(), - connectionRateDropped: () => connectionRateDroppedCount, - }; -} - -if (require.main === module) { - const socketPath = process.env.TCS_SOCKET || '/tmp/tcs.sock'; - const rawTarget = process.env.SANDBOX_FORWARD_TARGET || ''; - const socketUid = process.env.TCS_SOCKET_UID ? Number(process.env.TCS_SOCKET_UID) : undefined; - const socketGid = process.env.TCS_SOCKET_GID ? Number(process.env.TCS_SOCKET_GID) : undefined; - const maxConnections = parsePositiveInt(process.env.TCS_MAX_CONNECTIONS); - const headerTimeoutMs = parsePositiveInt(process.env.TCS_HEADER_TIMEOUT_MS); - /* Operator overrides for the connection-rate limiter — see - * createTokenBucket comment for the kernel-slab-pressure motivation. - * Defaults (burst=64, refill=20/sec) are sized for the SDK's tool-call - * pattern (one connection per call) plus generous headroom; raise only - * if a legitimate workload trips the dropped-connection counter. */ - const rateBurst = parsePositiveInt(process.env.TCS_CONNECTION_RATE_BURST); - const rateRefillPerSec = parsePositiveInt(process.env.TCS_CONNECTION_RATE_REFILL_PER_SEC); - let handle: ToolCallSocketProxyHandle | undefined; - let shuttingDown = false; - - const shutdown = (): void => { - if (shuttingDown) return; - shuttingDown = true; - if (!handle) { - /* Signal arrived before startToolCallSocketProxy() resolved. - * `process.exit` is terminal in Node, but the explicit `return` makes - * the contract local: no `handle.close()` on `undefined` even if a - * future runtime ever defers exit (atexit hook, async-cleanup mode). */ - process.exit(0); - return; - } - void handle.close().finally(() => process.exit(0)); - }; - - process.on('SIGTERM', shutdown); - process.on('SIGINT', shutdown); - - startToolCallSocketProxy({ - socketPath, rawTarget, socketUid, socketGid, - maxConnections, - headerTimeoutMs, - connectionRateBurst: rateBurst, - connectionRateRefillPerSec: rateRefillPerSec, - }) - .then(started => { - handle = started; - }) - .catch(error => { - console.error('tool-call socket proxy failed to start', error); - process.exit(1); - }); -} From 4509d73894b1aeb0bf0222a4dce40f82c618471e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B3n=20Levy?= Date: Mon, 12 Oct 2026 16:46:06 +0000 Subject: [PATCH 7/9] Sync upstream main: remove superseded path --- tests/specguard_fd_isolation.py | 289 -------------------------------- 1 file changed, 289 deletions(-) delete mode 100644 tests/specguard_fd_isolation.py diff --git a/tests/specguard_fd_isolation.py b/tests/specguard_fd_isolation.py deleted file mode 100644 index d7aa79cf..00000000 --- a/tests/specguard_fd_isolation.py +++ /dev/null @@ -1,289 +0,0 @@ -""" -spec-guard FD isolation regression test. - -Reproduces the audit's "permanent sandbox brick" pattern: - - 1. A first /api/v2/execute opens N socket FDs against /tmp/tcs.sock - and never closes them (simulates malicious sandbox code). - 2. After the first job exits, the runner's per-job FD slot bookkeeping - is supposed to be back to baseline. - 3. A SECOND /api/v2/execute runs trivial Python and reports its - inherited FD count via /proc/self/fd. - -Without `spec-guard` closing inherited descriptors before `execvp()`, the -second job's user process starts with whatever FDs leaked through the -runner -> NsJail -> child chain. With the fix, the second job's Python -process sees only stdin/stdout/stderr (plus whatever it itself opens). - -The test asserts BOTH: - - The simulated leak in job 1 actually opens many FDs. - - The follow-up job 2 starts with a clean FD table (<= a small bound). - -Usage (local docker compose stack): - - SANDBOX_REQUIRE_EGRESS_MANIFEST=false CODEAPI_HARDENED_SANDBOX_MODE=false \\ - docker compose up -d - python3 services/codeapi/tests/specguard_fd_isolation.py -""" -from __future__ import annotations - -import json -import os -import sys -import urllib.error -import urllib.request - -SANDBOX_URL = os.environ.get("SANDBOX_URL", "http://127.0.0.1:2000") -REQUEST_TIMEOUT_S = float(os.environ.get("SPEC_GUARD_TEST_TIMEOUT_S", "30")) - -# After spec-guard runs and the user's Python interpreter starts, a tiny -# number of FDs are legitimately open: 0/1/2 plus whatever Python opened -# during startup (typically the dirfd it iterates /proc/self/fd through). -# Anything above this bound is leaked descriptors. -MAX_LEGITIMATE_INHERITED_FDS = 10 - - -def execute(language: str, code: str, timeout_ms: int = 15_000) -> dict: - body = { - "language": language, - "version": "*", - "files": [{"name": "main.py" if language == "python" else "main.sh", - "content": code}], - "stdin": "", - "args": [], - "run_timeout": timeout_ms, - } - data = json.dumps(body).encode("utf-8") - req = urllib.request.Request( - f"{SANDBOX_URL}/api/v2/execute", - data=data, - headers={"Content-Type": "application/json"}, - method="POST", - ) - with urllib.request.urlopen(req, timeout=REQUEST_TIMEOUT_S) as resp: - return json.loads(resp.read().decode("utf-8")) - - -STORM_TARGET = 500 -"""Open many more connections than the proxy's maxConnections cap (64 by -default). The kernel queues some (listen backlog) and accepts the rest as -slots free up via the proxy's idle timer. This matches the audit's -literal red-team script — `for i in range(500): connect(); leak`. Without -this volume, the test wouldn't exercise the steady-state-pressure path -the audit broke on. The storm job exits without closing any of them; the -proxy must reclaim accept-side FDs and the runner must remain serviceable.""" - -MIN_STORM_OPENED = 200 -"""Lower bound on what the storm job must successfully open before we -trust the test was meaningful. Without this assertion the test would -silently degrade — if the proxy ever started refusing all connections -at the kernel layer, the storm would open 0 sockets, the FD-poisoning -preconditions would never be exercised, and the regression would still -report PASS. 200 is comfortably above the production cap of 64 so the -kernel accept queue + proxy reaping cycle is exercised.""" - - -def fd_storm_then_report() -> None: - """First job: open many connections to /tmp/tcs.sock and never close them. - The connections live until the user process exits — but the proxy and - runner-side accounting may retain state. Spec-guard must not let any of - that leak into the NEXT job's process.""" - code = ( - "import os, socket, json\n" - # The runner bind-mounts the proxy socket at this fixed path; the - # TOOL_CALL_SOCKET env var was dropped to keep the path off - # `os.environ`, so the test references the literal path instead. - "s = '/tmp/tcs.sock'\n" - "leaked = []\n" - "errors = []\n" - # Append every socket to a list so CPython's refcount keeps the FD - # alive until process exit. Without this, each loop iteration - # rebinds `sk` and the previous socket is GC'd before the next - # connect() runs — the loop reports `opened == STORM_TARGET` but - # only ONE FD is ever actually held, so the FD-poisoning - # preconditions the test exists to exercise are never created. - f"for _ in range({STORM_TARGET}):\n" - " try:\n" - " sk = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n" - " sk.connect(s); leaked.append(sk)\n" - " except OSError as e:\n" - " errors.append(str(e)); break\n" - # Read /proc/self/fd to PROVE we're actually holding `opened` FDs - # concurrently, so the assertion downstream isn't satisfied by - # the loop counter alone. - "concurrent_fds = len([x for x in os.listdir('/proc/self/fd') if x.isdigit()])\n" - "print(json.dumps({'opened': len(leaked), 'concurrent_fds': concurrent_fds, " - "'target': " - + str(STORM_TARGET) + ", 'errors': errors[:3]}))\n" - ) - result = execute("python", code) - run = result.get("run") or {} - if run.get("code") != 0: - print(f"FAIL: storm job exited non-zero: {run!r}", file=sys.stderr) - sys.exit(1) - stdout = (run.get("stdout") or "").strip() - print(f"[storm-job] {stdout}") - last_line = stdout.splitlines()[-1] if stdout else "" - try: - payload = json.loads(last_line) - except json.JSONDecodeError: - print( - f"FAIL: storm job did not produce JSON status line; got {last_line!r}", - file=sys.stderr, - ) - sys.exit(1) - opened = int(payload.get("opened", 0)) - concurrent = int(payload.get("concurrent_fds", 0)) - if opened < MIN_STORM_OPENED: - print( - f"FAIL: storm job opened only {opened} sockets (required >= " - f"{MIN_STORM_OPENED}); the FD-poisoning preconditions were not " - f"exercised, so a downstream PASS would be vacuous. errors: " - f"{payload.get('errors')}", - file=sys.stderr, - ) - sys.exit(1) - # Defend against the "vacuous pass via socket reuse" mode — assert - # the storm process is actually HOLDING the FDs concurrently in - # /proc/self/fd, not just opening-then-closing in a loop. - if concurrent < MIN_STORM_OPENED: - print( - f"FAIL: storm job reported {opened} opens but /proc/self/fd " - f"shows only {concurrent} concurrent FDs — sockets are being " - f"GC'd between iterations and the test is vacuous.", - file=sys.stderr, - ) - sys.exit(1) - - -def report_inherited_fd_count() -> int: - """Second job: introspect /proc/self/fd. Spec-guard should have closed - everything but stdin/out/err before this Python invocation began.""" - code = ( - "import os, json\n" - "fds = sorted(int(x) for x in os.listdir('/proc/self/fd') if x.isdigit())\n" - "print(json.dumps({'fd_count': len(fds), 'fds': fds[:20]}))\n" - ) - result = execute("python", code) - run = result.get("run") or {} - if run.get("code") != 0: - print(f"FAIL: report job exited non-zero: {run!r}", file=sys.stderr) - sys.exit(1) - stdout = (run.get("stdout") or "").strip() - print(f"[report-job] {stdout}") - payload = json.loads(stdout.split("\n")[-1]) - return int(payload["fd_count"]) - - -def post_storm_recovery() -> int: - """The audit's specific failure: after a socket storm, subsequent - dynamic-binary executions (python/node/bash/bun) failed during - library-load with `Error 24` (EMFILE). Run a real exec in each - language and require code 0 — proves the dynamic loader is no - longer poisoned by inherited descriptors.""" - cases = [ - ("python", "import sys; print('python', sys.version_info[:3])"), - ("node", "console.log('node', process.version)"), - ("bash", "echo bash $(/bin/bash --version | head -1)"), - ] - failed = 0 - for lang, code in cases: - result = execute(lang, code) - run = result.get("run") or {} - out = (run.get("stdout") or "").strip() - err = (run.get("stderr") or "").strip() - ok = run.get("code") == 0 and out and not err - marker = "ok " if ok else "FAIL" - print(f" [{marker}] {lang:6} code={run.get('code')!r:>4} stdout={out!r:<50} stderr={err[:80]!r}") - if not ok: - failed += 1 - # Specifically guard against the audit's EMFILE-on-loader pattern - if "Error 24" in err or "Too many open files" in err: - print(f" ^ audit-mode regression: dynamic loader hit EMFILE") - failed += 1 - return failed - - -def ptc_route_opacity_check() -> bool: - """Probe the proxy at /tool-call (with no PTC headers) and at unknown - paths from inside the sandbox; both must produce BYTE-IDENTICAL wire - responses, otherwise an attacker can still fingerprint the real route - by reading the response framing (ETag, X-Request-ID, charset, etc.). - The original audit broke on this exact gap — the response BODY matched - but the upstream's Express-added headers leaked through the proxy.""" - code = ( - "import os, socket, json\n" - # Same fixed-path reasoning as fd_storm_then_report(). - "s = '/tmp/tcs.sock'\n" - "responses = {}\n" - "for path in ['/tool-call', '/randomroute']:\n" - " sk = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)\n" - " sk.connect(s)\n" - " sk.sendall(('POST ' + path + ' HTTP/1.1\\r\\nHost: x\\r\\n" - "Content-Length: 0\\r\\nConnection: close\\r\\n\\r\\n').encode())\n" - " buf = b''\n" - " while True:\n" - " chunk = sk.recv(4096)\n" - " if not chunk: break\n" - " buf += chunk\n" - " sk.close()\n" - " # Strip the volatile Date header before comparing\n" - " lines = buf.split(b'\\r\\n')\n" - " filtered = b'\\r\\n'.join(l for l in lines if not l.lower().startswith(b'date:'))\n" - " responses[path] = filtered.decode('utf-8', 'replace')\n" - "print(json.dumps(responses))\n" - ) - result = execute("python", code) - run = result.get("run") or {} - if run.get("code") != 0: - print(f"FAIL: opacity probe exited non-zero: {run!r}", file=sys.stderr) - return False - payload = json.loads((run.get("stdout") or "").splitlines()[-1]) - tc, rr = payload["/tool-call"], payload["/randomroute"] - if tc == rr: - print(f" [ok ] /tool-call response is byte-identical to /randomroute " - f"({len(tc)} bytes)") - return True - print(f" [FAIL] wire responses differ — sandbox can fingerprint the route:") - print(f" /tool-call: {tc!r}") - print(f" /randomroute: {rr!r}") - return False - - -def main() -> int: - print(f"=== spec-guard FD isolation regression — runner: {SANDBOX_URL} ===") - print(f"step 1: open {STORM_TARGET} socket FDs in a first job (audit's exact red-team script)") - fd_storm_then_report() - print("step 2: launch a second job and report its inherited FD count") - fd_count = report_inherited_fd_count() - print(f" -> second job sees {fd_count} FDs (bound: {MAX_LEGITIMATE_INHERITED_FDS})") - if fd_count > MAX_LEGITIMATE_INHERITED_FDS: - print( - f"FAIL: second job inherited {fd_count} FDs — " - f"spec-guard did NOT close inherited descriptors before execvp", - file=sys.stderr, - ) - return 1 - print("step 3: post-storm recovery for python / node / bash") - failed = post_storm_recovery() - if failed: - print(f"FAIL: {failed} language(s) regressed", file=sys.stderr) - return 1 - print("step 4: PTC route opacity (byte-identical 404 wire responses)") - if not ptc_route_opacity_check(): - return 1 - print("PASS: spec-guard isolated the second job's FD table, " - "all runtimes recovered post-storm, and the /tool-call route " - "is wire-indistinguishable from any unknown path") - return 0 - - -if __name__ == "__main__": - try: - sys.exit(main()) - except urllib.error.HTTPError as exc: - print(f"HTTPError {exc.code}: {exc.read()[:300]!r}", file=sys.stderr) - sys.exit(2) - except urllib.error.URLError as exc: - print(f"URLError: {exc}", file=sys.stderr) - sys.exit(2) From db0c7f87abe8a83aba0fc51540806f1fc60d261c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B3n=20Levy?= Date: Mon, 12 Oct 2026 16:51:17 +0000 Subject: [PATCH 8/9] ci: invoke kvm guest dns test through bash --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 92e96ddb..ff98d111 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -35,7 +35,7 @@ jobs: run: bash tests/package_init_runtime_rollout.sh - name: KVM guest resolver handoff - run: tests/kvm_guest_dns.sh + run: bash tests/kvm_guest_dns.sh - name: Sandbox-runner liveness checks run: tests/sandbox_runner_healthcheck.sh From 3b5965cf59bc419ec286ff7f983e8e929c1657b8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B3n=20Levy?= Date: Mon, 12 Oct 2026 16:52:13 +0000 Subject: [PATCH 9/9] docs: invoke package builder through bash --- build-packages.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build-packages.sh b/build-packages.sh index 1d635297..47b3eca6 100644 --- a/build-packages.sh +++ b/build-packages.sh @@ -5,7 +5,7 @@ # Works on both arm64 (Apple Silicon) and amd64 (Linux/WSL). # # Usage: -# ./build-packages.sh # Build all packages +# bash ./build-packages.sh # Build all packages # # Environment Variables: # PYTHON_VERSION=3.14.4 # Python version to install