From dc1a4161a70b0a414b9c4d08300d0d48519c1cfe Mon Sep 17 00:00:00 2001 From: bradleyshep Date: Mon, 31 Aug 2026 13:33:12 -0400 Subject: [PATCH 001/127] Add Stack Bench benchmarking system --- .gitattributes | 2 + .../skills/typescript-client/SKILL.md | 6 +- .../skills/typescript-server/SKILL.md | 23 +- crates/bindings-typescript/src/lib/query.ts | 26 +- .../src/sdk/connection_manager.ts | 18 +- .../tests/connection_manager_liveness.test.ts | 45 +- .../tests/table_ref_error_message.test.ts | 79 + crates/cli/build.rs | 1 + crates/cli/src/subcommands/dev.rs | 9 +- crates/cli/src/subcommands/publish.rs | 4 +- skills/spacetimedb-typescript-core/SKILL.md | 101 + skills/typescript-client/SKILL.md | 6 +- skills/typescript-server/SKILL.md | 23 +- tools/llm-sequential-upgrade/.gitignore | 2 +- tools/llm-sequential-upgrade/read-guard.sh | 42 + tools/llm-sequential-upgrade/run.sh | 16 +- tools/stack-bench/.gitignore | 13 + tools/stack-bench/APPLIANCE-DESIGN.md | 157 + tools/stack-bench/README.md | 430 ++ tools/stack-bench/SETUP.md | 141 + .../appliance/Controller.Dockerfile | 91 + .../Controller.Dockerfile.dockerignore | 30 + tools/stack-bench/appliance/README.md | 408 ++ tools/stack-bench/appliance/RECOVERY.md | 67 + tools/stack-bench/appliance/RELEASE.md | 102 + ...paign.ecommerce-progression-reference.json | 97 + .../appliance/campaign.example.json | 115 + .../campaign.product-brief-reference.json | 114 + tools/stack-bench/appliance/controller.ts | 151 + .../appliance/dependency-volume.ts | 165 + .../stack-bench/appliance/docker-compose.yaml | 114 + .../appliance/operator.env.example | 20 + .../backends/minimal/mongodb-1.5.md | 19 + .../backends/minimal/postgres-1.5.md | 19 + .../backends/minimal/spacetime-1.6.md | 28 + tools/stack-bench/backends/model-free-stub.md | 7 + tools/stack-bench/backends/mongodb.md | 64 + tools/stack-bench/backends/postgres.md | 66 + tools/stack-bench/backends/spacetime.md | 99 + tools/stack-bench/commands/agent.ts | 979 +++ .../commands/archive-transcripts.ts | 126 + tools/stack-bench/commands/bench-arguments.ts | 331 + tools/stack-bench/commands/bench.ts | 2375 +++++++ tools/stack-bench/commands/campaign-cli.ts | 273 + tools/stack-bench/commands/check-actions.ts | 142 + .../stack-bench/commands/check-calibration.ts | 69 + .../stack-bench/commands/check-composition.ts | 87 + tools/stack-bench/commands/check-mutations.ts | 92 + tools/stack-bench/commands/check-scenarios.ts | 309 + tools/stack-bench/commands/composition-cli.ts | 335 + tools/stack-bench/commands/container-smoke.ts | 199 + tools/stack-bench/commands/cost-ledger.ts | 34 + .../commands/definition-snapshots.ts | 80 + tools/stack-bench/commands/fault-injection.ts | 254 + tools/stack-bench/commands/leak-audit.ts | 274 + tools/stack-bench/commands/lease-cli.ts | 64 + tools/stack-bench/commands/null-control.ts | 246 + tools/stack-bench/commands/preflight.ts | 19 + tools/stack-bench/commands/probe-sandbox.ts | 197 + .../stack-bench/commands/progression-graph.ts | 21 + .../stack-bench/commands/qualification-cli.ts | 278 + tools/stack-bench/commands/recovery.ts | 25 + tools/stack-bench/commands/repair-cli.ts | 219 + tools/stack-bench/commands/report-bugs.ts | 333 + tools/stack-bench/commands/reset-backend.ts | 24 + tools/stack-bench/commands/run-suite.ts | 1144 ++++ tools/stack-bench/commands/stdb-report.ts | 287 + tools/stack-bench/commands/stdb-review.ts | 299 + tools/stack-bench/commands/test-loop.ts | 307 + tools/stack-bench/conditions/catalog.json | 12 + .../conditions/guidance/model-free-stub.json | 22 + .../conditions/guidance/neutral-1.7.0.json | 33 + .../conditions/guidance/prescribed.json | 28 + .../conditions/repairs/scored-only.json | 9 + tools/stack-bench/container/Dockerfile | 51 + .../container/binary-provenance.ts | 185 + .../stack-bench/container/build-linux-cli.sh | 76 + tools/stack-bench/container/container-auth.ts | 64 + .../container/credential-broker.ts | 1086 ++++ .../container/reconcile-build-container.ts | 70 + .../container/recover-build-container.ts | 62 + tools/stack-bench/container/run-build.ts | 725 +++ .../container/spacetimedb-binaries.json | 21 + tools/stack-bench/dashboard/README.md | 37 + .../stack-bench/dashboard/dashboard-model.ts | 668 ++ .../stack-bench/dashboard/dashboard-server.ts | 405 ++ tools/stack-bench/dashboard/public/app.ts | 1400 ++++ .../dashboard/public/fonts/inter-LICENSE.txt | 93 + .../public/fonts/inter-latin-variable.woff2 | Bin 0 -> 48256 bytes .../public/fonts/source-code-pro-LICENSE.txt | 93 + .../source-code-pro-latin-variable.woff2 | Bin 0 -> 22044 bytes tools/stack-bench/dashboard/public/index.html | 88 + .../dashboard/public/spacetimedb-mark.svg | 5 + tools/stack-bench/dashboard/public/styles.css | 616 ++ tools/stack-bench/docker-compose.yaml | 49 + tools/stack-bench/docs/dependency-graph.html | 1257 ++++ tools/stack-bench/docs/stack-bench.html | 1806 ++++++ .../fixtures/app-broken/index.html | 37 + .../stack-bench/fixtures/app-good/index.html | 40 + tools/stack-bench/fixtures/fault-agent.ts | 75 + tools/stack-bench/fixtures/stub-agent.ts | 43 + tools/stack-bench/grader/README.md | 136 + tools/stack-bench/grader/grade.ts | 1073 ++++ tools/stack-bench/grader/mutation-test.ts | 792 +++ .../mutations/mongodb-ecommerce-2.0.1.json | 1826 ++++++ .../mutations/postgres-ecommerce-2.0.1.json | 1686 +++++ .../mutations/spacetime-ecommerce-2.0.1.json | 1926 ++++++ tools/stack-bench/linter/fixtures/agreed.html | 4 + .../linter/fixtures/divergent.html | 12 + .../linter/fixtures/mock-chat.html | 71 + .../linter/fixtures/mock-shop.html | 297 + .../linter/fixtures/spec-accounts.html | 29 + tools/stack-bench/linter/lint.ts | 246 + tools/stack-bench/package-lock.json | 95 + tools/stack-bench/package.json | 71 + .../mongodb-mutation.json | 375 ++ .../mongodb-reference.json | 372 ++ .../null.json | 1064 +++ .../postgres-mutation.json | 375 ++ .../postgres-reference.json | 372 ++ .../prompt-material-reuse.json | 106 + .../spacetime-mutation.json | 375 ++ .../spacetime-reference.json | 372 ++ tools/stack-bench/reference-apps/README.md | 73 + .../ecommerce/mongodb/client/index.html | 12 + .../mongodb/client/package-lock.json | 1046 +++ .../ecommerce/mongodb/client/package.json | 23 + .../ecommerce/mongodb/client/src/App.tsx | 1453 +++++ .../mongodb/client/src/ProgressionPanel.tsx | 331 + .../ecommerce/mongodb/client/src/index.css | 744 +++ .../ecommerce/mongodb/client/src/main.tsx | 10 + .../ecommerce/mongodb/client/tsconfig.json | 16 + .../ecommerce/mongodb/client/vite.config.ts | 24 + .../ecommerce/mongodb/package.json | 7 + .../ecommerce/mongodb/reference.json | 8 + .../mongodb/server/package-lock.json | 2128 ++++++ .../ecommerce/mongodb/server/package.json | 29 + .../ecommerce/mongodb/server/src/index.ts | 1060 +++ .../ecommerce/mongodb/server/src/models.ts | 130 + .../mongodb/server/src/progression-models.ts | 123 + .../mongodb/server/src/progression.ts | 486 ++ .../server/src/stock-reservations.test.ts | 31 + .../mongodb/server/src/stock-reservations.ts | 38 + .../ecommerce/mongodb/server/tsconfig.json | 14 + .../ecommerce/postgres/client/index.html | 12 + .../postgres/client/package-lock.json | 1046 +++ .../ecommerce/postgres/client/package.json | 22 + .../ecommerce/postgres/client/src/App.tsx | 1389 ++++ .../postgres/client/src/ProgressionPanel.tsx | 267 + .../ecommerce/postgres/client/src/index.css | 702 ++ .../ecommerce/postgres/client/src/main.tsx | 10 + .../ecommerce/postgres/client/tsconfig.json | 16 + .../ecommerce/postgres/client/vite.config.ts | 23 + .../ecommerce/postgres/package.json | 7 + .../ecommerce/postgres/reference.json | 8 + .../postgres/server/drizzle.config.ts | 11 + .../postgres/server/package-lock.json | 3047 +++++++++ .../ecommerce/postgres/server/package.json | 30 + .../ecommerce/postgres/server/src/auth.ts | 21 + .../ecommerce/postgres/server/src/db.ts | 14 + .../ecommerce/postgres/server/src/index.ts | 1033 +++ .../postgres/server/src/progression.ts | 896 +++ .../ecommerce/postgres/server/src/schema.ts | 102 + .../ecommerce/postgres/server/src/seed.ts | 79 + .../ecommerce/postgres/server/tsconfig.json | 15 + .../backend/spacetimedb/package-lock.json | 173 + .../backend/spacetimedb/package.json | 12 + .../backend/spacetimedb/src/index.ts | 1681 +++++ .../spacetimedb/src/progression-policy.ts | 38 + .../backend/spacetimedb/src/schema.ts | 423 ++ .../backend/spacetimedb/tsconfig.json | 13 + .../ecommerce/spacetime/client/index.html | 12 + .../spacetime/client/package-lock.json | 1079 ++++ .../ecommerce/spacetime/client/package.json | 23 + .../ecommerce/spacetime/client/src/App.tsx | 577 ++ .../client/src/components/AdminPanel.tsx | 324 + .../client/src/components/AuthWidget.tsx | 138 + .../client/src/components/CartPanel.tsx | 159 + .../client/src/components/FulfilmentPanel.tsx | 58 + .../client/src/components/ItemCard.tsx | 92 + .../client/src/components/ItemDetail.tsx | 127 + .../client/src/components/OrdersPanel.tsx | 144 + .../src/components/ProgressionWorkbench.tsx | 301 + .../ecommerce/spacetime/client/src/config.ts | 7 + .../ecommerce/spacetime/client/src/index.css | 776 +++ .../ecommerce/spacetime/client/src/main.tsx | 29 + .../ecommerce/spacetime/client/src/types.ts | 88 + .../spacetime/client/src/vite-env.d.ts | 1 + .../ecommerce/spacetime/client/tsconfig.json | 17 + .../ecommerce/spacetime/client/vite.config.ts | 9 + .../ecommerce/spacetime/reference.json | 10 + .../stack-bench/reference-apps/registry.json | 86 + tools/stack-bench/reset-db.sh | 98 + tools/stack-bench/restart-backend.sh | 177 + .../scripts/copy-dashboard-assets.ts | 11 + .../stack-bench/src/actions/action-catalog.ts | 140 + .../src/actions/action-contract.ts | 405 ++ .../actor-transport-action-executors.ts | 1161 ++++ .../src/actions/browser-action-executors.ts | 680 ++ .../src/actions/element-selector.ts | 6 + .../lifecycle-concurrency-action-executors.ts | 498 ++ .../src/actions/stack-action-operations.ts | 121 + .../src/agents/agent-adapter-contract.ts | 502 ++ .../stack-bench/src/agents/agent-adapters.ts | 97 + .../src/agents/agent-credentials.ts | 43 + .../stack-bench/src/agents/agent-materials.ts | 48 + .../src/agents/claude-terminal-recovery.ts | 389 ++ .../src/agents/coding-session-recovery.ts | 412 ++ .../src/agents/coding-session-timeouts.ts | 3 + .../src/campaigns/campaign-admission.ts | 395 ++ .../src/campaigns/campaign-compiler.ts | 1049 +++ .../src/campaigns/campaign-inspection.ts | 249 + .../src/campaigns/campaign-lock.ts | 218 + .../src/campaigns/campaign-mode.ts | 121 + .../campaigns/campaign-progression-grant.ts | 323 + .../src/campaigns/campaign-report.ts | 711 ++ .../src/campaigns/campaign-run-validation.ts | 544 ++ .../src/campaigns/campaign-runner.ts | 589 ++ .../src/campaigns/campaign-runtime.ts | 92 + .../src/campaigns/campaign-scheduler.ts | 570 ++ .../src/campaigns/condition-compiler.ts | 514 ++ .../progression-reference-campaign-audit.ts | 247 + .../src/composition/agent-visible-contract.ts | 81 + .../src/composition/calibration-compiler.ts | 1358 ++++ .../src/composition/composition-compiler.ts | 1215 ++++ .../src/composition/credential-aliases.ts | 60 + .../src/composition/definition-compiler.ts | 586 ++ .../src/composition/definition-plan.ts | 94 + .../src/composition/pack-budget.ts | 402 ++ .../src/composition/pack-runtime.ts | 184 + .../src/composition/product-config.ts | 51 + .../src/composition/qualification-scope.ts | 312 + .../src/composition/recipe-release.ts | 963 +++ .../src/composition/recipe-selection.ts | 770 +++ tools/stack-bench/src/composition/tracks.ts | 285 + tools/stack-bench/src/evidence/artifacts.ts | 1118 ++++ .../stack-bench/src/evidence/benchmark-run.ts | 341 + .../src/evidence/browser-evidence.ts | 24 + .../src/evidence/check-evidence.ts | 328 + .../src/evidence/claude-usage-cost.ts | 68 + tools/stack-bench/src/evidence/cost-proof.ts | 125 + .../src/evidence/diagnostic-sanitizer.ts | 123 + .../src/evidence/evidence-presentation.ts | 31 + .../src/evidence/evidence-timing.ts | 6 + .../stack-bench/src/evidence/grade-report.ts | 52 + .../src/evidence/harness-errors.ts | 62 + .../src/evidence/mutation-analysis.ts | 526 ++ .../src/evidence/mutation-checkpoint.ts | 82 + .../src/evidence/mutation-control.ts | 116 + .../src/evidence/mutation-rebase.ts | 241 + .../src/evidence/mutation-shards.ts | 144 + .../src/evidence/null-control-analysis.ts | 138 + tools/stack-bench/src/evidence/outcomes.ts | 242 + .../src/evidence/pricing-authority.ts | 57 + tools/stack-bench/src/evidence/provenance.ts | 135 + .../src/evidence/repair-evidence.ts | 40 + tools/stack-bench/src/evidence/scoring.ts | 140 + .../src/evidence/session-metrics.ts | 95 + .../src/grading/execution-shards.ts | 415 ++ tools/stack-bench/src/package-root.ts | 37 + .../src/progression/dependency-definition.ts | 361 ++ .../src/progression/dependency-mode.ts | 974 +++ .../src/progression/dependency-score.ts | 153 + .../progression/feature-catalog-selection.ts | 43 + .../src/progression/grade-bundle-result.ts | 395 ++ .../src/progression/live-progression.ts | 471 ++ .../src/progression/progression-definition.ts | 601 ++ .../src/progression/progression-engine.ts | 131 + .../src/progression/progression-graph.ts | 158 + .../progression-recipe-selection.ts | 409 ++ .../progression-reference-audit.ts | 336 + .../src/progression/progression-runner.ts | 176 + .../src/progression/progression-state.ts | 562 ++ .../src/references/reference-agent.ts | 287 + .../src/references/reference-build.ts | 293 + .../src/references/reference-fixtures.ts | 377 ++ .../src/references/reference-install.ts | 25 + .../src/references/reference-live.ts | 1053 +++ .../reference-qualification-audit.ts | 245 + .../src/references/reference-selection.ts | 31 + .../src/releases/release-bundle.ts | 157 + .../src/releases/release-manifest.ts | 403 ++ .../src/releases/release-source.ts | 201 + .../src/runtime/backend-control.ts | 108 + .../stack-bench/src/runtime/backend-lease.ts | 482 ++ .../src/runtime/backend-teardown.ts | 120 + .../src/runtime/bounded-process.ts | 150 + .../src/runtime/coding-container-policy.ts | 37 + .../src/runtime/command-executor.ts | 12 + .../src/runtime/container-image.ts | 39 + .../src/runtime/container-mount.ts | 43 + .../stack-bench/src/runtime/docker-network.ts | 13 + .../src/runtime/grading-timeout.ts | 32 + .../src/runtime/operational-paths.ts | 10 + tools/stack-bench/src/runtime/path-safety.ts | 16 + tools/stack-bench/src/runtime/platform.ts | 304 + tools/stack-bench/src/runtime/preflight.ts | 799 +++ tools/stack-bench/src/runtime/readiness.ts | 38 + tools/stack-bench/src/runtime/recovery.ts | 247 + tools/stack-bench/src/runtime/repair-grant.ts | 396 ++ .../src/runtime/runner-environment.ts | 90 + tools/stack-bench/src/runtime/sandbox.ts | 79 + .../src/runtime/source-checkpoint.ts | 68 + .../src/runtime/source-snapshot.ts | 161 + .../src/runtime/spacetime-layout.ts | 166 + .../src/runtime/spacetime-target.ts | 62 + .../src/stacks/backend-reset-guard.ts | 19 + tools/stack-bench/src/stacks/backend-reset.ts | 20 + .../src/stacks/backends/mongodb-adapter.ts | 46 + .../src/stacks/backends/mongodb-identity.ts | 1 + .../src/stacks/backends/mongodb-operations.ts | 122 + .../src/stacks/backends/postgres-adapter.ts | 46 + .../src/stacks/backends/postgres-identity.ts | 1 + .../stacks/backends/postgres-operations.ts | 122 + .../src/stacks/backends/spacetime-adapter.ts | 41 + .../src/stacks/backends/spacetime-identity.ts | 1 + .../stacks/backends/spacetime-operations.ts | 97 + .../src/stacks/backends/stub-adapter.ts | 37 + .../src/stacks/backends/stub-identity.ts | 1 + .../src/stacks/database-containers.ts | 19 + .../src/stacks/hosted-database-identity.ts | 5 + .../src/stacks/stack-adapter-common.ts | 128 + .../src/stacks/stack-adapter-contract.ts | 223 + .../stack-bench/src/stacks/stack-adapters.ts | 21 + .../src/stacks/stack-agent-operations.ts | 145 + .../src/stacks/stack-backend-operations.ts | 10 + .../src/stacks/stack-identities.ts | 17 + .../src/stacks/stack-lease-capabilities.ts | 167 + .../src/stacks/stack-lifecycle-operations.ts | 399 ++ .../stacks/stack-orchestrator-operations.ts | 31 + .../src/stacks/stack-reference-operations.ts | 191 + .../src/stacks/stack-teardown-operations.ts | 145 + .../stack-bench/tests/action-contract.test.ts | 207 + .../actor-transport-action-executors.test.ts | 627 ++ .../advanced-support-feature-packs.test.ts | 175 + .../stack-bench/tests/agent-adapters.test.ts | 188 + .../tests/agent-credentials.test.ts | 35 + .../stack-bench/tests/agent-database.test.ts | 126 + .../stack-bench/tests/agent-guidance.test.ts | 34 + .../stack-bench/tests/agent-materials.test.ts | 30 + .../tests/agent-runtime-identity.test.ts | 69 + .../tests/agent-task-selection.test.ts | 275 + tools/stack-bench/tests/artifacts.test.ts | 328 + .../stack-bench/tests/backend-control.test.ts | 203 + tools/stack-bench/tests/backend-lease.test.ts | 415 ++ tools/stack-bench/tests/bench-policy.test.ts | 403 ++ .../tests/binary-provenance.test.ts | 128 + .../tests/browser-action-executors.test.ts | 293 + .../tests/browser-evidence.test.ts | 20 + .../build-container-reconciliation.test.ts | 55 + .../tests/calibration-compiler.test.ts | 572 ++ .../tests/campaign-admission.test.ts | 103 + tools/stack-bench/tests/campaign-cli.test.ts | 111 + .../tests/campaign-compiler.test.ts | 790 +++ tools/stack-bench/tests/campaign-lock.test.ts | 158 + tools/stack-bench/tests/campaign-mode.test.ts | 44 + .../tests/campaign-progression-grant.test.ts | 158 + .../stack-bench/tests/campaign-report.test.ts | 379 ++ .../stack-bench/tests/campaign-runner.test.ts | 1123 ++++ .../tests/campaign-runtime.test.ts | 105 + .../tests/campaign-scheduler.test.ts | 336 + .../tests/cart-quantity-mutations.test.ts | 122 + .../catalog-search-feature-contracts.test.ts | 154 + .../stack-bench/tests/check-evidence.test.ts | 107 + .../tests/claude-terminal-recovery.test.ts | 207 + .../coding-interruption-recovery.test.ts | 581 ++ .../stack-bench/tests/composition-cli.test.ts | 135 + .../tests/composition-compiler.test.ts | 593 ++ .../tests/condition-compiler.test.ts | 172 + .../stack-bench/tests/container-auth.test.ts | 49 + .../stack-bench/tests/container-image.test.ts | 21 + .../stack-bench/tests/container-mount.test.ts | 29 + tools/stack-bench/tests/controller.test.ts | 120 + tools/stack-bench/tests/cost-ledger.test.ts | 44 + .../tests/credential-aliases.test.ts | 94 + .../tests/credential-broker.test.ts | 859 +++ tools/stack-bench/tests/dashboard.test.ts | 508 ++ .../tests/database-containers.test.ts | 18 + .../tests/definition-compiler.test.ts | 257 + .../tests/definition-snapshots.test.ts | 31 + .../tests/dependency-campaign-live.test.ts | 212 + .../stack-bench/tests/dependency-mode.test.ts | 851 +++ .../tests/dependency-neutral-prompt.test.ts | 208 + .../tests/dependency-volume.test.ts | 52 + .../tests/diagnostic-sanitizer.test.ts | 54 + .../stack-bench/tests/docker-network.test.ts | 17 + .../tests/ecommerce-progression-2.0.1.test.ts | 162 + .../ecommerce-progression-campaign.test.ts | 228 + .../tests/ecommerce-progression.test.ts | 352 + .../tests/element-selector.test.ts | 10 + .../stack-bench/tests/evidence-timing.test.ts | 13 + .../tests/execution-shards.test.ts | 230 + .../external-data-sync-candidate.test.ts | 314 + .../tests/feature-catalog-selection.test.ts | 16 + .../fixtures/definitions/all-actions.json | 74 + .../dependency-model-free-campaign.json | 143 + .../tests/fixtures/resource-lock-process.ts | 23 + .../fixtures/spacetime-module/package.json | 11 + .../fixtures/spacetime-module/src/index.ts | 14 + .../fixtures/spacetime-module/tsconfig.json | 13 + .../stack-bench/tests/grader-cleanup.test.ts | 56 + .../stack-bench/tests/grading-timeout.test.ts | 25 + .../stack-bench/tests/harness-errors.test.ts | 32 + .../tests/harness-regressions.test.ts | 60 + .../tests/helpers/progression-source.ts | 37 + .../identity-staff-feature-contracts.test.ts | 216 + .../inventory-order-feature-contracts.test.ts | 189 + tools/stack-bench/tests/isolation.test.ts | 51 + .../tests/l1-concurrency-completion.test.ts | 204 + .../tests/l3-composition-candidate.test.ts | 358 ++ tools/stack-bench/tests/leak-audit.test.ts | 36 + ...cycle-concurrency-action-executors.test.ts | 280 + .../stack-bench/tests/linter-coverage.test.ts | 111 + .../tests/live-progression.test.ts | 610 ++ .../tests/modular-selection.test.ts | 162 + tools/stack-bench/tests/module-layout.test.ts | 191 + .../mongodb-progression-reference.test.ts | 95 + .../tests/mutation-analysis.test.ts | 291 + .../tests/mutation-checkpoint.test.ts | 59 + .../tests/mutation-manifests.test.ts | 103 + .../tests/mutation-preflight.test.ts | 134 + .../stack-bench/tests/mutation-rebase.test.ts | 90 + .../stack-bench/tests/mutation-shards.test.ts | 57 + .../tests/neutral-guidance.test.ts | 25 + .../tests/null-control-analysis.test.ts | 73 + .../tests/null-control-cli.test.ts | 109 + .../tests/open-list-live-candidate.test.ts | 150 + .../tests/operational-paths.test.ts | 87 + .../operations-feature-contracts.test.ts | 26 + tools/stack-bench/tests/outcomes.test.ts | 209 + tools/stack-bench/tests/pack-budget.test.ts | 191 + tools/stack-bench/tests/pack-runtime.test.ts | 87 + tools/stack-bench/tests/package-root.test.ts | 15 + tools/stack-bench/tests/path-safety.test.ts | 49 + .../tests/platform-process-tree.test.ts | 27 + .../tests/preflight-command.test.ts | 15 + tools/stack-bench/tests/preflight.test.ts | 506 ++ .../tests/pricing-authority.test.ts | 32 + .../stack-bench/tests/product-config.test.ts | 32 + ...rogression-activity-feedback-packs.test.ts | 44 + ...rogression-behavior-evidence-packs.test.ts | 221 + .../tests/progression-definition.test.ts | 92 + .../progression-depth3-composition.test.ts | 82 + .../tests/progression-feature-packs.test.ts | 220 + .../tests/progression-fulfilment-pack.test.ts | 39 + .../tests/progression-grade-bundle.test.ts | 209 + .../tests/progression-graph.test.ts | 47 + .../tests/progression-integration.test.ts | 264 + .../tests/progression-mutations.test.ts | 205 + ...rogression-order-cancellation-pack.test.ts | 39 + ...ogression-pagination-compatibility.test.ts | 180 + .../progression-postgres-reference.test.ts | 95 + .../tests/progression-product-packs.test.ts | 46 + ...ssion-promotion-notification-packs.test.ts | 163 + .../tests/progression-reference-audit.test.ts | 198 + ...ogression-reference-campaign-audit.test.ts | 107 + .../tests/progression-runner.test.ts | 116 + .../tests/progression-shopping-packs.test.ts | 69 + .../progression-spacetime-policy.test.ts | 47 + .../progression-spacetime-reference.test.ts | 196 + .../tests/progression-state.test.ts | 200 + .../progression-survivor-regressions.test.ts | 203 + .../progression-timed-feature-packs.test.ts | 43 + tools/stack-bench/tests/provenance.test.ts | 71 + .../tests/qualification-cli.test.ts | 122 + .../tests/qualification-scope.test.ts | 233 + tools/stack-bench/tests/readiness.test.ts | 21 + .../stack-bench/tests/recipe-release.test.ts | 459 ++ .../tests/recovery-command.test.ts | 16 + tools/stack-bench/tests/recovery.test.ts | 162 + .../stack-bench/tests/reference-agent.test.ts | 257 + .../tests/reference-fixtures.test.ts | 188 + .../tests/reference-install.test.ts | 36 + .../stack-bench/tests/reference-live.test.ts | 730 +++ .../stack-bench/tests/release-bundle.test.ts | 96 + .../tests/release-manifest.test.ts | 180 + .../stack-bench/tests/release-source.test.ts | 121 + tools/stack-bench/tests/repair-cli.test.ts | 32 + tools/stack-bench/tests/repair-grant.test.ts | 164 + tools/stack-bench/tests/report-bugs.test.ts | 326 + tools/stack-bench/tests/reset-backend.test.ts | 386 ++ tools/stack-bench/tests/reset-safety.test.ts | 113 + .../tests/run-session-artifacts.test.ts | 118 + tools/stack-bench/tests/run-suite.test.ts | 505 ++ .../tests/runner-environment.test.ts | 51 + tools/stack-bench/tests/sandbox.test.ts | 34 + .../tests/scenario-contract-check.test.ts | 28 + tools/stack-bench/tests/scoring.test.ts | 125 + .../stack-bench/tests/selected-grade.test.ts | 254 + .../tests/sequential-l1-calibration.test.ts | 68 + .../tests/sequential-l2-calibration.test.ts | 100 + .../stack-bench/tests/session-metrics.test.ts | 39 + .../definitions/all-actions.snapshot.json | 342 + .../snapshots/definitions/chat.snapshot.json | 3440 ++++++++++ .../definitions/ecommerce.snapshot.json | 5716 +++++++++++++++++ .../snapshots/definitions/loop.snapshot.json | 71 + .../tests/source-checkpoint.test.ts | 100 + .../stack-bench/tests/source-snapshot.test.ts | 209 + .../tests/spacetime-guidance.test.ts | 17 + .../tests/stack-action-operations.test.ts | 60 + .../stack-bench/tests/stack-adapters.test.ts | 244 + .../tests/stack-reference-operations.test.ts | 117 + .../tests/support-entry-feature-packs.test.ts | 154 + tools/stack-bench/tests/tracks.test.ts | 59 + tools/stack-bench/tracks/chat/LEVELS.md | 67 + .../tracks/chat/contracts/01-basic-chat.json | 277 + .../chat/contracts/02-authorization.json | 108 + .../tracks/chat/contracts/appendix-01.md | 64 + .../tracks/chat/contracts/appendix-02.md | 82 + .../tracks/chat/prompts/01-basic-chat.md | 163 + .../tracks/chat/prompts/02-authorization.md | 57 + .../tracks/chat/prompts/03-contention.md | 34 + .../tracks/chat/prompts/04-deferred.md | 31 + .../tracks/chat/prompts/05-volume.md | 30 + .../tracks/chat/scenarios/01-basic-chat.json | 813 +++ .../chat/scenarios/01-contention-wip.json | 282 + .../tracks/chat/scenarios/01-delivery.json | 218 + .../tracks/chat/scenarios/01-invariants.json | 974 +++ .../tracks/chat/scenarios/01-systems.json | 227 + .../tracks/chat/scenarios/02-features.json | 196 + .../tracks/chat/scenarios/02-invariants.json | 107 + .../tracks/chat/scenarios/02-systems.json | 82 + tools/stack-bench/tracks/chat/track.json | 50 + tools/stack-bench/tracks/chat/walk.ts | 89 + tools/stack-bench/tracks/ecommerce/LEVELS.md | 107 + .../tracks/ecommerce/composition/README.md | 100 + .../progression-depth3-2.0.1.json | 138 + .../calibrations/sequential-l1-2.5.0.json | 167 + .../calibrations/sequential-l2-1.6.0.json | 195 + .../ecommerce/composition/candidates.json | 55 + .../fixtures/operations-1.0.0.json | 28 + .../fixtures/storefront-1.0.0.json | 27 + .../composition/framing/smoke-requirements.md | 7 + .../packs/feature-accounts-1.1.0.json | 29 + .../packs/feature-accounts-1.2.0.json | 29 + .../composition/packs/feature-cart-2.0.0.json | 62 + .../packs/feature-cart-checkout-1.2.0.json | 63 + .../packs/feature-catalog-1.1.0.json | 26 + .../feature-catalog-discovery-1.0.0.json | 71 + .../packs/feature-catalog-items-1.0.0.json | 59 + .../packs/feature-checkout-2.0.0.json | 61 + .../packs/feature-purchasing-1.1.0.json | 26 + .../packs/feature-purchasing-1.2.1.json | 60 + .../packs/feature-reviews-1.1.0.json | 26 + .../packs/feature-reviews-1.2.1.json | 61 + .../packs/feature-warehouse-admin-1.1.0.json | 26 + .../packs/feature-warehouse-admin-1.2.1.json | 87 + .../inventory-operations-features-1.2.0.json | 142 + ...ntory-operations-specifications-1.0.0.json | 28 + .../packs/l2-inventory-dashboard-2.0.0.json | 62 + .../l2-order-cancellation-features-1.0.1.json | 70 + .../l2-price-history-features-2.0.1.json | 85 + .../packs/l2-recommendations-2.0.0.json | 60 + .../packs/l2-sales-dashboard-2.0.0.json | 69 + .../l2-stock-transfers-features-1.0.1.json | 70 + .../l3-cart-expiration-features-1.0.0.json | 23 + .../l3-cart-expiration-features-2.0.0.json | 56 + ...-deferred-access-specifications-1.0.0.json | 23 + ...erred-durability-specifications-1.0.0.json | 31 + ...ferred-integrity-specifications-1.0.0.json | 30 + .../l3-order-delivery-features-1.0.0.json | 24 + .../l3-order-delivery-features-1.1.1.json | 64 + .../l3-order-returns-features-1.1.1.json | 70 + .../packs/l3-reservations-features-1.0.0.json | 27 + .../packs/l3-reservations-features-2.0.0.json | 85 + .../l3-scheduled-restocks-features-1.0.0.json | 28 + .../l3-scheduled-restocks-features-1.1.1.json | 70 + .../l3-server-time-specifications-1.0.0.json | 27 + .../operations-access-features-1.0.0.json | 31 + ...perations-access-specifications-1.0.0.json | 29 + .../progression-automatic-reorder-2.0.0.json | 73 + ...ation-accounting-specifications-1.0.0.json | 23 + ...ncellation-queue-specifications-1.0.0.json | 25 + .../progression-cart-recovery-2.0.0.json | 68 + .../progression-catalog-management-1.0.2.json | 56 + .../progression-customer-profile-1.0.0.json | 11 + ...gression-delivery-notifications-1.0.1.json | 69 + .../progression-faceted-search-1.0.1.json | 69 + .../progression-fulfilment-queue-1.0.1.json | 92 + ...ory-conservation-specifications-1.0.0.json | 29 + .../progression-managed-support-1.0.0.json | 54 + ...ession-notification-preferences-1.0.0.json | 14 + ...perations-access-specifications-1.0.0.json | 31 + .../progression-order-support-1.0.1.json | 69 + .../progression-payment-records-2.0.0.json | 56 + ...on-personalized-recommendations-2.0.0.json | 68 + ...price-accounting-specifications-1.0.0.json | 23 + .../progression-promotion-checkout-2.0.0.json | 79 + ...progression-promotion-reporting-2.0.0.json | 68 + .../progression-promotion-rules-1.0.1.json | 69 + ...ression-recommendation-feedback-2.0.0.json | 55 + .../packs/progression-staff-access-1.0.0.json | 11 + .../progression-staff-activity-1.0.2.json | 56 + .../packs/progression-staff-roles-1.0.0.json | 11 + .../packs/progression-stock-alerts-1.0.1.json | 70 + .../progression-support-history-1.0.0.json | 14 + .../progression-support-intake-1.0.0.json | 11 + .../progression-support-refunds-1.0.1.json | 79 + .../progression-support-triage-1.0.0.json | 15 + .../packs/returns-pricing-features-1.1.0.json | 139 + .../returns-pricing-specifications-1.0.0.json | 25 + .../packs/spec-access-control-1.2.0.json | 34 + .../packs/spec-access-control-2.0.0.json | 165 + .../packs/spec-concurrency-safety-1.3.0.json | 29 + .../packs/spec-concurrency-safety-2.0.1.json | 129 + .../packs/spec-external-data-sync-1.1.0.json | 27 + .../packs/spec-external-data-sync-1.1.1.json | 117 + .../packs/spec-live-state-1.2.0.json | 32 + .../packs/spec-live-state-2.0.0.json | 167 + .../packs/spec-state-durability-1.1.0.json | 28 + .../packs/spec-state-durability-2.0.1.json | 126 + .../spec-transactional-integrity-1.3.0.json | 28 + .../spec-transactional-integrity-1.4.0.json | 28 + .../ecommerce/composition/promotions.json | 25 + .../recipes/progression-catalog-2.0.1.json | 510 ++ .../recipes/progression-depth3-2.0.1.json | 64 + .../recipes/sequential-l1-2.5.0.json | 368 ++ .../recipes/sequential-l2-1.6.0.json | 458 ++ .../recipes/sequential-l3-1.0.0.json | 557 ++ .../ecommerce/contracts/accounts-1.2.md | 27 + .../tracks/ecommerce/contracts/appendix-02.md | 95 + .../tracks/ecommerce/contracts/cart-2.0.md | 9 + .../contracts/cart-expiration-1.0.md | 5 + .../contracts/catalog-discovery-1.0.md | 9 + .../ecommerce/contracts/catalog-items-1.0.md | 10 + .../contracts/catalog-management-1.0.md | 5 + .../ecommerce/contracts/checkout-2.0.md | 4 + .../contracts/customer-profile-1.0.md | 5 + .../contracts/delivery-notifications-1.0.md | 4 + .../ecommerce/contracts/faceted-search-1.0.md | 12 + .../contracts/inventory-dashboard-2.0.md | 4 + .../ecommerce/contracts/l2-order-hooks-1.4.md | 4 + .../ecommerce/contracts/l3-hooks-1.0.md | 30 + .../contracts/managed-support-1.0.md | 7 + .../contracts/notification-preferences-1.0.md | 5 + .../contracts/operations-access-1.1.md | 32 + .../contracts/order-cancellation-1.0.md | 14 + .../ecommerce/contracts/order-delivery-1.0.md | 4 + .../ecommerce/contracts/order-returns-1.0.md | 7 + .../ecommerce/contracts/order-support-1.0.md | 11 + .../contracts/payment-records-1.0.md | 3 + .../ecommerce/contracts/price-history-2.0.md | 16 + .../progression-automatic-reorder-2.0.md | 11 + .../progression-cart-recovery-1.0.md | 9 + ...ession-personalized-recommendations-1.0.md | 5 + .../contracts/promotion-checkout-1.0.md | 4 + .../contracts/promotion-reporting-1.0.md | 4 + .../contracts/promotion-rules-1.0.md | 5 + .../ecommerce/contracts/purchasing-1.0.md | 5 + .../contracts/recommendation-feedback-1.0.md | 3 + .../contracts/recommendations-2.0.md | 4 + .../ecommerce/contracts/reservations-1.0.md | 5 + .../tracks/ecommerce/contracts/reviews-1.0.md | 5 + .../contracts/sales-dashboard-2.0.md | 6 + .../contracts/scheduled-restocks-1.0.md | 18 + .../ecommerce/contracts/staff-access-1.0.md | 11 + .../ecommerce/contracts/staff-activity-1.0.md | 4 + .../ecommerce/contracts/staff-roles-1.0.md | 6 + .../ecommerce/contracts/stock-alerts-1.0.md | 4 + .../contracts/stock-transfers-1.0.md | 19 + .../contracts/support-history-1.0.md | 3 + .../ecommerce/contracts/support-intake-1.0.md | 5 + .../contracts/support-refunds-1.0.md | 12 + .../ecommerce/contracts/support-triage-1.0.md | 5 + .../contracts/warehouse-administration-1.0.md | 22 + .../progression/ecommerce-2.0.1.json | 939 +++ .../tracks/ecommerce/prompts/02-operations.md | 120 + .../tracks/ecommerce/prompts/03-scheduled.md | 67 + ...amed-actions-inventory-operations-1.0.0.md | 12 + .../named-actions-returns-pricing-1.0.0.md | 14 + .../named-actions-scheduled-restocks-1.0.0.md | 14 + .../prompts/fragments/startup-probe-1.0.0.md | 6 + .../ecommerce/prompts/l2-action-inputs.md | 22 + .../prompts/modular/accounts-1.2.0.md | 5 + .../cancellation-conservation-1.0.0.md | 4 + .../modular/cancellation-queue-1.0.0.md | 4 + .../ecommerce/prompts/modular/cart-2.0.0.md | 4 + .../prompts/modular/cart-expiration-1.0.0.md | 4 + .../modular/catalog-discovery-1.0.0.md | 4 + .../prompts/modular/catalog-items-1.0.0.md | 3 + .../modular/catalog-management-1.0.0.md | 4 + .../prompts/modular/checkout-2.0.0.md | 4 + .../prompts/modular/customer-profile-1.0.0.md | 4 + .../modular/delivery-notifications-1.0.0.md | 4 + .../prompts/modular/faceted-search-1.0.0.md | 5 + .../modular/inventory-dashboard-2.0.0.md | 4 + .../prompts/modular/l1-action-inputs-1.1.0.md | 23 + .../prompts/modular/l1-features-1.1.0.md | 57 + .../ecommerce/prompts/modular/l1-features.md | 56 + .../ecommerce/prompts/modular/l1-framing.md | 23 + .../ecommerce/prompts/modular/l1-hooks.md | 87 + .../modular/l1-specifications-1.1.0.md | 89 + .../prompts/modular/l1-specifications.md | 88 + .../prompts/modular/l3-features-1.0.0.md | 26 + .../modular/l3-specifications-1.0.0.md | 41 + .../prompts/modular/managed-support-1.0.0.md | 4 + .../modular/notification-preferences-1.0.0.md | 4 + .../modular/operations-access-1.1.0.md | 8 + .../prompts/modular/order-accounting-1.0.0.md | 7 + .../modular/order-cancellation-1.0.0.md | 4 + .../prompts/modular/order-delivery-1.0.0.md | 4 + .../prompts/modular/order-ownership-1.0.0.md | 3 + .../prompts/modular/order-returns-1.0.0.md | 7 + .../prompts/modular/order-support-1.0.0.md | 4 + .../prompts/modular/payment-records-1.0.0.md | 4 + .../modular/price-authorization-1.0.0.md | 3 + .../prompts/modular/price-history-1.0.0.md | 7 + .../progression-automatic-reorder-1.0.0.md | 6 + .../progression-cart-recovery-1.0.0.md | 4 + .../modular/progression-framing-1.0.0.md | 7 + ...sion-personalized-recommendations-1.0.0.md | 6 + .../modular/promotion-checkout-1.0.0.md | 4 + .../modular/promotion-reporting-1.0.0.md | 4 + .../prompts/modular/promotion-rules-1.0.0.md | 4 + .../prompts/modular/purchasing-1.0.0.md | 6 + .../modular/recommendation-feedback-1.0.0.md | 4 + .../prompts/modular/recommendations-2.0.0.md | 5 + .../prompts/modular/reservations-1.0.0.md | 5 + .../prompts/modular/reviews-1.0.0.md | 5 + .../prompts/modular/sales-dashboard-2.0.0.md | 5 + .../modular/scheduled-restocks-1.0.0.md | 5 + .../modular/shipping-authorization-1.0.0.md | 3 + .../prompts/modular/staff-access-1.0.0.md | 3 + .../prompts/modular/staff-activity-1.0.0.md | 4 + .../prompts/modular/staff-roles-1.0.0.md | 4 + .../prompts/modular/stock-alerts-1.0.0.md | 4 + .../modular/stock-conservation-1.0.0.md | 4 + .../prompts/modular/stock-transfers-1.0.0.md | 8 + .../prompts/modular/support-history-1.0.0.md | 4 + .../prompts/modular/support-intake-1.0.0.md | 4 + .../prompts/modular/support-refunds-1.0.0.md | 4 + .../prompts/modular/support-triage-1.0.0.md | 3 + .../modular/transfer-authorization-1.0.0.md | 3 + .../modular/warehouse-administration-1.0.0.md | 6 + .../scenarios/01-account-create-2.4.0.json | 28 + .../scenarios/01-account-duplicate-2.4.0.json | 28 + .../scenarios/01-account-password-2.4.0.json | 28 + .../scenarios/01-account-reload-2.4.0.json | 27 + .../scenarios/01-account-signout-2.4.0.json | 29 + .../01-account-state-reconnect-2.4.0.json | 48 + .../01-account-state-reload-2.4.0.json | 40 + .../scenarios/01-admin-write-2.4.0.json | 99 + .../scenarios/01-admin-write-staff-1.0.0.json | 80 + .../scenarios/01-books-balance-2.4.0.json | 86 + .../ecommerce/scenarios/01-buying-2.4.0.json | 76 + .../ecommerce/scenarios/01-cart-2.4.0.json | 79 + .../scenarios/01-cart-boundary-2.4.0.json | 121 + .../scenarios/01-catalog-ranking-1.0.0.json | 43 + .../scenarios/01-catalog-search-1.0.0.json | 27 + .../scenarios/01-catalog-values-1.0.0.json | 28 + .../ecommerce/scenarios/01-contention.json | 595 ++ .../ecommerce/scenarios/01-core-2.4.0.json | 62 + .../01-duplicate-checkout-2.3.0.json | 98 + .../01-duplicate-checkout-2.3.1.json | 88 + .../01-external-live-sync-1.1.0.json | 51 + .../01-external-reconnect-sync-1.1.0.json | 53 + .../01-external-reload-sync-1.1.0.json | 52 + ...01-external-server-restart-sync-1.1.0.json | 53 + .../ecommerce/scenarios/01-features.json | 1180 ++++ .../ecommerce/scenarios/01-invariants.json | 851 +++ .../scenarios/01-last-unit-2.3.0.json | 116 + .../scenarios/01-last-unit-2.3.1.json | 110 + .../scenarios/01-open-list-live-2.3.0.json | 84 + .../scenarios/01-order-ownership-2.4.0.json | 38 + .../01-purchase-attribution-2.4.0.json | 45 + .../scenarios/01-purchase-session-2.4.0.json | 81 + .../scenarios/01-restock-race-2.3.0.json | 140 + .../01-review-eligibility-2.4.0.json | 70 + .../01-review-rating-live-2.4.0.json | 48 + .../scenarios/01-review-uniqueness-2.4.0.json | 42 + .../scenarios/01-review-visibility-2.4.0.json | 37 + .../scenarios/01-server-price-2.4.0.json | 50 + .../ecommerce/scenarios/01-systems.json | 251 + .../scenarios/01-warehouse-admin-2.4.0.json | 46 + .../01-warehouse-admin-staff-1.0.0.json | 51 + .../01-warehouse-stock-live-2.4.0.json | 35 + .../01-warehouse-stock-live-staff-1.0.0.json | 36 + .../02-cancellation-queue-1.0.0.json | 37 + .../ecommerce/scenarios/02-features.json | 1199 ++++ .../scenarios/02-fulfilment-access-1.0.0.json | 32 + .../scenarios/02-fulfilment-live-1.0.0.json | 33 + .../scenarios/02-fulfilment-ship-1.0.0.json | 35 + .../ecommerce/scenarios/02-invariants.json | 764 +++ .../scenarios/02-live-price-1.4.0.json | 33 + .../scenarios/02-low-stock-1.4.0.json | 80 + .../02-operational-best-sellers-1.0.0.json | 32 + .../02-operational-category-totals-1.0.0.json | 34 + .../02-operational-recommendations-1.0.0.json | 33 + .../02-order-cancellation-core-1.0.0.json | 40 + .../02-order-cancellation-history-1.0.0.json | 31 + .../02-paid-price-history-1.4.0.json | 39 + .../scenarios/02-queue-warehouse-1.4.0.json | 33 + .../scenarios/02-self-contained-1.4.0.json | 180 + .../scenarios/02-server-actions-1.1.0.json | 136 + .../scenarios/02-strengthened-1.4.0.json | 265 + .../scenarios/02-transfer-totals-1.4.0.json | 36 + .../scenarios/03-cart-expiration-1.0.0.json | 33 + .../scenarios/03-deferred-access-1.0.0.json | 33 + .../03-deferred-durability-1.0.0.json | 56 + .../03-deferred-integrity-1.0.0.json | 43 + .../ecommerce/scenarios/03-features.json | 336 + .../ecommerce/scenarios/03-invariants.json | 313 + .../scenarios/03-order-delivery-1.0.0.json | 59 + .../scenarios/03-reservations-1.0.0.json | 115 + .../03-scheduled-restocks-1.0.0.json | 67 + .../scenarios/03-server-time-1.0.0.json | 30 + ...ression-account-state-reconnect-1.0.0.json | 36 + ...rogression-account-state-reload-1.0.1.json | 40 + .../progression-automatic-reorder-1.0.0.json | 67 + .../progression-books-balance-1.0.0.json | 47 + .../progression-cart-checkout-1.0.0.json | 52 + .../progression-cart-recovery-1.0.0.json | 51 + .../progression-catalog-management-1.0.1.json | 46 + .../progression-core-business-1.0.0.json | 217 + .../progression-customer-profile-1.0.0.json | 49 + ...gression-delivery-notifications-1.0.0.json | 48 + .../progression-faceted-filters-1.0.0.json | 38 + .../progression-faceted-pagination-1.0.0.json | 30 + ...ression-managed-support-privacy-1.0.0.json | 41 + ...gression-managed-support-shared-1.0.0.json | 42 + ...ession-notification-preferences-1.0.0.json | 44 + .../progression-open-list-live-1.0.0.json | 39 + ...ogression-order-return-boundary-1.0.0.json | 31 + ...ogression-order-return-complete-1.0.0.json | 43 + ...gression-order-support-boundary-1.0.0.json | 64 + ...progression-order-support-owned-1.0.0.json | 37 + ...on-personalized-recommendations-1.0.0.json | 51 + ...progression-price-cart-checkout-1.0.0.json | 39 + .../progression-promotion-checkout-1.0.0.json | 88 + ...progression-promotion-reporting-1.0.0.json | 50 + .../progression-promotion-rules-1.0.0.json | 51 + .../progression-purchasing-1.0.0.json | 31 + ...ression-recommendation-feedback-1.0.0.json | 42 + ...progression-signed-out-purchase-1.0.0.json | 28 + .../progression-staff-access-1.0.0.json | 49 + .../progression-staff-activity-1.0.0.json | 60 + .../progression-staff-roles-1.0.0.json | 59 + .../progression-stock-alerts-1.0.0.json | 51 + .../progression-stock-limit-1.0.0.json | 41 + .../progression-support-history-1.0.0.json | 45 + .../progression-support-intake-1.0.0.json | 31 + ...gression-support-refunds-access-1.0.0.json | 59 + ...sion-support-refunds-accounting-1.0.0.json | 46 + ...sion-support-refunds-resolution-1.0.0.json | 40 + .../progression-support-triage-1.0.0.json | 60 + tools/stack-bench/tracks/ecommerce/track.json | 127 + tools/stack-bench/tracks/ecommerce/walk.ts | 209 + .../tracks/loop/contracts/01-loop.json | 11 + .../tracks/loop/prompts/01-loop.md | 4 + .../tracks/loop/scenarios/01-loop.json | 29 + tools/stack-bench/tracks/loop/track.json | 19 + tools/stack-bench/tracks/loop/walk.ts | 6 + tools/stack-bench/tsconfig.build.json | 10 + tools/stack-bench/tsconfig.json | 31 + 854 files changed, 154574 insertions(+), 40 deletions(-) create mode 100644 crates/bindings-typescript/tests/table_ref_error_message.test.ts create mode 100644 skills/spacetimedb-typescript-core/SKILL.md create mode 100644 tools/llm-sequential-upgrade/read-guard.sh create mode 100644 tools/stack-bench/.gitignore create mode 100644 tools/stack-bench/APPLIANCE-DESIGN.md create mode 100644 tools/stack-bench/README.md create mode 100644 tools/stack-bench/SETUP.md create mode 100644 tools/stack-bench/appliance/Controller.Dockerfile create mode 100644 tools/stack-bench/appliance/Controller.Dockerfile.dockerignore create mode 100644 tools/stack-bench/appliance/README.md create mode 100644 tools/stack-bench/appliance/RECOVERY.md create mode 100644 tools/stack-bench/appliance/RELEASE.md create mode 100644 tools/stack-bench/appliance/campaign.ecommerce-progression-reference.json create mode 100644 tools/stack-bench/appliance/campaign.example.json create mode 100644 tools/stack-bench/appliance/campaign.product-brief-reference.json create mode 100644 tools/stack-bench/appliance/controller.ts create mode 100644 tools/stack-bench/appliance/dependency-volume.ts create mode 100644 tools/stack-bench/appliance/docker-compose.yaml create mode 100644 tools/stack-bench/appliance/operator.env.example create mode 100644 tools/stack-bench/backends/minimal/mongodb-1.5.md create mode 100644 tools/stack-bench/backends/minimal/postgres-1.5.md create mode 100644 tools/stack-bench/backends/minimal/spacetime-1.6.md create mode 100644 tools/stack-bench/backends/model-free-stub.md create mode 100644 tools/stack-bench/backends/mongodb.md create mode 100644 tools/stack-bench/backends/postgres.md create mode 100644 tools/stack-bench/backends/spacetime.md create mode 100644 tools/stack-bench/commands/agent.ts create mode 100644 tools/stack-bench/commands/archive-transcripts.ts create mode 100644 tools/stack-bench/commands/bench-arguments.ts create mode 100644 tools/stack-bench/commands/bench.ts create mode 100644 tools/stack-bench/commands/campaign-cli.ts create mode 100644 tools/stack-bench/commands/check-actions.ts create mode 100644 tools/stack-bench/commands/check-calibration.ts create mode 100644 tools/stack-bench/commands/check-composition.ts create mode 100644 tools/stack-bench/commands/check-mutations.ts create mode 100644 tools/stack-bench/commands/check-scenarios.ts create mode 100644 tools/stack-bench/commands/composition-cli.ts create mode 100644 tools/stack-bench/commands/container-smoke.ts create mode 100644 tools/stack-bench/commands/cost-ledger.ts create mode 100644 tools/stack-bench/commands/definition-snapshots.ts create mode 100644 tools/stack-bench/commands/fault-injection.ts create mode 100644 tools/stack-bench/commands/leak-audit.ts create mode 100644 tools/stack-bench/commands/lease-cli.ts create mode 100644 tools/stack-bench/commands/null-control.ts create mode 100644 tools/stack-bench/commands/preflight.ts create mode 100644 tools/stack-bench/commands/probe-sandbox.ts create mode 100644 tools/stack-bench/commands/progression-graph.ts create mode 100644 tools/stack-bench/commands/qualification-cli.ts create mode 100644 tools/stack-bench/commands/recovery.ts create mode 100644 tools/stack-bench/commands/repair-cli.ts create mode 100644 tools/stack-bench/commands/report-bugs.ts create mode 100644 tools/stack-bench/commands/reset-backend.ts create mode 100644 tools/stack-bench/commands/run-suite.ts create mode 100644 tools/stack-bench/commands/stdb-report.ts create mode 100644 tools/stack-bench/commands/stdb-review.ts create mode 100644 tools/stack-bench/commands/test-loop.ts create mode 100644 tools/stack-bench/conditions/catalog.json create mode 100644 tools/stack-bench/conditions/guidance/model-free-stub.json create mode 100644 tools/stack-bench/conditions/guidance/neutral-1.7.0.json create mode 100644 tools/stack-bench/conditions/guidance/prescribed.json create mode 100644 tools/stack-bench/conditions/repairs/scored-only.json create mode 100644 tools/stack-bench/container/Dockerfile create mode 100644 tools/stack-bench/container/binary-provenance.ts create mode 100644 tools/stack-bench/container/build-linux-cli.sh create mode 100644 tools/stack-bench/container/container-auth.ts create mode 100644 tools/stack-bench/container/credential-broker.ts create mode 100644 tools/stack-bench/container/reconcile-build-container.ts create mode 100644 tools/stack-bench/container/recover-build-container.ts create mode 100644 tools/stack-bench/container/run-build.ts create mode 100644 tools/stack-bench/container/spacetimedb-binaries.json create mode 100644 tools/stack-bench/dashboard/README.md create mode 100644 tools/stack-bench/dashboard/dashboard-model.ts create mode 100644 tools/stack-bench/dashboard/dashboard-server.ts create mode 100644 tools/stack-bench/dashboard/public/app.ts create mode 100644 tools/stack-bench/dashboard/public/fonts/inter-LICENSE.txt create mode 100644 tools/stack-bench/dashboard/public/fonts/inter-latin-variable.woff2 create mode 100644 tools/stack-bench/dashboard/public/fonts/source-code-pro-LICENSE.txt create mode 100644 tools/stack-bench/dashboard/public/fonts/source-code-pro-latin-variable.woff2 create mode 100644 tools/stack-bench/dashboard/public/index.html create mode 100644 tools/stack-bench/dashboard/public/spacetimedb-mark.svg create mode 100644 tools/stack-bench/dashboard/public/styles.css create mode 100644 tools/stack-bench/docker-compose.yaml create mode 100644 tools/stack-bench/docs/dependency-graph.html create mode 100644 tools/stack-bench/docs/stack-bench.html create mode 100644 tools/stack-bench/fixtures/app-broken/index.html create mode 100644 tools/stack-bench/fixtures/app-good/index.html create mode 100644 tools/stack-bench/fixtures/fault-agent.ts create mode 100644 tools/stack-bench/fixtures/stub-agent.ts create mode 100644 tools/stack-bench/grader/README.md create mode 100644 tools/stack-bench/grader/grade.ts create mode 100644 tools/stack-bench/grader/mutation-test.ts create mode 100644 tools/stack-bench/grader/mutations/mongodb-ecommerce-2.0.1.json create mode 100644 tools/stack-bench/grader/mutations/postgres-ecommerce-2.0.1.json create mode 100644 tools/stack-bench/grader/mutations/spacetime-ecommerce-2.0.1.json create mode 100644 tools/stack-bench/linter/fixtures/agreed.html create mode 100644 tools/stack-bench/linter/fixtures/divergent.html create mode 100644 tools/stack-bench/linter/fixtures/mock-chat.html create mode 100644 tools/stack-bench/linter/fixtures/mock-shop.html create mode 100644 tools/stack-bench/linter/fixtures/spec-accounts.html create mode 100644 tools/stack-bench/linter/lint.ts create mode 100644 tools/stack-bench/package-lock.json create mode 100644 tools/stack-bench/package.json create mode 100644 tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/mongodb-mutation.json create mode 100644 tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/mongodb-reference.json create mode 100644 tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/null.json create mode 100644 tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/postgres-mutation.json create mode 100644 tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/postgres-reference.json create mode 100644 tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/prompt-material-reuse.json create mode 100644 tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/spacetime-mutation.json create mode 100644 tools/stack-bench/qualification-evidence/ecommerce-progression-depth3-2.0.1/spacetime-reference.json create mode 100644 tools/stack-bench/reference-apps/README.md create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/client/index.html create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/client/package-lock.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/client/package.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/client/src/App.tsx create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/client/src/ProgressionPanel.tsx create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/client/src/index.css create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/client/src/main.tsx create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/client/tsconfig.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/client/vite.config.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/package.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/reference.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/server/package-lock.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/server/package.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/server/src/index.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/server/src/models.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/server/src/progression-models.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/server/src/progression.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/server/src/stock-reservations.test.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/server/src/stock-reservations.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/mongodb/server/tsconfig.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/client/index.html create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/client/package-lock.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/client/package.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/client/src/App.tsx create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/client/src/ProgressionPanel.tsx create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/client/src/index.css create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/client/src/main.tsx create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/client/tsconfig.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/client/vite.config.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/package.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/reference.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/server/drizzle.config.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/server/package-lock.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/server/package.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/server/src/auth.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/server/src/db.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/server/src/index.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/server/src/progression.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/server/src/schema.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/server/src/seed.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/postgres/server/tsconfig.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/backend/spacetimedb/package-lock.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/backend/spacetimedb/package.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/backend/spacetimedb/src/index.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/backend/spacetimedb/src/progression-policy.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/backend/spacetimedb/src/schema.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/backend/spacetimedb/tsconfig.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/index.html create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/package-lock.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/package.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/src/App.tsx create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/src/components/AdminPanel.tsx create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/src/components/AuthWidget.tsx create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/src/components/CartPanel.tsx create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/src/components/FulfilmentPanel.tsx create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/src/components/ItemCard.tsx create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/src/components/ItemDetail.tsx create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/src/components/OrdersPanel.tsx create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/src/components/ProgressionWorkbench.tsx create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/src/config.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/src/index.css create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/src/main.tsx create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/src/types.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/src/vite-env.d.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/tsconfig.json create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/client/vite.config.ts create mode 100644 tools/stack-bench/reference-apps/ecommerce/spacetime/reference.json create mode 100644 tools/stack-bench/reference-apps/registry.json create mode 100644 tools/stack-bench/reset-db.sh create mode 100644 tools/stack-bench/restart-backend.sh create mode 100644 tools/stack-bench/scripts/copy-dashboard-assets.ts create mode 100644 tools/stack-bench/src/actions/action-catalog.ts create mode 100644 tools/stack-bench/src/actions/action-contract.ts create mode 100644 tools/stack-bench/src/actions/actor-transport-action-executors.ts create mode 100644 tools/stack-bench/src/actions/browser-action-executors.ts create mode 100644 tools/stack-bench/src/actions/element-selector.ts create mode 100644 tools/stack-bench/src/actions/lifecycle-concurrency-action-executors.ts create mode 100644 tools/stack-bench/src/actions/stack-action-operations.ts create mode 100644 tools/stack-bench/src/agents/agent-adapter-contract.ts create mode 100644 tools/stack-bench/src/agents/agent-adapters.ts create mode 100644 tools/stack-bench/src/agents/agent-credentials.ts create mode 100644 tools/stack-bench/src/agents/agent-materials.ts create mode 100644 tools/stack-bench/src/agents/claude-terminal-recovery.ts create mode 100644 tools/stack-bench/src/agents/coding-session-recovery.ts create mode 100644 tools/stack-bench/src/agents/coding-session-timeouts.ts create mode 100644 tools/stack-bench/src/campaigns/campaign-admission.ts create mode 100644 tools/stack-bench/src/campaigns/campaign-compiler.ts create mode 100644 tools/stack-bench/src/campaigns/campaign-inspection.ts create mode 100644 tools/stack-bench/src/campaigns/campaign-lock.ts create mode 100644 tools/stack-bench/src/campaigns/campaign-mode.ts create mode 100644 tools/stack-bench/src/campaigns/campaign-progression-grant.ts create mode 100644 tools/stack-bench/src/campaigns/campaign-report.ts create mode 100644 tools/stack-bench/src/campaigns/campaign-run-validation.ts create mode 100644 tools/stack-bench/src/campaigns/campaign-runner.ts create mode 100644 tools/stack-bench/src/campaigns/campaign-runtime.ts create mode 100644 tools/stack-bench/src/campaigns/campaign-scheduler.ts create mode 100644 tools/stack-bench/src/campaigns/condition-compiler.ts create mode 100644 tools/stack-bench/src/campaigns/progression-reference-campaign-audit.ts create mode 100644 tools/stack-bench/src/composition/agent-visible-contract.ts create mode 100644 tools/stack-bench/src/composition/calibration-compiler.ts create mode 100644 tools/stack-bench/src/composition/composition-compiler.ts create mode 100644 tools/stack-bench/src/composition/credential-aliases.ts create mode 100644 tools/stack-bench/src/composition/definition-compiler.ts create mode 100644 tools/stack-bench/src/composition/definition-plan.ts create mode 100644 tools/stack-bench/src/composition/pack-budget.ts create mode 100644 tools/stack-bench/src/composition/pack-runtime.ts create mode 100644 tools/stack-bench/src/composition/product-config.ts create mode 100644 tools/stack-bench/src/composition/qualification-scope.ts create mode 100644 tools/stack-bench/src/composition/recipe-release.ts create mode 100644 tools/stack-bench/src/composition/recipe-selection.ts create mode 100644 tools/stack-bench/src/composition/tracks.ts create mode 100644 tools/stack-bench/src/evidence/artifacts.ts create mode 100644 tools/stack-bench/src/evidence/benchmark-run.ts create mode 100644 tools/stack-bench/src/evidence/browser-evidence.ts create mode 100644 tools/stack-bench/src/evidence/check-evidence.ts create mode 100644 tools/stack-bench/src/evidence/claude-usage-cost.ts create mode 100644 tools/stack-bench/src/evidence/cost-proof.ts create mode 100644 tools/stack-bench/src/evidence/diagnostic-sanitizer.ts create mode 100644 tools/stack-bench/src/evidence/evidence-presentation.ts create mode 100644 tools/stack-bench/src/evidence/evidence-timing.ts create mode 100644 tools/stack-bench/src/evidence/grade-report.ts create mode 100644 tools/stack-bench/src/evidence/harness-errors.ts create mode 100644 tools/stack-bench/src/evidence/mutation-analysis.ts create mode 100644 tools/stack-bench/src/evidence/mutation-checkpoint.ts create mode 100644 tools/stack-bench/src/evidence/mutation-control.ts create mode 100644 tools/stack-bench/src/evidence/mutation-rebase.ts create mode 100644 tools/stack-bench/src/evidence/mutation-shards.ts create mode 100644 tools/stack-bench/src/evidence/null-control-analysis.ts create mode 100644 tools/stack-bench/src/evidence/outcomes.ts create mode 100644 tools/stack-bench/src/evidence/pricing-authority.ts create mode 100644 tools/stack-bench/src/evidence/provenance.ts create mode 100644 tools/stack-bench/src/evidence/repair-evidence.ts create mode 100644 tools/stack-bench/src/evidence/scoring.ts create mode 100644 tools/stack-bench/src/evidence/session-metrics.ts create mode 100644 tools/stack-bench/src/grading/execution-shards.ts create mode 100644 tools/stack-bench/src/package-root.ts create mode 100644 tools/stack-bench/src/progression/dependency-definition.ts create mode 100644 tools/stack-bench/src/progression/dependency-mode.ts create mode 100644 tools/stack-bench/src/progression/dependency-score.ts create mode 100644 tools/stack-bench/src/progression/feature-catalog-selection.ts create mode 100644 tools/stack-bench/src/progression/grade-bundle-result.ts create mode 100644 tools/stack-bench/src/progression/live-progression.ts create mode 100644 tools/stack-bench/src/progression/progression-definition.ts create mode 100644 tools/stack-bench/src/progression/progression-engine.ts create mode 100644 tools/stack-bench/src/progression/progression-graph.ts create mode 100644 tools/stack-bench/src/progression/progression-recipe-selection.ts create mode 100644 tools/stack-bench/src/progression/progression-reference-audit.ts create mode 100644 tools/stack-bench/src/progression/progression-runner.ts create mode 100644 tools/stack-bench/src/progression/progression-state.ts create mode 100644 tools/stack-bench/src/references/reference-agent.ts create mode 100644 tools/stack-bench/src/references/reference-build.ts create mode 100644 tools/stack-bench/src/references/reference-fixtures.ts create mode 100644 tools/stack-bench/src/references/reference-install.ts create mode 100644 tools/stack-bench/src/references/reference-live.ts create mode 100644 tools/stack-bench/src/references/reference-qualification-audit.ts create mode 100644 tools/stack-bench/src/references/reference-selection.ts create mode 100644 tools/stack-bench/src/releases/release-bundle.ts create mode 100644 tools/stack-bench/src/releases/release-manifest.ts create mode 100644 tools/stack-bench/src/releases/release-source.ts create mode 100644 tools/stack-bench/src/runtime/backend-control.ts create mode 100644 tools/stack-bench/src/runtime/backend-lease.ts create mode 100644 tools/stack-bench/src/runtime/backend-teardown.ts create mode 100644 tools/stack-bench/src/runtime/bounded-process.ts create mode 100644 tools/stack-bench/src/runtime/coding-container-policy.ts create mode 100644 tools/stack-bench/src/runtime/command-executor.ts create mode 100644 tools/stack-bench/src/runtime/container-image.ts create mode 100644 tools/stack-bench/src/runtime/container-mount.ts create mode 100644 tools/stack-bench/src/runtime/docker-network.ts create mode 100644 tools/stack-bench/src/runtime/grading-timeout.ts create mode 100644 tools/stack-bench/src/runtime/operational-paths.ts create mode 100644 tools/stack-bench/src/runtime/path-safety.ts create mode 100644 tools/stack-bench/src/runtime/platform.ts create mode 100644 tools/stack-bench/src/runtime/preflight.ts create mode 100644 tools/stack-bench/src/runtime/readiness.ts create mode 100644 tools/stack-bench/src/runtime/recovery.ts create mode 100644 tools/stack-bench/src/runtime/repair-grant.ts create mode 100644 tools/stack-bench/src/runtime/runner-environment.ts create mode 100644 tools/stack-bench/src/runtime/sandbox.ts create mode 100644 tools/stack-bench/src/runtime/source-checkpoint.ts create mode 100644 tools/stack-bench/src/runtime/source-snapshot.ts create mode 100644 tools/stack-bench/src/runtime/spacetime-layout.ts create mode 100644 tools/stack-bench/src/runtime/spacetime-target.ts create mode 100644 tools/stack-bench/src/stacks/backend-reset-guard.ts create mode 100644 tools/stack-bench/src/stacks/backend-reset.ts create mode 100644 tools/stack-bench/src/stacks/backends/mongodb-adapter.ts create mode 100644 tools/stack-bench/src/stacks/backends/mongodb-identity.ts create mode 100644 tools/stack-bench/src/stacks/backends/mongodb-operations.ts create mode 100644 tools/stack-bench/src/stacks/backends/postgres-adapter.ts create mode 100644 tools/stack-bench/src/stacks/backends/postgres-identity.ts create mode 100644 tools/stack-bench/src/stacks/backends/postgres-operations.ts create mode 100644 tools/stack-bench/src/stacks/backends/spacetime-adapter.ts create mode 100644 tools/stack-bench/src/stacks/backends/spacetime-identity.ts create mode 100644 tools/stack-bench/src/stacks/backends/spacetime-operations.ts create mode 100644 tools/stack-bench/src/stacks/backends/stub-adapter.ts create mode 100644 tools/stack-bench/src/stacks/backends/stub-identity.ts create mode 100644 tools/stack-bench/src/stacks/database-containers.ts create mode 100644 tools/stack-bench/src/stacks/hosted-database-identity.ts create mode 100644 tools/stack-bench/src/stacks/stack-adapter-common.ts create mode 100644 tools/stack-bench/src/stacks/stack-adapter-contract.ts create mode 100644 tools/stack-bench/src/stacks/stack-adapters.ts create mode 100644 tools/stack-bench/src/stacks/stack-agent-operations.ts create mode 100644 tools/stack-bench/src/stacks/stack-backend-operations.ts create mode 100644 tools/stack-bench/src/stacks/stack-identities.ts create mode 100644 tools/stack-bench/src/stacks/stack-lease-capabilities.ts create mode 100644 tools/stack-bench/src/stacks/stack-lifecycle-operations.ts create mode 100644 tools/stack-bench/src/stacks/stack-orchestrator-operations.ts create mode 100644 tools/stack-bench/src/stacks/stack-reference-operations.ts create mode 100644 tools/stack-bench/src/stacks/stack-teardown-operations.ts create mode 100644 tools/stack-bench/tests/action-contract.test.ts create mode 100644 tools/stack-bench/tests/actor-transport-action-executors.test.ts create mode 100644 tools/stack-bench/tests/advanced-support-feature-packs.test.ts create mode 100644 tools/stack-bench/tests/agent-adapters.test.ts create mode 100644 tools/stack-bench/tests/agent-credentials.test.ts create mode 100644 tools/stack-bench/tests/agent-database.test.ts create mode 100644 tools/stack-bench/tests/agent-guidance.test.ts create mode 100644 tools/stack-bench/tests/agent-materials.test.ts create mode 100644 tools/stack-bench/tests/agent-runtime-identity.test.ts create mode 100644 tools/stack-bench/tests/agent-task-selection.test.ts create mode 100644 tools/stack-bench/tests/artifacts.test.ts create mode 100644 tools/stack-bench/tests/backend-control.test.ts create mode 100644 tools/stack-bench/tests/backend-lease.test.ts create mode 100644 tools/stack-bench/tests/bench-policy.test.ts create mode 100644 tools/stack-bench/tests/binary-provenance.test.ts create mode 100644 tools/stack-bench/tests/browser-action-executors.test.ts create mode 100644 tools/stack-bench/tests/browser-evidence.test.ts create mode 100644 tools/stack-bench/tests/build-container-reconciliation.test.ts create mode 100644 tools/stack-bench/tests/calibration-compiler.test.ts create mode 100644 tools/stack-bench/tests/campaign-admission.test.ts create mode 100644 tools/stack-bench/tests/campaign-cli.test.ts create mode 100644 tools/stack-bench/tests/campaign-compiler.test.ts create mode 100644 tools/stack-bench/tests/campaign-lock.test.ts create mode 100644 tools/stack-bench/tests/campaign-mode.test.ts create mode 100644 tools/stack-bench/tests/campaign-progression-grant.test.ts create mode 100644 tools/stack-bench/tests/campaign-report.test.ts create mode 100644 tools/stack-bench/tests/campaign-runner.test.ts create mode 100644 tools/stack-bench/tests/campaign-runtime.test.ts create mode 100644 tools/stack-bench/tests/campaign-scheduler.test.ts create mode 100644 tools/stack-bench/tests/cart-quantity-mutations.test.ts create mode 100644 tools/stack-bench/tests/catalog-search-feature-contracts.test.ts create mode 100644 tools/stack-bench/tests/check-evidence.test.ts create mode 100644 tools/stack-bench/tests/claude-terminal-recovery.test.ts create mode 100644 tools/stack-bench/tests/coding-interruption-recovery.test.ts create mode 100644 tools/stack-bench/tests/composition-cli.test.ts create mode 100644 tools/stack-bench/tests/composition-compiler.test.ts create mode 100644 tools/stack-bench/tests/condition-compiler.test.ts create mode 100644 tools/stack-bench/tests/container-auth.test.ts create mode 100644 tools/stack-bench/tests/container-image.test.ts create mode 100644 tools/stack-bench/tests/container-mount.test.ts create mode 100644 tools/stack-bench/tests/controller.test.ts create mode 100644 tools/stack-bench/tests/cost-ledger.test.ts create mode 100644 tools/stack-bench/tests/credential-aliases.test.ts create mode 100644 tools/stack-bench/tests/credential-broker.test.ts create mode 100644 tools/stack-bench/tests/dashboard.test.ts create mode 100644 tools/stack-bench/tests/database-containers.test.ts create mode 100644 tools/stack-bench/tests/definition-compiler.test.ts create mode 100644 tools/stack-bench/tests/definition-snapshots.test.ts create mode 100644 tools/stack-bench/tests/dependency-campaign-live.test.ts create mode 100644 tools/stack-bench/tests/dependency-mode.test.ts create mode 100644 tools/stack-bench/tests/dependency-neutral-prompt.test.ts create mode 100644 tools/stack-bench/tests/dependency-volume.test.ts create mode 100644 tools/stack-bench/tests/diagnostic-sanitizer.test.ts create mode 100644 tools/stack-bench/tests/docker-network.test.ts create mode 100644 tools/stack-bench/tests/ecommerce-progression-2.0.1.test.ts create mode 100644 tools/stack-bench/tests/ecommerce-progression-campaign.test.ts create mode 100644 tools/stack-bench/tests/ecommerce-progression.test.ts create mode 100644 tools/stack-bench/tests/element-selector.test.ts create mode 100644 tools/stack-bench/tests/evidence-timing.test.ts create mode 100644 tools/stack-bench/tests/execution-shards.test.ts create mode 100644 tools/stack-bench/tests/external-data-sync-candidate.test.ts create mode 100644 tools/stack-bench/tests/feature-catalog-selection.test.ts create mode 100644 tools/stack-bench/tests/fixtures/definitions/all-actions.json create mode 100644 tools/stack-bench/tests/fixtures/dependency-model-free-campaign.json create mode 100644 tools/stack-bench/tests/fixtures/resource-lock-process.ts create mode 100644 tools/stack-bench/tests/fixtures/spacetime-module/package.json create mode 100644 tools/stack-bench/tests/fixtures/spacetime-module/src/index.ts create mode 100644 tools/stack-bench/tests/fixtures/spacetime-module/tsconfig.json create mode 100644 tools/stack-bench/tests/grader-cleanup.test.ts create mode 100644 tools/stack-bench/tests/grading-timeout.test.ts create mode 100644 tools/stack-bench/tests/harness-errors.test.ts create mode 100644 tools/stack-bench/tests/harness-regressions.test.ts create mode 100644 tools/stack-bench/tests/helpers/progression-source.ts create mode 100644 tools/stack-bench/tests/identity-staff-feature-contracts.test.ts create mode 100644 tools/stack-bench/tests/inventory-order-feature-contracts.test.ts create mode 100644 tools/stack-bench/tests/isolation.test.ts create mode 100644 tools/stack-bench/tests/l1-concurrency-completion.test.ts create mode 100644 tools/stack-bench/tests/l3-composition-candidate.test.ts create mode 100644 tools/stack-bench/tests/leak-audit.test.ts create mode 100644 tools/stack-bench/tests/lifecycle-concurrency-action-executors.test.ts create mode 100644 tools/stack-bench/tests/linter-coverage.test.ts create mode 100644 tools/stack-bench/tests/live-progression.test.ts create mode 100644 tools/stack-bench/tests/modular-selection.test.ts create mode 100644 tools/stack-bench/tests/module-layout.test.ts create mode 100644 tools/stack-bench/tests/mongodb-progression-reference.test.ts create mode 100644 tools/stack-bench/tests/mutation-analysis.test.ts create mode 100644 tools/stack-bench/tests/mutation-checkpoint.test.ts create mode 100644 tools/stack-bench/tests/mutation-manifests.test.ts create mode 100644 tools/stack-bench/tests/mutation-preflight.test.ts create mode 100644 tools/stack-bench/tests/mutation-rebase.test.ts create mode 100644 tools/stack-bench/tests/mutation-shards.test.ts create mode 100644 tools/stack-bench/tests/neutral-guidance.test.ts create mode 100644 tools/stack-bench/tests/null-control-analysis.test.ts create mode 100644 tools/stack-bench/tests/null-control-cli.test.ts create mode 100644 tools/stack-bench/tests/open-list-live-candidate.test.ts create mode 100644 tools/stack-bench/tests/operational-paths.test.ts create mode 100644 tools/stack-bench/tests/operations-feature-contracts.test.ts create mode 100644 tools/stack-bench/tests/outcomes.test.ts create mode 100644 tools/stack-bench/tests/pack-budget.test.ts create mode 100644 tools/stack-bench/tests/pack-runtime.test.ts create mode 100644 tools/stack-bench/tests/package-root.test.ts create mode 100644 tools/stack-bench/tests/path-safety.test.ts create mode 100644 tools/stack-bench/tests/platform-process-tree.test.ts create mode 100644 tools/stack-bench/tests/preflight-command.test.ts create mode 100644 tools/stack-bench/tests/preflight.test.ts create mode 100644 tools/stack-bench/tests/pricing-authority.test.ts create mode 100644 tools/stack-bench/tests/product-config.test.ts create mode 100644 tools/stack-bench/tests/progression-activity-feedback-packs.test.ts create mode 100644 tools/stack-bench/tests/progression-behavior-evidence-packs.test.ts create mode 100644 tools/stack-bench/tests/progression-definition.test.ts create mode 100644 tools/stack-bench/tests/progression-depth3-composition.test.ts create mode 100644 tools/stack-bench/tests/progression-feature-packs.test.ts create mode 100644 tools/stack-bench/tests/progression-fulfilment-pack.test.ts create mode 100644 tools/stack-bench/tests/progression-grade-bundle.test.ts create mode 100644 tools/stack-bench/tests/progression-graph.test.ts create mode 100644 tools/stack-bench/tests/progression-integration.test.ts create mode 100644 tools/stack-bench/tests/progression-mutations.test.ts create mode 100644 tools/stack-bench/tests/progression-order-cancellation-pack.test.ts create mode 100644 tools/stack-bench/tests/progression-pagination-compatibility.test.ts create mode 100644 tools/stack-bench/tests/progression-postgres-reference.test.ts create mode 100644 tools/stack-bench/tests/progression-product-packs.test.ts create mode 100644 tools/stack-bench/tests/progression-promotion-notification-packs.test.ts create mode 100644 tools/stack-bench/tests/progression-reference-audit.test.ts create mode 100644 tools/stack-bench/tests/progression-reference-campaign-audit.test.ts create mode 100644 tools/stack-bench/tests/progression-runner.test.ts create mode 100644 tools/stack-bench/tests/progression-shopping-packs.test.ts create mode 100644 tools/stack-bench/tests/progression-spacetime-policy.test.ts create mode 100644 tools/stack-bench/tests/progression-spacetime-reference.test.ts create mode 100644 tools/stack-bench/tests/progression-state.test.ts create mode 100644 tools/stack-bench/tests/progression-survivor-regressions.test.ts create mode 100644 tools/stack-bench/tests/progression-timed-feature-packs.test.ts create mode 100644 tools/stack-bench/tests/provenance.test.ts create mode 100644 tools/stack-bench/tests/qualification-cli.test.ts create mode 100644 tools/stack-bench/tests/qualification-scope.test.ts create mode 100644 tools/stack-bench/tests/readiness.test.ts create mode 100644 tools/stack-bench/tests/recipe-release.test.ts create mode 100644 tools/stack-bench/tests/recovery-command.test.ts create mode 100644 tools/stack-bench/tests/recovery.test.ts create mode 100644 tools/stack-bench/tests/reference-agent.test.ts create mode 100644 tools/stack-bench/tests/reference-fixtures.test.ts create mode 100644 tools/stack-bench/tests/reference-install.test.ts create mode 100644 tools/stack-bench/tests/reference-live.test.ts create mode 100644 tools/stack-bench/tests/release-bundle.test.ts create mode 100644 tools/stack-bench/tests/release-manifest.test.ts create mode 100644 tools/stack-bench/tests/release-source.test.ts create mode 100644 tools/stack-bench/tests/repair-cli.test.ts create mode 100644 tools/stack-bench/tests/repair-grant.test.ts create mode 100644 tools/stack-bench/tests/report-bugs.test.ts create mode 100644 tools/stack-bench/tests/reset-backend.test.ts create mode 100644 tools/stack-bench/tests/reset-safety.test.ts create mode 100644 tools/stack-bench/tests/run-session-artifacts.test.ts create mode 100644 tools/stack-bench/tests/run-suite.test.ts create mode 100644 tools/stack-bench/tests/runner-environment.test.ts create mode 100644 tools/stack-bench/tests/sandbox.test.ts create mode 100644 tools/stack-bench/tests/scenario-contract-check.test.ts create mode 100644 tools/stack-bench/tests/scoring.test.ts create mode 100644 tools/stack-bench/tests/selected-grade.test.ts create mode 100644 tools/stack-bench/tests/sequential-l1-calibration.test.ts create mode 100644 tools/stack-bench/tests/sequential-l2-calibration.test.ts create mode 100644 tools/stack-bench/tests/session-metrics.test.ts create mode 100644 tools/stack-bench/tests/snapshots/definitions/all-actions.snapshot.json create mode 100644 tools/stack-bench/tests/snapshots/definitions/chat.snapshot.json create mode 100644 tools/stack-bench/tests/snapshots/definitions/ecommerce.snapshot.json create mode 100644 tools/stack-bench/tests/snapshots/definitions/loop.snapshot.json create mode 100644 tools/stack-bench/tests/source-checkpoint.test.ts create mode 100644 tools/stack-bench/tests/source-snapshot.test.ts create mode 100644 tools/stack-bench/tests/spacetime-guidance.test.ts create mode 100644 tools/stack-bench/tests/stack-action-operations.test.ts create mode 100644 tools/stack-bench/tests/stack-adapters.test.ts create mode 100644 tools/stack-bench/tests/stack-reference-operations.test.ts create mode 100644 tools/stack-bench/tests/support-entry-feature-packs.test.ts create mode 100644 tools/stack-bench/tests/tracks.test.ts create mode 100644 tools/stack-bench/tracks/chat/LEVELS.md create mode 100644 tools/stack-bench/tracks/chat/contracts/01-basic-chat.json create mode 100644 tools/stack-bench/tracks/chat/contracts/02-authorization.json create mode 100644 tools/stack-bench/tracks/chat/contracts/appendix-01.md create mode 100644 tools/stack-bench/tracks/chat/contracts/appendix-02.md create mode 100644 tools/stack-bench/tracks/chat/prompts/01-basic-chat.md create mode 100644 tools/stack-bench/tracks/chat/prompts/02-authorization.md create mode 100644 tools/stack-bench/tracks/chat/prompts/03-contention.md create mode 100644 tools/stack-bench/tracks/chat/prompts/04-deferred.md create mode 100644 tools/stack-bench/tracks/chat/prompts/05-volume.md create mode 100644 tools/stack-bench/tracks/chat/scenarios/01-basic-chat.json create mode 100644 tools/stack-bench/tracks/chat/scenarios/01-contention-wip.json create mode 100644 tools/stack-bench/tracks/chat/scenarios/01-delivery.json create mode 100644 tools/stack-bench/tracks/chat/scenarios/01-invariants.json create mode 100644 tools/stack-bench/tracks/chat/scenarios/01-systems.json create mode 100644 tools/stack-bench/tracks/chat/scenarios/02-features.json create mode 100644 tools/stack-bench/tracks/chat/scenarios/02-invariants.json create mode 100644 tools/stack-bench/tracks/chat/scenarios/02-systems.json create mode 100644 tools/stack-bench/tracks/chat/track.json create mode 100644 tools/stack-bench/tracks/chat/walk.ts create mode 100644 tools/stack-bench/tracks/ecommerce/LEVELS.md create mode 100644 tools/stack-bench/tracks/ecommerce/composition/README.md create mode 100644 tools/stack-bench/tracks/ecommerce/composition/calibrations/progression-depth3-2.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/calibrations/sequential-l1-2.5.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/calibrations/sequential-l2-1.6.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/candidates.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/fixtures/operations-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/fixtures/storefront-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/framing/smoke-requirements.md create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/feature-accounts-1.1.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/feature-accounts-1.2.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/feature-cart-2.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/feature-cart-checkout-1.2.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/feature-catalog-1.1.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/feature-catalog-discovery-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/feature-catalog-items-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/feature-checkout-2.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/feature-purchasing-1.1.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/feature-purchasing-1.2.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/feature-reviews-1.1.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/feature-reviews-1.2.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/feature-warehouse-admin-1.1.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/feature-warehouse-admin-1.2.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/inventory-operations-features-1.2.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/inventory-operations-specifications-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l2-inventory-dashboard-2.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l2-order-cancellation-features-1.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l2-price-history-features-2.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l2-recommendations-2.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l2-sales-dashboard-2.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l2-stock-transfers-features-1.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l3-cart-expiration-features-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l3-cart-expiration-features-2.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l3-deferred-access-specifications-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l3-deferred-durability-specifications-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l3-deferred-integrity-specifications-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l3-order-delivery-features-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l3-order-delivery-features-1.1.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l3-order-returns-features-1.1.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l3-reservations-features-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l3-reservations-features-2.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l3-scheduled-restocks-features-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l3-scheduled-restocks-features-1.1.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/l3-server-time-specifications-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/operations-access-features-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/operations-access-specifications-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-automatic-reorder-2.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-cancellation-accounting-specifications-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-cancellation-queue-specifications-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-cart-recovery-2.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-catalog-management-1.0.2.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-customer-profile-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-delivery-notifications-1.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-faceted-search-1.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-fulfilment-queue-1.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-inventory-conservation-specifications-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-managed-support-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-notification-preferences-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-operations-access-specifications-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-order-support-1.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-payment-records-2.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-personalized-recommendations-2.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-price-accounting-specifications-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-promotion-checkout-2.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-promotion-reporting-2.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-promotion-rules-1.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-recommendation-feedback-2.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-staff-access-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-staff-activity-1.0.2.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-staff-roles-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-stock-alerts-1.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-support-history-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-support-intake-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-support-refunds-1.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/progression-support-triage-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/returns-pricing-features-1.1.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/returns-pricing-specifications-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/spec-access-control-1.2.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/spec-access-control-2.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/spec-concurrency-safety-1.3.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/spec-concurrency-safety-2.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/spec-external-data-sync-1.1.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/spec-external-data-sync-1.1.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/spec-live-state-1.2.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/spec-live-state-2.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/spec-state-durability-1.1.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/spec-state-durability-2.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/spec-transactional-integrity-1.3.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/packs/spec-transactional-integrity-1.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/promotions.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/recipes/progression-catalog-2.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/recipes/progression-depth3-2.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/recipes/sequential-l1-2.5.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/recipes/sequential-l2-1.6.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/composition/recipes/sequential-l3-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/accounts-1.2.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/appendix-02.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/cart-2.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/cart-expiration-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/catalog-discovery-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/catalog-items-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/catalog-management-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/checkout-2.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/customer-profile-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/delivery-notifications-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/faceted-search-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/inventory-dashboard-2.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/l2-order-hooks-1.4.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/l3-hooks-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/managed-support-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/notification-preferences-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/operations-access-1.1.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/order-cancellation-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/order-delivery-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/order-returns-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/order-support-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/payment-records-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/price-history-2.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/progression-automatic-reorder-2.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/progression-cart-recovery-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/progression-personalized-recommendations-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/promotion-checkout-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/promotion-reporting-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/promotion-rules-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/purchasing-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/recommendation-feedback-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/recommendations-2.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/reservations-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/reviews-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/sales-dashboard-2.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/scheduled-restocks-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/staff-access-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/staff-activity-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/staff-roles-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/stock-alerts-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/stock-transfers-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/support-history-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/support-intake-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/support-refunds-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/support-triage-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/contracts/warehouse-administration-1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/progression/ecommerce-2.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/02-operations.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/03-scheduled.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/fragments/named-actions-inventory-operations-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/fragments/named-actions-returns-pricing-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/fragments/named-actions-scheduled-restocks-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/fragments/startup-probe-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/l2-action-inputs.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/accounts-1.2.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/cancellation-conservation-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/cancellation-queue-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/cart-2.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/cart-expiration-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/catalog-discovery-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/catalog-items-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/catalog-management-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/checkout-2.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/customer-profile-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/delivery-notifications-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/faceted-search-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/inventory-dashboard-2.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/l1-action-inputs-1.1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/l1-features-1.1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/l1-features.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/l1-framing.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/l1-hooks.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/l1-specifications-1.1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/l1-specifications.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/l3-features-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/l3-specifications-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/managed-support-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/notification-preferences-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/operations-access-1.1.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/order-accounting-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/order-cancellation-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/order-delivery-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/order-ownership-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/order-returns-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/order-support-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/payment-records-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/price-authorization-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/price-history-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/progression-automatic-reorder-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/progression-cart-recovery-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/progression-framing-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/progression-personalized-recommendations-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/promotion-checkout-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/promotion-reporting-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/promotion-rules-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/purchasing-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/recommendation-feedback-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/recommendations-2.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/reservations-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/reviews-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/sales-dashboard-2.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/scheduled-restocks-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/shipping-authorization-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/staff-access-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/staff-activity-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/staff-roles-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/stock-alerts-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/stock-conservation-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/stock-transfers-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/support-history-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/support-intake-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/support-refunds-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/support-triage-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/transfer-authorization-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/prompts/modular/warehouse-administration-1.0.0.md create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-account-create-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-account-duplicate-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-account-password-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-account-reload-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-account-signout-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-account-state-reconnect-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-account-state-reload-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-admin-write-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-admin-write-staff-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-books-balance-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-buying-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-cart-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-cart-boundary-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-catalog-ranking-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-catalog-search-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-catalog-values-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-contention.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-core-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-duplicate-checkout-2.3.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-duplicate-checkout-2.3.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-external-live-sync-1.1.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-external-reconnect-sync-1.1.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-external-reload-sync-1.1.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-external-server-restart-sync-1.1.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-features.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-invariants.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-last-unit-2.3.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-last-unit-2.3.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-open-list-live-2.3.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-order-ownership-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-purchase-attribution-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-purchase-session-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-restock-race-2.3.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-review-eligibility-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-review-rating-live-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-review-uniqueness-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-review-visibility-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-server-price-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-systems.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-warehouse-admin-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-warehouse-admin-staff-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-warehouse-stock-live-2.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/01-warehouse-stock-live-staff-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-cancellation-queue-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-features.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-fulfilment-access-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-fulfilment-live-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-fulfilment-ship-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-invariants.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-live-price-1.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-low-stock-1.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-operational-best-sellers-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-operational-category-totals-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-operational-recommendations-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-order-cancellation-core-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-order-cancellation-history-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-paid-price-history-1.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-queue-warehouse-1.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-self-contained-1.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-server-actions-1.1.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-strengthened-1.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/02-transfer-totals-1.4.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/03-cart-expiration-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/03-deferred-access-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/03-deferred-durability-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/03-deferred-integrity-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/03-features.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/03-invariants.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/03-order-delivery-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/03-reservations-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/03-scheduled-restocks-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/03-server-time-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-account-state-reconnect-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-account-state-reload-1.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-automatic-reorder-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-books-balance-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-cart-checkout-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-cart-recovery-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-catalog-management-1.0.1.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-core-business-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-customer-profile-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-delivery-notifications-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-faceted-filters-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-faceted-pagination-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-managed-support-privacy-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-managed-support-shared-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-notification-preferences-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-open-list-live-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-order-return-boundary-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-order-return-complete-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-order-support-boundary-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-order-support-owned-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-personalized-recommendations-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-price-cart-checkout-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-promotion-checkout-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-promotion-reporting-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-promotion-rules-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-purchasing-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-recommendation-feedback-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-signed-out-purchase-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-staff-access-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-staff-activity-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-staff-roles-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-stock-alerts-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-stock-limit-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-support-history-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-support-intake-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-support-refunds-access-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-support-refunds-accounting-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-support-refunds-resolution-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/scenarios/progression-support-triage-1.0.0.json create mode 100644 tools/stack-bench/tracks/ecommerce/track.json create mode 100644 tools/stack-bench/tracks/ecommerce/walk.ts create mode 100644 tools/stack-bench/tracks/loop/contracts/01-loop.json create mode 100644 tools/stack-bench/tracks/loop/prompts/01-loop.md create mode 100644 tools/stack-bench/tracks/loop/scenarios/01-loop.json create mode 100644 tools/stack-bench/tracks/loop/track.json create mode 100644 tools/stack-bench/tracks/loop/walk.ts create mode 100644 tools/stack-bench/tsconfig.build.json create mode 100644 tools/stack-bench/tsconfig.json diff --git a/.gitattributes b/.gitattributes index 05b1a132398..1e9bf994ead 100644 --- a/.gitattributes +++ b/.gitattributes @@ -2,3 +2,5 @@ **/ModuleBindings/** linguist-generated=true eol=lf /docs/llms/** linguist-generated=true /docs/llms/*-details.json linguist-generated=false +/tools/stack-bench/** text eol=lf +/tools/stack-bench/**/*.woff2 -text -diff diff --git a/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md index 3a31183133f..62f86ff36fc 100644 --- a/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/typescript-client/SKILL.md @@ -18,7 +18,7 @@ Generated bindings convert snake_case names to camelCase, including row fields: ## React: main.tsx ```typescript -import React, { useEffect, useMemo } from 'react'; +import React, { useMemo } from 'react'; import ReactDOM from 'react-dom/client'; import { SpacetimeDBProvider } from 'spacetimedb/react'; import { DbConnection } from './module_bindings'; @@ -30,6 +30,7 @@ function Root() { DbConnection.builder() .withUri(SPACETIMEDB_URI) .withDatabaseName(MODULE_NAME) + // Reuse the token issued on the previous connection. .withToken(localStorage.getItem('auth_token') || undefined), [] ); @@ -46,6 +47,7 @@ ReactDOM.createRoot(document.getElementById('root')!).render(); ## React: App.tsx ```typescript +import { useEffect } from 'react'; import { useTable, useSpacetimeDB } from 'spacetimedb/react'; import { DbConnection, tables } from './module_bindings'; @@ -53,7 +55,7 @@ function App() { const { isActive, identity: myIdentity, token, getConnection } = useSpacetimeDB(); const conn = getConnection() as DbConnection | null; - // Save auth token + // Persist the issued token for the next page load. useEffect(() => { if (token) localStorage.setItem('auth_token', token); }, [token]); // Subscribe when connected. Prefer typed query builders over raw SQL diff --git a/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md b/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md index c9f2e7343fd..ba24bf781d5 100644 --- a/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md +++ b/codex-plugin/plugins/spacetimedb/skills/typescript-server/SKILL.md @@ -98,6 +98,10 @@ Every column is a `t` builder value: Modifiers: `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')`, `.default(value)`. +`.primaryKey()` and `.unique()` apply to one column. For uniqueness across +multiple columns, use a surrogate key, the multi-column index below, and a +reducer that rejects an existing index match before inserting. + Use `.default(value)` only for a newly appended migration-safe field. Do not put defaults on primary-key, unique, or auto-increment columns. Optional columns: `nickname: t.option(t.string())` @@ -130,7 +134,9 @@ export { default } from './schema'; // re-export the schema for the module ent ## Reducers -Reducers are created with `spacetimedb.reducer(...)`; the export name becomes the reducer name: +Reducers are created with `spacetimedb.reducer(...)`. An exported `signUp` +becomes `signUp` in generated clients and `sign_up` in `spacetime call` and +`describe`: ```typescript export const createEntity = spacetimedb.reducer( @@ -278,9 +284,22 @@ const Shape = t.enum('Shape', { A client subscribing to a view receives only the rows it returns. Use a per-user view (keyed on `ctx.sender`) for per-viewer access control: deleting a row it depends on (e.g. a membership row) automatically drops the rows it was exposing from that client. +Use index accessors in views. Do not scan a whole table with `.iter()` when an +indexed lookup can select the required rows. `t.row(...)` and `t.object(...)` return schema builders, not TypeScript runtime row types. Let a view callback infer its result, or annotate a separately declared structural type such as `Array<{ sku: bigint; label: string }>`. A named output type must not reuse the generated PascalCase name of its view accessor (for example, reserve `DiscountedProduct` for a `discounted_product` view). +A view context is `ViewCtx` (and `AnonymousViewCtx`), both exported from +`spacetimedb/server`. It carries `sender`, a read-only `db`, and `from`; it is +not a `ReducerCtx`, so a helper shared between a reducer and a view must accept +either: + +```typescript +import type { ReducerCtx, ViewCtx, InferSchema } from 'spacetimedb/server'; +type S = InferSchema; +function stockOf(ctx: ReducerCtx | ViewCtx, itemId: bigint) { ... } +``` + Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arguments: view options, the declared return schema, and the callback. ```typescript @@ -288,7 +307,7 @@ Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arg export const activeUsers = spacetimedb.anonymousView( { name: 'active_users', public: true }, t.array(entity.rowType), - (ctx) => [...ctx.db.entity.iter()].filter(e => e.active) + (ctx) => [...ctx.db.entity.active.filter(true)] // active: t.bool().index('btree') ); // Per-user view (varies by ctx.sender): diff --git a/crates/bindings-typescript/src/lib/query.ts b/crates/bindings-typescript/src/lib/query.ts index bb93b0e6ce3..0c17e3d145a 100644 --- a/crates/bindings-typescript/src/lib/query.ts +++ b/crates/bindings-typescript/src/lib/query.ts @@ -248,19 +248,21 @@ export type NamespacedQueryBuilder = * A runtime reference to a table. This materializes the RowExpr for us. * TODO: Maybe add the full SchemaDef to the type signature depending on how joins will work. */ -export type TableRef = Readonly<{ - type: 'table'; - sourceName: TableDef['sourceName']; - accessorName: string; - cols: RowExpr; - indexedCols: IndexedRowExpr; - tableDef: TableDef; +// Keep this named so TypeScript diagnostics show `TableRef` instead of its +// expanded structure. +export interface TableRef { + readonly type: 'table'; + readonly sourceName: TableDef['sourceName']; + readonly accessorName: string; + readonly cols: RowExpr; + readonly indexedCols: IndexedRowExpr; + readonly tableDef: TableDef; // Delegated UntypedTableDef properties for compatibility. - columns: TableDef['columns']; - indexes: TableDef['indexes']; - rowType: TableDef['rowType']; - constraints: any; -}>; + readonly columns: TableDef['columns']; + readonly indexes: TableDef['indexes']; + readonly rowType: TableDef['rowType']; + readonly constraints: any; +} class TableRefImpl implements TableRef, From diff --git a/crates/bindings-typescript/src/sdk/connection_manager.ts b/crates/bindings-typescript/src/sdk/connection_manager.ts index 211b01b5add..42febd5d221 100644 --- a/crates/bindings-typescript/src/sdk/connection_manager.ts +++ b/crates/bindings-typescript/src/sdk/connection_manager.ts @@ -145,9 +145,7 @@ class ConnectionManagerImpl { clearTimeout(managed.reconnectTimer); managed.reconnectTimer = null; managed.reconnectAttempt = 0; - if (managed.builder) { - this.#buildManagedConnection(managed, managed.builder); - } + this.#reconnectManagedConnection(managed); continue; } @@ -179,9 +177,7 @@ class ConnectionManagerImpl { connection.disconnect(); this.#updateState(managed, { isActive: false }); managed.reconnectAttempt = 0; - if (managed.builder) { - this.#buildManagedConnection(managed, managed.builder); - } + this.#reconnectManagedConnection(managed); } /** Generates a unique key for a connection based on URI and module name. */ @@ -294,6 +290,14 @@ class ConnectionManagerImpl { } } + /** Reconnect with the issued token. Explicit rebuilds use the caller's token. */ + #reconnectManagedConnection(managed: ManagedConnection): void { + if (!managed.builder) return; + const token = managed.state.token; + if (token) managed.builder.withToken(token); + this.#buildManagedConnection(managed, managed.builder); + } + #buildManagedConnection>( managed: ManagedConnection, builder: DbConnectionBuilder @@ -349,7 +353,7 @@ class ConnectionManagerImpl { return; } - this.#buildManagedConnection(managed, managed.builder); + this.#reconnectManagedConnection(managed); }, delay); } diff --git a/crates/bindings-typescript/tests/connection_manager_liveness.test.ts b/crates/bindings-typescript/tests/connection_manager_liveness.test.ts index ea24ea6887e..e9f76a909e9 100644 --- a/crates/bindings-typescript/tests/connection_manager_liveness.test.ts +++ b/crates/bindings-typescript/tests/connection_manager_liveness.test.ts @@ -12,7 +12,7 @@ type ErrorContextInterface = { isActive: boolean }; class MockConnection { isActive = false; identity = undefined; - token = undefined; + token: string | undefined; connectionId = ConnectionId.random(); isDisconnectRequested = false; disconnected = false; @@ -28,6 +28,10 @@ class MockConnection { (ctx: ErrorContextInterface, error: Error) => void >(); + constructor(private readonly issuedToken?: string) { + this.token = undefined; + } + get isSocketClosed(): boolean { return this.socketClosed; } @@ -63,6 +67,7 @@ class MockConnection { simulateConnect(): void { this.isActive = true; + this.token = this.issuedToken; for (const cb of this.#onConnect) cb(this); } simulateDisconnect(error?: Error): void { @@ -74,7 +79,9 @@ class MockConnection { class MockBuilder { buildCount = 0; + presentedTokens: Array = []; connections: MockConnection[] = []; + #token: string | undefined; #onConnect = new Set<(conn: MockConnection) => void>(); #onDisconnect = new Set< @@ -84,9 +91,17 @@ class MockBuilder { (ctx: ErrorContextInterface, error: Error) => void >(); + constructor(private readonly issuedToken?: string) {} + + withToken(token?: string): MockBuilder { + this.#token = token; + return this; + } + build(): MockConnection { - const connection = new MockConnection(); + const connection = new MockConnection(this.issuedToken); this.buildCount += 1; + this.presentedTokens.push(this.#token); this.connections.push(connection); for (const cb of this.#onConnect) connection.register('connect', cb); for (const cb of this.#onDisconnect) connection.register('disconnect', cb); @@ -201,6 +216,32 @@ describe('ConnectionManager liveness recovery', () => { ConnectionManager.release(key); }); + test('reuses the issued token when reviving a dead socket', () => { + const key = nextKey(); + const builder = new MockBuilder('issued-token'); + const first = retain(key, builder); + expect(builder.presentedTokens).toEqual([undefined]); + + first.simulateConnect(); + first.socketClosed = true; + fire('win:online'); + + expect(builder.presentedTokens).toEqual([undefined, 'issued-token']); + ConnectionManager.release(key); + }); + + test('an explicit rebuild uses the caller token', () => { + const key = nextKey(); + const firstBuilder = new MockBuilder('issued-token'); + retain(key, firstBuilder).simulateConnect(); + + const replacement = new MockBuilder().withToken('caller-token'); + ConnectionManager.rebuild(key, replacement as any); + + expect(replacement.presentedTokens).toEqual(['caller-token']); + ConnectionManager.release(key); + }); + test('does not rebuild a healthy connection on resume', () => { const key = nextKey(); const builder = new MockBuilder(); diff --git a/crates/bindings-typescript/tests/table_ref_error_message.test.ts b/crates/bindings-typescript/tests/table_ref_error_message.test.ts new file mode 100644 index 00000000000..009a9c192dd --- /dev/null +++ b/crates/bindings-typescript/tests/table_ref_error_message.test.ts @@ -0,0 +1,79 @@ +import { mkdtempSync, rmSync, writeFileSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import path from 'node:path'; +import { fileURLToPath } from 'node:url'; +import * as ts from 'typescript'; +import { describe, expect, it } from 'vitest'; + +const bindingsRoot = path.resolve( + path.dirname(fileURLToPath(import.meta.url)), + '..' +); + +function runTypecheck(source: string) { + const tmpDir = mkdtempSync(path.join(tmpdir(), 'stdb-tableref-diag-')); + const reproPath = path.join(tmpDir, 'repro.ts'); + writeFileSync(reproPath, source); + + try { + const options: ts.CompilerOptions = { + target: ts.ScriptTarget.ESNext, + module: ts.ModuleKind.ESNext, + strict: true, + noEmit: true, + skipLibCheck: true, + forceConsistentCasingInFileNames: true, + allowImportingTsExtensions: true, + noImplicitAny: true, + moduleResolution: ts.ModuleResolutionKind.Bundler, + useDefineForClassFields: true, + verbatimModuleSyntax: true, + isolatedModules: true, + }; + + const host = ts.createCompilerHost(options); + const program = ts.createProgram( + [reproPath, path.join(bindingsRoot, 'src/server/sys.d.ts')], + options, + host + ); + const diagnostics = ts.getPreEmitDiagnostics(program); + return diagnostics.map(d => + ts.flattenDiagnosticMessageText(d.messageText, '\n') + ); + } finally { + rmSync(tmpDir, { recursive: true, force: true }); + } +} + +describe('TableRef diagnostics', () => { + const source = ` +import { t } from ${JSON.stringify(path.join(bindingsRoot, 'src/server/index.ts'))}; +import { table } from ${JSON.stringify(path.join(bindingsRoot, 'src/lib/table.ts'))}; +import { createTableRefFromDef } from ${JSON.stringify(path.join(bindingsRoot, 'src/lib/query.ts'))}; +import type { AllUnique } from ${JSON.stringify(path.join(bindingsRoot, 'src/lib/constraints.ts'))}; + +const cartItem = table( + { name: 'cart_item' }, + { id: t.u64().primaryKey().autoInc(), accountId: t.u64(), quantity: t.u32() } +); + +const ref = createTableRefFromDef(cartItem as any, 'cartItem'); +type Boom = AllUnique; +declare const b: Boom; +`; + + it('names the type instead of dumping its structure', () => { + const messages = runTypecheck(source); + const constraintError = messages.find(m => + m.includes("does not satisfy the constraint 'UntypedTableDef'") + ); + + expect(constraintError).toBeDefined(); + // The name, not the shape. + expect(constraintError).toContain('TableRef<'); + expect(constraintError).not.toContain('type: "table"'); + expect(constraintError).not.toContain('accessorName'); + expect(constraintError.length).toBeLessThan(250); + }, 15000); +}); diff --git a/crates/cli/build.rs b/crates/cli/build.rs index c5bd4303464..90da9fa3bdd 100644 --- a/crates/cli/build.rs +++ b/crates/cli/build.rs @@ -110,6 +110,7 @@ fn generate_template_files() { // Embed skill files from skills/*/SKILL.md let skills_dir = repo_root.join("skills"); + println!("cargo:rerun-if-changed={}", skills_dir.display()); let skill_names = discover_skill_names(&skills_dir); generated_code.push_str("pub fn get_skill(name: &str) -> Option<&'static str> {\n"); diff --git a/crates/cli/src/subcommands/dev.rs b/crates/cli/src/subcommands/dev.rs index a0588b6de50..a2622c8ea0c 100644 --- a/crates/cli/src/subcommands/dev.rs +++ b/crates/cli/src/subcommands/dev.rs @@ -738,7 +738,7 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E let loaded_config_dir = loaded_config.as_ref().map(|lc| lc.config_dir.clone()); generate_build_and_publish( - &config, + &mut config, &project_dir, loaded_config_dir.as_deref(), &spacetimedb_dir, @@ -853,7 +853,7 @@ pub async fn exec(mut config: Config, args: &ArgMatches) -> Result<(), anyhow::E println!("\n{}", "File change detected, rebuilding...".yellow()); match generate_build_and_publish( - &config, + &mut config, &project_dir, loaded_config_dir.as_deref(), &spacetimedb_dir, @@ -1000,7 +1000,7 @@ fn upsert_env_db_names_and_hosts(env_path: &Path, server_host_url: &str, databas #[allow(clippy::too_many_arguments)] async fn generate_build_and_publish( - config: &Config, + config: &mut Config, project_dir: &Path, config_dir: Option<&Path>, spacetimedb_dir: &Path, @@ -1146,7 +1146,8 @@ async fn generate_build_and_publish( publish_entry.insert("break-clients".to_string(), json!(true)); } - publish::exec_from_entry(config.clone(), publish_entry, config_dir, clear_database, yes).await?; + // Preserve a token created during publish for logs and later rebuilds. + publish::exec_from_entry(config, publish_entry, config_dir, clear_database, yes).await?; } println!("{}", "Published successfully!".green().bold()); diff --git a/crates/cli/src/subcommands/publish.rs b/crates/cli/src/subcommands/publish.rs index 745664880f0..63d704d25af 100644 --- a/crates/cli/src/subcommands/publish.rs +++ b/crates/cli/src/subcommands/publish.rs @@ -450,7 +450,7 @@ pub async fn exec_with_options( } pub async fn exec_from_entry( - mut config: Config, + config: &mut Config, entry: HashMap, config_dir: Option<&std::path::Path>, clear_database: ClearMode, @@ -465,7 +465,7 @@ pub async fn exec_from_entry( let yes = if force { YesFlags::all() } else { YesFlags::default() }; - execute_publish_configs(&mut config, vec![command_config], true, config_dir, clear_database, yes).await + execute_publish_configs(config, vec![command_config], true, config_dir, clear_database, yes).await } async fn execute_publish_configs<'a>( diff --git a/skills/spacetimedb-typescript-core/SKILL.md b/skills/spacetimedb-typescript-core/SKILL.md new file mode 100644 index 00000000000..993a527a452 --- /dev/null +++ b/skills/spacetimedb-typescript-core/SKILL.md @@ -0,0 +1,101 @@ +--- +name: spacetimedb-typescript-core +description: Core SpacetimeDB TypeScript server and client SDK syntax for building an application without framework or architecture guidance. +license: Apache-2.0 +metadata: + author: clockworklabs + version: "1.0" + language: typescript +--- + +# SpacetimeDB TypeScript Core API + +## Server module + +Define tables with `table()`, bind them with `schema()`, and export the schema +as the module default. Export reducers from the same module or its entry file. + +```typescript +import { schema, table, t } from 'spacetimedb/server'; + +const record = table( + { name: 'record', public: true }, + { + id: t.u64().primaryKey().autoInc(), + label: t.string().index('btree'), + value: t.u32(), + }, +); + +const spacetimedb = schema({ record }); +export default spacetimedb; + +export const createRecord = spacetimedb.reducer( + { label: t.string(), value: t.u32() }, + (ctx, { label, value }) => { + ctx.db.record.insert({ id: 0n, label, value }); + }, +); +``` + +Table names must be snake_case. The keys passed to `schema({ ... })` are the +server-side `ctx.db` accessor names. A split module must re-export the schema +as the default export from its entry file. + +## Types and table access + +Common builders are `t.string()`, `t.bool()`, `t.u32()`, `t.i32()`, +`t.u64()`, `t.i64()`, `t.identity()`, `t.timestamp()`, and +`t.option(inner)`. The 64-bit integer builders use TypeScript `bigint` values. +Use `0n` for an auto-increment `u64` or `i64` field during insertion. + +Column modifiers include `.primaryKey()`, `.autoInc()`, `.unique()`, and +`.index('btree')`. + +```typescript +const row = ctx.db.record.id.find(id); // row | null +const inserted = ctx.db.record.insert(values); // inserted row +if (row) ctx.db.record.id.update({ ...row, value: 2 }); // update by primary key +ctx.db.record.id.delete(id); // delete by primary key +const matching = [...ctx.db.record.label.filter(label)]; +const all = [...ctx.db.record.iter()]; +``` + +`iter()` and `filter()` return iterators. Spread them before using array +methods. Insert through the table accessor, not through an index accessor. + +## Generated client bindings + +Generated bindings convert snake_case table, reducer, and field names to +camelCase. A server reducer named `createRecord` is called as `createRecord` in a +TypeScript client. + +Create a connection with the generated `DbConnection`: + +```typescript +import { DbConnection, tables } from './module_bindings'; + +const connection = DbConnection.builder() + .withUri(serverUri) + .withDatabaseName(moduleName) + .onConnect(ctx => { + ctx.subscriptionBuilder() + .onApplied(() => console.log('ready')) + .subscribe([tables.record]); + }) + .build(); +``` + +Call reducers with an object argument: + +```typescript +await connection.reducers.createRecord({ label: 'Example', value: 1 }); +``` + +The generated database accessors support row callbacks: + +```typescript +connection.db.record.onInsert((_ctx, row) => console.log(row.label)); +connection.db.record.onUpdate((_ctx, oldRow, newRow) => console.log(oldRow, newRow)); +connection.db.record.onDelete((_ctx, row) => console.log(row.id)); +``` diff --git a/skills/typescript-client/SKILL.md b/skills/typescript-client/SKILL.md index 3a31183133f..62f86ff36fc 100644 --- a/skills/typescript-client/SKILL.md +++ b/skills/typescript-client/SKILL.md @@ -18,7 +18,7 @@ Generated bindings convert snake_case names to camelCase, including row fields: ## React: main.tsx ```typescript -import React, { useEffect, useMemo } from 'react'; +import React, { useMemo } from 'react'; import ReactDOM from 'react-dom/client'; import { SpacetimeDBProvider } from 'spacetimedb/react'; import { DbConnection } from './module_bindings'; @@ -30,6 +30,7 @@ function Root() { DbConnection.builder() .withUri(SPACETIMEDB_URI) .withDatabaseName(MODULE_NAME) + // Reuse the token issued on the previous connection. .withToken(localStorage.getItem('auth_token') || undefined), [] ); @@ -46,6 +47,7 @@ ReactDOM.createRoot(document.getElementById('root')!).render(); ## React: App.tsx ```typescript +import { useEffect } from 'react'; import { useTable, useSpacetimeDB } from 'spacetimedb/react'; import { DbConnection, tables } from './module_bindings'; @@ -53,7 +55,7 @@ function App() { const { isActive, identity: myIdentity, token, getConnection } = useSpacetimeDB(); const conn = getConnection() as DbConnection | null; - // Save auth token + // Persist the issued token for the next page load. useEffect(() => { if (token) localStorage.setItem('auth_token', token); }, [token]); // Subscribe when connected. Prefer typed query builders over raw SQL diff --git a/skills/typescript-server/SKILL.md b/skills/typescript-server/SKILL.md index c9f2e7343fd..ba24bf781d5 100644 --- a/skills/typescript-server/SKILL.md +++ b/skills/typescript-server/SKILL.md @@ -98,6 +98,10 @@ Every column is a `t` builder value: Modifiers: `.primaryKey()`, `.autoInc()`, `.unique()`, `.index('btree')`, `.default(value)`. +`.primaryKey()` and `.unique()` apply to one column. For uniqueness across +multiple columns, use a surrogate key, the multi-column index below, and a +reducer that rejects an existing index match before inserting. + Use `.default(value)` only for a newly appended migration-safe field. Do not put defaults on primary-key, unique, or auto-increment columns. Optional columns: `nickname: t.option(t.string())` @@ -130,7 +134,9 @@ export { default } from './schema'; // re-export the schema for the module ent ## Reducers -Reducers are created with `spacetimedb.reducer(...)`; the export name becomes the reducer name: +Reducers are created with `spacetimedb.reducer(...)`. An exported `signUp` +becomes `signUp` in generated clients and `sign_up` in `spacetime call` and +`describe`: ```typescript export const createEntity = spacetimedb.reducer( @@ -278,9 +284,22 @@ const Shape = t.enum('Shape', { A client subscribing to a view receives only the rows it returns. Use a per-user view (keyed on `ctx.sender`) for per-viewer access control: deleting a row it depends on (e.g. a membership row) automatically drops the rows it was exposing from that client. +Use index accessors in views. Do not scan a whole table with `.iter()` when an +indexed lookup can select the required rows. `t.row(...)` and `t.object(...)` return schema builders, not TypeScript runtime row types. Let a view callback infer its result, or annotate a separately declared structural type such as `Array<{ sku: bigint; label: string }>`. A named output type must not reuse the generated PascalCase name of its view accessor (for example, reserve `DiscountedProduct` for a `discounted_product` view). +A view context is `ViewCtx` (and `AnonymousViewCtx`), both exported from +`spacetimedb/server`. It carries `sender`, a read-only `db`, and `from`; it is +not a `ReducerCtx`, so a helper shared between a reducer and a view must accept +either: + +```typescript +import type { ReducerCtx, ViewCtx, InferSchema } from 'spacetimedb/server'; +type S = InferSchema; +function stockOf(ctx: ReducerCtx | ViewCtx, itemId: bigint) { ... } +``` + Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arguments: view options, the declared return schema, and the callback. ```typescript @@ -288,7 +307,7 @@ Both `spacetimedb.view(...)` and `spacetimedb.anonymousView(...)` take three arg export const activeUsers = spacetimedb.anonymousView( { name: 'active_users', public: true }, t.array(entity.rowType), - (ctx) => [...ctx.db.entity.iter()].filter(e => e.active) + (ctx) => [...ctx.db.entity.active.filter(true)] // active: t.bool().index('btree') ); // Per-user view (varies by ctx.sender): diff --git a/tools/llm-sequential-upgrade/.gitignore b/tools/llm-sequential-upgrade/.gitignore index 14aa619a63d..35223d12b8f 100644 --- a/tools/llm-sequential-upgrade/.gitignore +++ b/tools/llm-sequential-upgrade/.gitignore @@ -27,4 +27,4 @@ telemetry/metrics.jsonl **/telemetry/**/metadata.json # Sequential-upgrade run output lives in the external spacetimedb-ai-test-results repo -sequential-upgrade/sequential-upgrade-*/ +sequential-upgrade/ diff --git a/tools/llm-sequential-upgrade/read-guard.sh b/tools/llm-sequential-upgrade/read-guard.sh new file mode 100644 index 00000000000..314c52e7cde --- /dev/null +++ b/tools/llm-sequential-upgrade/read-guard.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Write Claude Code settings that deny direct Read tool access to benchmark +# internals. Bash is allowed, so this is not filesystem isolation. + +write_read_guard() { + local app_dir="$1" backend="$2" out siblings="" + out="$app_dir/.read-guard-settings.json" + + local b + for b in spacetime postgres mongodb; do + [[ "$b" == "$backend" ]] && continue + siblings+=" \"Read(**/$b/results/**)\", +" + done + + cat > "$out" < \ + --attempt --grant-id --level \ + --feature --strikes +``` + +Stack adapters are interchangeable. The model and requested work can remain +fixed while the stack changes. + +The optional local dashboard is another client of the same controller, not a +replacement for the CLI. It reads the same durable campaign artifacts and +submits the same bounded commands, so CLI-started work appears in the browser +and dashboard-started work remains fully operable from the CLI. See +`dashboard/README.md` for the Docker service. + +## Documentation + +- `SETUP.md` — local prerequisites and first-run setup +- `APPLIANCE-DESIGN.md` — appliance boundaries and execution model +- `appliance/README.md` — Docker appliance operation +- `dashboard/README.md` — optional web dashboard +- `grader/README.md` — grader architecture and evidence model +- `tracks/ecommerce/composition/README.md` — packs, recipes, and release composition +- `docs/dependency-graph.html` — generated ecommerce feature dependency graph +- `docs/stack-bench.html` - Stack Bench presentation + +Working notes, generated reports, and run artifacts are local operator material +and are intentionally not tracked in the repository. Reviewable project +presentations belong in `docs/`. + +## Source layout + +| Path | Purpose | +|---|---| +| `commands/` | executable operator and harness commands | +| `src/actions/` | scenario action contracts and executors | +| `src/agents/` | coding-agent adapters and credentials | +| `src/campaigns/` | campaign compilation, scheduling, locking, and reports | +| `src/composition/` | tracks, packs, recipes, definitions, and calibration | +| `src/evidence/` | artifacts, scoring, provenance, and evidence states | +| `src/grading/` | grader execution and isolated check workers | +| `src/progression/` | feature graphs, dependency state, strikes, and progression scoring | +| `src/references/` | reference fixture selection and qualification | +| `src/releases/` | release source, bundle, and signature verification | +| `src/runtime/` | leases, containers, snapshots, recovery, and platform control | +| `src/stacks/` | stack adapter contracts and stack-specific operations | + +Use the `npm run` commands documented below instead of depending on internal +module locations. `src/package-root.ts` is the single source of truth for the +repository and Stack Bench roots. + +During development, run the smallest test or validator that covers the change. +After a shared runtime, composition, grading, campaign, or release change is +stable, run this integrated source gate once: + +```bash +npm run lint +npm run typecheck +npm test +``` + +Documentation-only changes do not require the integrated gate. Run Docker +qualification only when a changed file affects its recorded scope. Run the +complete mutation set only for a release candidate. + +The optional model-based SpacetimeDB behavioral review is separate from the +measured coding sessions. Run it deliberately with `--behavioral-review`; it is +off by default so campaign cost and token accounting never omit an unrecorded model +call. The model-free friction report remains automatic. + +## Run it + +The supported v1 deployment is the Docker appliance on a dedicated Linux +runner. Follow `appliance/README.md` to build or verify the controller image, +configure credentials, run preflight, and start a campaign. + +### Local harness development + +The commands in this section run from a source checkout. Use them to develop +and qualify the harness. They are not the distributable appliance workflow. + +Install the locked harness dependencies and browser once per checkout: + +```bash +cd tools/stack-bench +npm ci +npm run bootstrap:browsers +npm test +npm run check:prompts +npm run preflight -- --backend spacetime,postgres,mongodb --track ecommerce --levels 1-2 --smoke +npm run test:null +npm run test:container +``` + +`check:prompts` is model-free and Docker-free. It renders the actual L1-L3 +dependency-mode requests for every packaged stack. It verifies their exact +bytes, confirms that the product request is the same across stacks, and rejects +language that tells the agent about grading or testing. + +`preflight` says whether the exact requested run can start and gives a concrete +fix for each failure. It checks Docker/Compose, resource floors, image and +database identities, credential presence, selected agent materials, ports, +clock, storage, and inherited run state. `--smoke` uses no model: it starts the real build image, checks declared +outbound destinations, and proves its result-volume write survives on the host. +Every real benchmark runs that full smoke automatically before any model call +and stores the result as `preflight.json`; the explicit command is for fixing +the machine before launching a campaign. + +`test:null` drives the selected grader against a reachable blank app and fails +if any point-bearing criterion passes or becomes inconclusive. It takes several +minutes because it uses the real browser suites, not fixture reports. Every +public track is included by default and the complete criterion evidence is +written under `results/`. A passing null control is only one input to release +qualification. It is not benchmark data. + +```bash +npm run bench -- --backend spacetime --track ecommerce --levels 1-2 +npm run bench -- --backend postgres --track ecommerce --levels 1-2 --run-index 1 +npm run bench -- --backend mongodb --track ecommerce --levels 1-2 --run-index 2 +npm run bench -- --backend postgres --track ecommerce --levels 1 \ + --pack ecommerce.feature.accounts \ + --check ecommerce.feature.accounts.accounts.1a +npm run bench -- --backend postgres --track ecommerce --levels 1 \ + --recipe ecommerce.sequential-l1@2.5.0 + +# Inspect an exhausted level, then grant at most four more repair rounds. +npm run repair -- status --level 1 +npm run repair -- grant --level 1 --rounds 4 \ + --max-budget-usd 25 --timeout-minutes 120 +``` + +A repair grant is accepted only when the parent has a complete, conclusive +application failure, an exhausted round budget, an intact level checkpoint, +and the exact current harness and adapter identities. A short setup session +installs dependencies and starts the saved app; it is timed and costed +separately from repair rounds. The setup session may not change source. +The controller then verifies the source bytes and reproduces the prior score, +test selection, denominator, and failed-criterion set before spending a +repair round. If any of those checks differ, the continuation stops. + +Each grant creates `continuations/grant-/` below its parent result. Its +`run.json` records the original run, immediate parent, grant size, rounds used, +cumulative rounds/cost/time, reproduced baseline, setup session, and new source +checkpoint. `process.json` records the bounded controller process and retained +logs. Repairing an earlier ladder level invalidates the meaning of later-level +results; those levels are listed explicitly as needing a fresh run and are not +charged to that earlier level's cumulative correction path. + +`--pack` changes requested scope: the agent receives only global recipe framing +plus the selected packs' requirements and testing contracts. Declared pack +dependencies are included automatically and recorded as resolved task packs. +`--check` only narrows measurement inside that requested task; it never removes +requirements by itself, and a check outside explicitly selected packs is +rejected. With neither option, the current catalog recipe is requested and +graded. + +`--recipe @` selects one exact non-retired catalogued release for a +single-level run. It uses the same preflight, agent, grader, artifact, null, and +qualification paths as the current default. Omitting it resolves the current +L1/L2 candidates. Selecting an exact release never changes a public label. + +Give concurrent runs distinct `--run-index` values; ports and databases are +allocated from it. Results land under a unique run id inside +`results/-run/`. + +Public result JSON uses artifact schema v2. Each file records what kind of +evidence it contains, the attempt and parent attempt that produced it, start and +completion times, and every applicable engine, recipe, pack, fixture, +calibration, experiment, agent-adapter, and stack-adapter identity. Evidence +payload fields are checked by kind and files are replaced atomically. Active +readers accept only schema v2 and reject unknown fields, kinds, versions, +malformed hashes, and secret-bearing keys. Files outside the current artifact +schema are not interpreted as benchmark evidence. Operators may retain them +separately as local archival material. + +### Cost evidence + +`costUsd` is the normalized benchmark cost. The credential broker records the +provider usage and applies the pricing rates frozen in the campaign. This value +controls the benchmark budget. `calculatedCostUsd` is the receipt calculation +from the same recorded usage and rates. It must match the broker ledger. + +`cliCostUsd` preserves the coding CLI's own cost value for comparison. It can +differ from the normalized benchmark cost and does not control the benchmark +budget. `costComplete: true` means that every recorded billable session has a +complete, reconciled receipt. Do not use a cost result when this field is false. + +Every check records exactly one typed state: `passed`, `failed`, `inconclusive`, +or `harness_failure`. One shared status table drives scoring, run outcomes, +mutation/null controls, comparisons, repair eligibility, and console labels. +Diagnostic wording is only rendered or redacted for people; changing that prose +cannot turn missing evidence into a product failure or send a harness defect to +the repair agent. + +The scenario action language is also startup-validated. Every registered action +declares a versioned input compiler, required capabilities, hard +deadline, evidence type, redaction tags, renderer metadata, and a narrow +executor boundary. Actions run through independent registered executors with +capability-scoped access; concurrency, browser lifecycle, backend/app control, +and direct database writes use the same typed contract as browser observations. + +For local harness development, bring up the database services first. The +SpacetimeDB adapter starts its own dedicated run host: + +```bash +docker compose -f tools/stack-bench/docker-compose.yaml up -d +``` + +Local development requires the Claude Code CLI, Node, and Docker. The services +use their own ports (6532 Postgres, 6537 MongoDB), container names, and volumes. +A run does not share their state with other services on the machine. + +A run owns only what it starts, and stops it again when finished or interrupted. +A SpacetimeDB host that was already running belongs to whoever started it, so the +benchmark refuses to reuse or restart it. Use a dedicated `STACK_BENCH_STDB_URI` +whose explicit loopback port is free; the benchmark starts that host and stops it +at the end, or retains it for debugging with `--retain-backend`. + +Coding sessions are selected through a statically registered agent adapter. +`claude-code` is the default; deterministic, fault-injection, and model-free +reference adapters exercise the same versioned request/result contract in +harness qualification. Select one with `--agent-adapter `. Arbitrary +executable paths are not accepted as production adapters. + +## Tracks + +A track is one application the benchmark can build and grade. Everything +application-specific, including level prompts, the UI contract, scenario suites, +and the core flow the linter walks, lives under `tracks//`, declared by a +`track.json`. Adding an application is a matter of dropping in a directory; the +harness needs no change. Pick one with `--track` (default `chat`). + +Tracks whose fixture must be recreated after a database reset set +`reseedOnReset`. Stack Bench restarts the application and verifies its public URL +before grading. The scored feature checks then verify the required data and behavior. + +Composition authoring is read-only and does not require Docker: + +```bash +npm run pack -- validate tracks/ecommerce/composition/packs/feature-accounts-1.1.0.json --track ecommerce +npm run recipe -- validate tracks/ecommerce/composition/recipes/sequential-l1-2.5.0.json --track ecommerce +npm run recipe -- show tracks/ecommerce/composition/recipes/sequential-l1-2.5.0.json --track ecommerce --pack ecommerce.feature.accounts +npm run recipe -- diff --track ecommerce +npm run graph +``` + +`npm run graph` rebuilds `docs/dependency-graph.html` from the versioned +ecommerce progression definition. + +`recipe diff` reports meaning, scoring, fixture, execution, and metadata changes +separately, names requirement/contract fragments added or removed, then names the calibration bindings and evidence repetitions that +must be redone. `recipe show --pack` and `--check` produce a selected scope with +its own deterministic selection hash, bound to the source recipe hash. The same +flags on `npm run bench` run that scope. Packs and individual checks are combined as +a union. Run, bundle, and grade artifacts record the request, the exact checks +it resolved to, which checks were attempted, and any checks not run with their +reason. A subset can be an intentional benchmark run. Compare only results with +the same recipe and selection identities. Working notes and superseded local +artifacts are not part of the public source tree and must not be presented as +benchmark results. + +Packs own ordered public requirement fragments, testing-hook fragments, and +their checks; recipes retain global framing and choose exact pack versions. +Removing a pack therefore removes its unique instructions and checks together. +Explicitly shared fragments are deduplicated only when their source slice, +order, modes, and bytes match exactly. A `--check` filter intentionally narrows +measurement without changing the task the app was built to satisfy. `recipe +show` displays that exact composed task and its independent hash. + +| Track | Application | Why it exists | +|---|---|---| +| `chat` | rooms, messages, presence | baseline real-time collaboration workload | +| `ecommerce` | storefront, cart, warehouses | numeric shared state and contention workload | + +Tracks are isolated by a port offset and a name slug, so two can run at the same +`--run-index` without colliding on ports, databases or result directories. + +## Levels + +The current ecommerce sequential definitions cover L1 through L3. All three are +candidates. No current qualification result is accepted. L1 contains 46 scored +checks. L2 contains 74 scored checks. Each scored L1 and L2 check has an exact +known-defect definition for all three supported stacks. Chat definitions remain +available through L2. + +Every artifact records the exact levels and recipe that ran. A level without a +launchable catalog release fails instead of falling back to another level. + +Ordered by the property each makes verifiable, not by feature novelty — see each +track's `LEVELS.md`. + +| Level | Chat adds | Ecommerce adds | Makes verifiable | +|---|---|---|---| +| 1 | accounts + basic chat | storefront, cart, warehouses | identity, durable state, real-time | +| 2 | private rooms, membership | fulfilment, transfers, returns, pricing | authorization / multi-view consistency | +| 3 | reactions, polls, capacity | reservations and scheduled work | atomicity, isolation, deferred-work durability | +| 4 | scheduling, expiry | per-customer ranking and catalogue search | per-viewer derivation at catalogue scale | +| 5 | volume | volume | throughput, latency, efficiency | + +Levels are cumulative: an app at L3 is still checked against L1 and L2, so a +regression is caught rather than scored around. + +## How verification works + +Apps differ in structure, so the harness locates elements only through a +contract of stable element IDs that the prompt requires (`tracks//contracts/`). +Scenarios (`tracks//scenarios/`) then drive real browser clients — one isolated +context per actor, so identities are genuinely separate — and assert on what a +user would observe. + +Scoring groups are track-defined because the applications expose different +failure modes. Ecommerce uses features, invariants, contention, and systems +coverage; chat uses features, invariants, delivery, and systems coverage. The +compiled recipe—not a hard-coded universal axis list—is the source of truth for +which checks and points apply to a run. + +The separation matters: a feature score cannot detect cross-cutting failures +such as broken ownership, durability, transaction boundaries, or reconnect +recovery. + +Scoring is the exact sum of passed checks. Console errors remain visible diagnostics, +but do not silently change unrelated check scores. Failed or unavailable checks earn +zero without changing the declared denominator. The grader does not reload a failed +live-update assertion and retry it, so "real-time" means real-time. + +## Files + +| Path | Purpose | +|---|---| +| `commands/bench.ts` | runs one benchmark attempt | +| `commands/agent.ts` | drives one coding session (build, upgrade, or repair) | +| `commands/run-suite.ts` | resets and grades one prepared app | +| `commands/report-bugs.ts` | turns failed checks into a repair report | +| `grader/grade.ts` | executes scenarios against real clients | +| `grader/mutation-test.ts` | validates checks with known defects | +| `linter/lint.ts` | checks the app exposes the required test ids | +| `docker-compose.yaml` | the Postgres and MongoDB services | +| `appliance/` | dedicated Linux runner controller image, Compose bundle, and operator guide | +| `reset-db.sh`, `restart-backend.sh` | environment control used by the suites | +| `src/composition/tracks.ts` | resolves a track: its paths, suites, ports and names | +| `tracks//` | one application: prompts, contracts, scenarios, lint walk | +| `backends/` | per-backend setup and deploy instructions given to the agent | +## Validation safeguards + +- **Reference qualification.** Every scored recipe must pass against its exact, + source-bound reference implementation before promotion. +- **Mutation testing.** `dist/grader/mutation-test.js` injects declared defects and + requires the intended criterion to fail conclusively without unrelated + regressions. Setup and infrastructure failures do not count as detections. + `npm run check:mutations -- --app --mutations ` + verifies every source edit is present exactly once before a Docker run. + During development, validate only the mutation definitions affected by the + change. The complete mutation set is a release-candidate gate. Live full-set + qualification requires the explicit `--release-candidate` option. A targeted + live check uses `--mutation-id ` and cannot become promotion evidence. +- **Null controls.** A blank application must not earn points or produce + inconclusive scored evidence. +- **State isolation.** The database is reset before each suite, and each run + receives distinct ports, database names, leases, and result paths. +- **Fail-closed evidence.** Missing, malformed, mismatched, or inconclusive + evidence cannot become a passing score. + +## What a run records + +Evidence emitted into the result directory and `/stack-bench/` includes: + +| Artifact | What it is | +|---|---| +| `preflight.json` | exact environment admission checks completed before model spend | +| `bundle.json` | scores per suite, code metrics, environment checks | +| `grading-.json` | every criterion's typed verdict and structured evidence | +| `contract-lint.json` | which test ids resolved | +| `media/*.webm` | one video per actor per feature — what each user saw | +| `media/*.png` | full-page screenshot at the exact moment an assertion failed | +| `media/*.trace.zip` | Playwright trace: steppable, with DOM snapshots and network | +| `records/bug-report-l-round.md` | what the agent was told each repair round | +| `run.json` | exact stack, model, recipe, test pack, prompt identity, image, repair budget, outcome, usage, and timing for the run | +| `level-l-checkpoint.json` | strict parent-linked identity for the source accepted at the end of a level | +| `level-l-source/` | source-only level checkpoint; dependencies, build output, and grading evidence are excluded | +| `continuations/grant-/run.json` | immutable child result for one finite post-run correction grant, including reproduced baseline and cumulative effort | +| `continuations/grant-/process.json` | bounded continuation-process outcome plus retained stdout/stderr identities | + +Recording is on by default; `--no-media` turns it off for a quick check. Watching +the failing actor's video is the fastest way to confirm a verdict is real before +reporting it, and the recordings are the evidence published alongside results. + +## Watching a run + +Recorded videos carry an annotation banner showing which actor is being driven, +the feature and criterion under test, and the step in progress — so a recording +explains itself rather than needing the log beside it. The banner turns green on +a passing criterion and red on a failure, with the failure message, immediately +before the screenshot is captured. + +It is injected outside the app's root and carries no test id, so scoped +assertions cannot see it and it cannot affect a score. diff --git a/tools/stack-bench/SETUP.md b/tools/stack-bench/SETUP.md new file mode 100644 index 00000000000..6faadcce316 --- /dev/null +++ b/tools/stack-bench/SETUP.md @@ -0,0 +1,141 @@ +# Reproducible run setup + +Stack Bench records the inputs needed to explain and compare a result. A run is +usable only when its requested scope passes preflight and its artifact identities +remain consistent through build, grading, repair, and reporting. + +For a distributable deployment, use the dedicated Linux appliance described in +[`APPLIANCE-DESIGN.md`](APPLIANCE-DESIGN.md) and +[`appliance/README.md`](appliance/README.md). The commands below are also useful +for local development. + +## Prerequisites + +- Node.js 22 or newer +- Docker Engine and Docker Compose v2 +- enough CPU, memory, disk, and free ports to pass exact-scope preflight +- Chromium installed through the pinned Playwright dependency +- a credential accepted by the selected coding-agent adapter +- for SpacetimeDB, the repository's Linux CLI and TypeScript bindings + +Install the Node.js dependencies and browser: + +```bash +cd tools/stack-bench +npm ci +npm run bootstrap:browsers +``` + +Build the local coding image: + +```bash +docker build -t stack-bench-build:2.1.226 container +``` + +On a Windows checkout, build the repository's Linux SpacetimeDB CLI before a +SpacetimeDB run: + +```bash +bash container/build-linux-cli.sh +``` + +The supported appliance uses digest-pinned images from its release manifest. +The local image tag above is for development; preflight resolves the image that +will actually execute and records its immutable content ID. + +## Coding-agent credentials + +The selected agent adapter declares its accepted credential sources. The +current Claude Code adapter supports two mutually exclusive modes: + +- a long-lived subscription token through `CLAUDE_CODE_OAUTH_TOKEN_FILE`; +- an API key through `ANTHROPIC_API_KEY` or the appliance secret-file mapping. + +Select one mode. Conflicting sources fail closed. Secret values are not written +to run artifacts or Docker command arguments. The controller exchanges the +selected credential for a short-lived session token through its local provider +broker. The coding container receives that token and never receives the +provider credential or credential file. Production runs still belong on a +dedicated runner because the controller holds the credential and has +root-equivalent Docker access. + +The adapter does not accept the rotating credential file at +`~/.claude/.credentials.json`. Use a subscription-token file or an API key. + +## Validate before model spend + +Run preflight for the exact stacks, track, levels, recipe, and image you intend +to execute. Include `--smoke` for the model-free container and network checks: + +```bash +npm run preflight -- \ + --backend spacetime,postgres,mongodb \ + --track ecommerce \ + --levels 1-2 \ + --smoke +``` + +Preflight checks the selected recipe scope, Docker and platform support, +resources, ports, credentials, image identity, dependency availability, +outbound access, persistent result writes, and stack-specific runtime paths. +The benchmark command repeats admission checks and refuses before a coding +session when the requested environment is not ready. + +Run the model-free container regression separately when changing container, +network, credential, or SpacetimeDB CLI behavior: + +```bash +npm run test:container +``` + +## Configuration recorded with each run + +`run.json` binds the result to the configuration that produced it, including: + +- track, stack, recipe, selected packs/checks, level, and run index; +- agent adapter, model, effort, prompt treatment, and repair budget; +- engine, prompt, contract, scenario, recipe, and calibration identities; +- coding image reference and immutable image ID; +- stack adapter, database image, platform, and Node versions; +- SpacetimeDB CLI and TypeScript binding identities when selected; +- build and repair sessions, token usage, cost, turns, and model duration; +- grading evidence, media, lifecycle events, cleanup, and contamination status. + +Campaign plans additionally freeze the attempt matrix, parallelism, pricing, +runtime images, and per-level gate policy. Dashboard and CLI execution consume +the same compiled plan. + +## Isolation rules + +Coding sessions run only in the build container. They receive the generated app +workspace, the dependencies declared by the stack adapter, their transcript +directory, and a short-lived session token from the credential broker. They do not receive the controller, +grader, scenarios, recipes, calibration, results, or source checkout. + +The harness authenticates backend and container operations through the exact +run lease. It removes only resources whose recorded identities still match and +quarantines a run when cleanup ownership cannot be proven. + +## Run and inspect + +The common CLI flows are documented in [`README.md`](README.md). Use package +commands instead of importing internal module paths: + +```bash +npm run bench -- --backend spacetime --track ecommerce --levels 1 +npm run campaign -- status +npm run dashboard +``` + +The dashboard is an optional view and controller for the same campaign engine; +it does not replace the CLI or define a second run format. + +## Comparing results + +Use `npm run campaign -- report ` to summarize a completed +campaign. Only compare attempts with compatible engine, recipe, selection, +grader, image, model, effort, and prompt treatment. Never treat a missing, +inconclusive, contaminated, or harness-failed check as a pass. + +Raw artifacts are the source of truth. Generated summaries and dashboard views +must remain reproducible from those retained artifacts. diff --git a/tools/stack-bench/appliance/Controller.Dockerfile b/tools/stack-bench/appliance/Controller.Dockerfile new file mode 100644 index 00000000000..413d6c0b6af --- /dev/null +++ b/tools/stack-bench/appliance/Controller.Dockerfile @@ -0,0 +1,91 @@ +# syntax=docker/dockerfile:1.7 + +FROM docker:29.6.2-cli@sha256:feb2d49bd65f274b3e4b4620beabe2f4691e5287e496da9fbc9830ed5f780676 AS docker-cli +FROM mcr.microsoft.com/playwright:v1.62.1-noble@sha256:c091b21d9fae78c76e85cd4356431e9b018402f172a214fc7d7a5e9a7e29d8ac AS sdk-build + +WORKDIR /workspace +COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +COPY crates/bindings-typescript/ crates/bindings-typescript/ +RUN corepack enable \ + && pnpm --filter spacetimedb install --frozen-lockfile --ignore-scripts \ + && pnpm --filter spacetimedb run build \ + && test -f crates/bindings-typescript/dist/server/index.d.ts \ + && test -f crates/bindings-typescript/dist/server/index.mjs + +FROM mcr.microsoft.com/playwright:v1.62.1-noble@sha256:c091b21d9fae78c76e85cd4356431e9b018402f172a214fc7d7a5e9a7e29d8ac AS stack-bench-build + +WORKDIR /opt/stack-bench +COPY tools/stack-bench/package.json tools/stack-bench/package-lock.json ./ +RUN npm ci --ignore-scripts --no-audit --no-fund +COPY tools/stack-bench/ ./ +RUN npm run build + +FROM mcr.microsoft.com/playwright:v1.62.1-noble@sha256:c091b21d9fae78c76e85cd4356431e9b018402f172a214fc7d7a5e9a7e29d8ac + +ENV NODE_ENV=production \ + PLAYWRIGHT_BROWSERS_PATH=/ms-playwright + +COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker +COPY --from=docker-cli /usr/local/libexec/docker/cli-plugins /usr/local/libexec/docker/cli-plugins +ADD --checksum=sha256:4629c757b7618056f8ddd7e2625ae9fdd94c0372a65049520bc7d9df9efc7f71 \ + https://github.com/sigstore/cosign/releases/download/v3.1.3/cosign-linux-amd64 \ + /usr/local/bin/cosign + +RUN apt-get update \ + && apt-get install -y --no-install-recommends lsof=4.95.0-1build3 \ + && chmod 0555 /usr/local/bin/cosign \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /opt/stack-bench +COPY tools/stack-bench/package.json tools/stack-bench/package-lock.json ./ +RUN npm ci --omit=dev --ignore-scripts --no-audit --no-fund \ + && test "$(node -p "require('playwright/package.json').version")" = "1.62.1" + +ARG SOURCE_REVISION +ARG SOURCE_SHA256 +ARG BINARY_SOURCE_SHA256 +RUN printf '%s' "$SOURCE_REVISION" | grep -Eq '^[0-9a-f]{40}([0-9a-f]{24})?$' \ + && printf '%s' "$SOURCE_SHA256" | grep -Eq '^[0-9a-f]{64}$' \ + && printf '%s' "$BINARY_SOURCE_SHA256" | grep -Eq '^[0-9a-f]{64}$' +LABEL org.opencontainers.image.title="Stack Bench controller" \ + org.opencontainers.image.revision="$SOURCE_REVISION" \ + io.spacetimedb.stack-bench.binary-source-sha256="$BINARY_SOURCE_SHA256" +ENV STACK_BENCH_SOURCE_REVISION=$SOURCE_REVISION \ + STACK_BENCH_SOURCE_SHA256=$SOURCE_SHA256 \ + STACK_BENCH_BINARY_SOURCE_SHA256=$BINARY_SOURCE_SHA256 + +COPY tools/stack-bench/ ./ +COPY --from=stack-bench-build /opt/stack-bench/dist/ ./dist/ +COPY skills/ /skills/ +COPY crates/bindings-typescript/ /opt/stack-bench-embedded-deps/bindings-typescript/ +COPY --from=sdk-build /workspace/crates/bindings-typescript/dist/ /opt/stack-bench-embedded-deps/bindings-typescript/dist/ +COPY licenses/BSL.txt /opt/stack-bench-embedded-deps/BSL.txt + +RUN node dist/container/binary-provenance.js verify \ + --root /opt/stack-bench --source-sha256 "$BINARY_SOURCE_SHA256" \ + && install -m 0555 container/bin/spacetimedb-cli \ + /opt/stack-bench-embedded-deps/spacetimedb-cli \ + && install -m 0555 container/bin/spacetimedb-standalone \ + /opt/stack-bench-embedded-deps/spacetimedb-standalone \ + && rm /opt/stack-bench-embedded-deps/bindings-typescript/LICENSE.txt \ + && mv /opt/stack-bench-embedded-deps/BSL.txt \ + /opt/stack-bench-embedded-deps/bindings-typescript/LICENSE.txt \ + && chmod 0444 /opt/stack-bench-embedded-deps/bindings-typescript/LICENSE.txt \ + && chmod 0555 /opt/stack-bench-embedded-deps/spacetimedb-cli \ + /opt/stack-bench-embedded-deps/spacetimedb-standalone \ + && cd /opt/stack-bench-embedded-deps/bindings-typescript \ + && pack_name="$(npm pack --pack-destination /opt/stack-bench-embedded-deps --silent)" \ + && mv "/opt/stack-bench-embedded-deps/$pack_name" /opt/stack-bench-embedded-deps/spacetimedb.tgz \ + && tar -tzf /opt/stack-bench-embedded-deps/spacetimedb.tgz | grep -Fxq package/dist/server/index.d.ts \ + && tar -tzf /opt/stack-bench-embedded-deps/spacetimedb.tgz | grep -Fxq package/dist/server/index.mjs \ + && cd /opt/stack-bench \ + && node dist/appliance/dependency-volume.js manifest \ + --source /opt/stack-bench-embedded-deps \ + --out /opt/stack-bench/dependency-manifest.json \ + && node dist/appliance/dependency-volume.js verify \ + --target /opt/stack-bench-embedded-deps \ + --manifest /opt/stack-bench/dependency-manifest.json \ + && rm -rf results .spacetime-data .loop-test + +ENTRYPOINT ["node", "/opt/stack-bench/dist/appliance/controller.js"] +CMD ["--help"] diff --git a/tools/stack-bench/appliance/Controller.Dockerfile.dockerignore b/tools/stack-bench/appliance/Controller.Dockerfile.dockerignore new file mode 100644 index 00000000000..a7bf601854f --- /dev/null +++ b/tools/stack-bench/appliance/Controller.Dockerfile.dockerignore @@ -0,0 +1,30 @@ +* +!tools +!tools/stack-bench +!tools/stack-bench/** +!skills +!skills/** +!crates +!crates/bindings-typescript +!crates/bindings-typescript/** +!licenses +!licenses/BSL.txt +!package.json +!pnpm-lock.yaml +!pnpm-workspace.yaml + +**/node_modules +**/dist +**/target +tools/stack-bench/results +tools/stack-bench/.spacetime-data* +tools/stack-bench/.loop-test +tools/stack-bench/*.local.md +tools/stack-bench/**/*.local.md +tools/stack-bench/transcripts +tools/stack-bench/local-notes +tools/stack-bench/media +tools/stack-bench/snapshot-l* +tools/stack-bench/grader/.candidates +tools/stack-bench/grader/.mutation-report.json +tools/stack-bench/tracks/*/overview.html diff --git a/tools/stack-bench/appliance/README.md b/tools/stack-bench/appliance/README.md new file mode 100644 index 00000000000..ba5e1036148 --- /dev/null +++ b/tools/stack-bench/appliance/README.md @@ -0,0 +1,408 @@ +# Stack Bench appliance + +This directory is the dedicated-runner packaging for Stack Bench. It is not a +general workstation install. The supported v1 runner is a disposable +Linux/amd64 machine with a local Docker Engine and the fixed persistent path +`/var/lib/stack-bench`. + +The controller has the Docker socket, which is root-equivalent access to the +runner. Use a machine that contains no unrelated workloads or credentials and +destroy it after copying verified results elsewhere. + +## What is packaged + +- `Controller.Dockerfile` builds the grader/controller with Playwright, a + digest-pinned Docker CLI, and checksum-pinned Cosign 3.1.3 for qualified + release verification. +- The dependency-volume command copies the release's exact SDK, CLI, and standalone + runtime into one checksummed named volume. It refuses unmarked, changed, or + wrong-release content. +- `docker-compose.yaml` starts the controller, one-shot dependency initializer, + and digest-pinned PostgreSQL and MongoDB services. +- `operator.env.example` documents the required operator values. It never + contains a real secret or a usable mutable image tag. + +The coding container receives the selected app, its own transcript directory, +and only the dependency paths declared by its stack adapter. It never receives +the controller image filesystem, scenarios, grader, results, or release +manifest. It also never receives the provider API key, subscription token, or +login file. A controller-owned session broker adds the selected provider +credential to model requests and stops when that coding session ends. + +## Build a development candidate + +From a clean repository checkout, compute the exact tracked release-source +identity and build Linux/amd64: + +```powershell +$source = npm --prefix tools/stack-bench run release:source --silent | ConvertFrom-Json +docker build --platform linux/amd64 ` + -f tools/stack-bench/appliance/Controller.Dockerfile ` + --build-arg SOURCE_REVISION=$($source.revision) ` + --build-arg SOURCE_SHA256=$($source.sha256) ` + --build-arg BINARY_SOURCE_SHA256=$($source.binarySourceSha256) ` + -t stack-bench-controller:development . +``` + +The controller build accepts only Linux CLI and standalone binaries recorded in +`container/spacetimedb-binaries.json`. From a clean checkout, run +`bash tools/stack-bench/container/build-linux-cli.sh`, review the updated +provenance file, and commit it. The binary files stay ignored. Build them again +after a recorded binary source input changes. + +This creates a local development candidate only. A distributable release still +requires registry digests, generated SBOMs, signatures/attestations, and a +verified release manifest. + +The identity command refuses changed or untracked release inputs. The +Dockerfile-specific ignore file excludes local notes, dependencies, generated +results, transcripts, runtime state, and archived applications so those bytes +cannot leak into the image or make otherwise identical builds diverge. + +Candidate assembly and qualified Cosign verification are documented in +[`RELEASE.md`](RELEASE.md). Candidate integrity is not a release signature, and +a qualified verification requires a trusted public key supplied from outside +the downloaded bundle. + +## Run on the dedicated runner + +1. Create `/var/lib/stack-bench/{work,results,secrets,controller-home}` with + access limited to the appliance operator. `controller-home` holds only + controller CLI state and the CLI's live transcript cache; it is outside the + read-only image. Completed runs archive transcripts and the generated + SpacetimeDB friction log under `results/` for durable artifact collection. +2. For subscription billing, run `claude setup-token` once for the dedicated + runner, write only the returned token to + `/var/lib/stack-bench/secrets/claude_subscription_token`, and set the file to + mode `0600`. Select `subscription-token` in the operator environment. For + API billing, write the provider API key as the only line in a mode-`0600` file + below `/var/lib/stack-bench/secrets` and select `api-key`. Never configure + more than one mode for a run. +3. Copy `operator.env.example` to `/var/lib/stack-bench/operator.env`, select the + intended credential mode, replace the two + example image values with exact `@sha256:` references from the release + manifest, and copy that manifest to the configured path below + `/var/lib/stack-bench`. + If you use the dashboard, write at least 32 random characters to + `/var/lib/stack-bench/secrets/dashboard_control_secret` and set the file to + mode `0600`. The browser asks for this secret when you start or resume a run. +4. Pull and verify every manifest image before starting any service. +5. Render the Compose file and run the exact requested preflight. + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml config --quiet + +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + preflight --backend spacetime,postgres,mongodb \ + --track ecommerce --levels 1-2 --run-index 0 --smoke +``` + +After preflight is green, run only the scope the operator decided to test: + +Compile and inspect a campaign without starting model work: + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + campaign show /var/lib/stack-bench/results/plans/campaign.json +``` + +The campaign file is caller-owned under `/var/lib/stack-bench`. A draft can be +inspected while definitions are still candidates. A frozen campaign requires +qualified/promoted definitions plus exact controller and build image identities; +the compiler refuses to fill those in from ambient state. An internal campaign +may leave `releaseManifestSha256` null so distribution packaging does not block +measurement. When a distributed release manifest is named, execution hashes and +validates it and checks its controller/build-sandbox image references. In either +case, the running Compose controller must match the campaign. This runtime binding +does not replace separate release-bundle signature/integrity verification. +`campaign.example.json` is a zero-cost deterministic draft showing the complete +shape; copy it outside the image and replace its study inputs before use. + +The feature catalog and run mode are separate choices. A campaign can bind the +same versioned feature graph in either mode: + +- `sequential` runs each selected level in order. Each level gets its own + features. Earlier checks run again as regression checks. +- `dependency` opens only features whose parents passed. Each branch has its + own strike budget and can stop without stopping unrelated branches. + +Dependency campaigns use `"repairSelection": "feature"` by default. The first +build receives all open features. Each later repair receives one failed feature, +while grading still checks passed prerequisites for regressions. Only the failed +feature in the repair request spends a strike. Set `"repairSelection": "batch"` +to send all currently failed features in one repair. Sequential mode does not +use this setting. + +Both modes use the same prompt modules, checks, points, and feature ownership. +A sequential campaign can select a contiguous prefix such as levels 1 through +3 without copying or redefining the graph. + +`campaign.product-brief-reference.json` is the model-free draft gate for the +primary sequential L1 condition. The prompt requests the six product features and +uses neutral stack guidance. Its evaluation scope also covers access control, +state continuity, live updates, concurrency safety, transactional integrity, +and synchronization after direct database writes. Those checks count in the +ordinary 58-point score without adding the quality specifications to the build +prompt. The exact registered reference fixture runs twice on each stack: six +planned attempts. This gate is qualification input, not comparative model data. +Null and exact-mutation controls must pass separately before the condition can +be frozen. + +Run a draft through the complete campaign path only when it uses registered +non-billable adapters and declares zero pricing for every selected model: + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + campaign trial /opt/stack-bench/appliance/campaign.product-brief-reference.json \ + --out /var/lib/stack-bench/results/campaigns/product-brief-reference-trial +``` + +`campaign trial` refuses paid adapters and frozen plans. It exists to validate +draft orchestration and evidence without model spend; its output is not +comparative benchmark data. `campaign run` remains restricted to frozen plans. +After a dependency reference campaign completes, the command also replays its +progression evidence. It reports graph coverage separately from the full recipe +catalog. The catalog status remains `not-run` until a separate full-catalog +audit exists. Repeat the evidence replay without rerunning the campaign: + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + campaign audit /var/lib/stack-bench/results/campaigns/product-brief-reference-trial +``` + +Run counts and concurrency are explicit campaign inputs. `repetitions` is the +default for every selected stack; a stack can override it independently. +`parallelism` is the maximum number of attempts allowed to run at once: + +```json +{ + "stacks": [ + { "id": "spacetime", "adapterVersion": "1.0.0", "repetitions": 5 }, + { "id": "postgres", "adapterVersion": "1.3.0", "repetitions": 8 }, + { "id": "mongodb", "adapterVersion": "1.2.0", "repetitions": 3 } + ], + "repetitions": 1, + "parallelism": 8 +} +``` + +This plan contains 16 attempts and may run any eight simultaneously, including +multiple attempts of the same stack. The scheduler assigns each live attempt a +different run slot, which isolates its app ports, database/module name, work +directory, resource locks, and evidence directory. SpacetimeDB slots also get +separate benchmark-owned host ports. A campaign can request at most 21-way +parallelism because slots 0 through 20 are the collision-tested local port +range. Set a smaller value when CPU, memory, provider limits, or database +capacity are the practical constraint. Both the counts and the concurrency +limit are included in the frozen campaign identity. + +Prepare durable state without launching an attempt, then run that exact frozen +plan (or resume its remaining attempts) from the same persistent directory: + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + campaign prepare /var/lib/stack-bench/results/plans/campaign.json \ + --out /var/lib/stack-bench/results/campaigns/campaign-001 + +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + campaign run /var/lib/stack-bench/results/plans/campaign.json \ + --out /var/lib/stack-bench/results/campaigns/campaign-001 +``` + +`campaign status /var/lib/stack-bench/results/campaigns/campaign-001` prints a compact +summary. Add `--full` only when you need the complete durable state. Two controllers +cannot own the directory at once. Failed harness or +inconclusive measurement attempts remain visible and retries append new +execution records. Multiple running attempts are checkpointed independently. +All controller processes also share persistent resource locks, so another +campaign or qualification command cannot claim a live run slot. After an +interrupted controller, reconciliation proves cleanup for every live +slot before changing any of their records. A comparison attempt is completed +only when every selected check produced pass-or-fail evidence on both the first and final build and each +score denominator equals the campaign's declared points. An inconclusive check +never shrinks the denominator or contributes comparison metrics. If a +controller ends while an attempt is still marked running, automatic resume +refuses. Run `campaign reconcile --out `; +it advances the record only if the private supervisor evidence proves exact- +owned cleanup. It never invents a result or silently starts a duplicate. + +The same plans and durable state are available through the optional local +dashboard. It is a client of these commands, not a second scheduler: + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml --profile dashboard up -d dashboard +``` + +Open `http://127.0.0.1:7331`. CLI-started campaigns appear there, and a +dashboard-started campaign remains an ordinary directory for `campaign status`, +`campaign report`, and reconciliation. See `../dashboard/README.md` for the exact +boundary and stop command. + +After any completed or stopped campaign, regenerate the report only from its +stored evidence: + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + campaign report /var/lib/stack-bench/results/campaigns/campaign-001 +``` + +This writes `report/report.json` and self-contained `report/report.html`. The +report states the exact scope and campaign status, keeps invalid executions and +retries visible, links raw run and admission artifacts, includes exact recipe, +fixture, calibration, runtime-image, release, and pricing identities, applies +the declared dispersion, and does not impute missing metrics. When a condition +selects observed-only first-build checks, the report gives them a separate +diagnostic section, denominator, coverage value, and raw-evidence link. Those +observations never alter the scored result or correction metrics. Expected +specifications are different: they are absent from the initial prompt but do +count toward first-build/final scores and may enter repair rounds. Deleting +only the `report` directory and running the command again produces the same +report identity and bytes. + +Qualification is also an explicit appliance operation. Select one validated +track/level and backend. A full mutation command retains its verified clean +baseline as a separate sibling reference artifact, so the same clean grade is +not repeated. The null gate is stack-independent: + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + preflight --agent-adapter reference-fixture \ + --backend mongodb,postgres,spacetime --track ecommerce --levels 2 \ + --run-index 0 --smoke + +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + qualification status --track ecommerce --level 2 +``` + +The `reference-fixture` adapter makes this qualification preflight model-free; +it does not call a provider or spend model tokens. Paid campaign preflight must +instead select the campaign's real agent adapter and have the credential mode +used by that campaign ready. + +Each scoped qualification artifact binds the executable calibration identity. +Evidence from another recipe, fixture, mutation set, control policy, or declared +repetition plan cannot be substituted during promotion. +Reference qualification also retains each complete underlying benchmark run in +a sibling `.runs/` directory. Keep that directory with the JSON; +the run paths recorded in the artifact are relative to the artifact itself. +New reference and null-control artifacts record the controller mode, operating +system, CPU architecture, Docker Engine version, Docker-reported kernel, +Docker architecture, CPU allocation, and memory allocation. A calibration may +bind a supported class of runner; ecommerce L2 requires the Linux/amd64 +appliance for reference, mutation, null, and budget evidence. Every artifact in +one qualification or budget-measurement set must also report the same complete +runner snapshot, so timings from materially different environments cannot be +silently combined. +Local-controller runs remain useful diagnostics but cannot promote that recipe. +Artifacts created before runner identity or the complete runner observation was +recorded remain readable, but are not accepted where the selected calibration +requires that evidence. + +Mutation qualification is serial by default. Use `--mutation-workers N` with +`--mutations` to distribute individual defects across 1 to 8 isolated workers. +The complete mutation set is a release-candidate gate and requires +`--release-candidate`. During development, run only the definitions affected +by the change. Add one `--mutation-id ` option for each affected defect. +Targeted output is diagnostic evidence and cannot satisfy a promotion gate. +Defects from one scenario may run on different workers. The parent grades the +clean fixture once. Each worker then receives a separate run slot, source tree, +backend lease, ports, logs, and artifact directory. The parent accepts the +result only when all workers use the same recipe, calibration, fixture, engine, +image, and harness, and their results cover the exact mutation selection once. + +A mutation batch runs for at most 60 minutes by default. At the deadline it +stops the active grade, restores the fixture, saves completed results atomically, +and reports the remaining count without qualifying the fixture. Use the same +checkpoint directory in a later command to continue. A changed engine, recipe, +fixture, image, track, or worker assignment rejects the checkpoint. A changed +scenario or mutation reruns that scenario group. Use a new `--out` file for +each command so prior run evidence is not replaced. + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + qualify-reference --backend postgres --track ecommerce --level 2 \ + --recipe ecommerce.sequential-l2@1.6.0 --mutations --release-candidate \ + --mutation-workers 4 --run-index 8 --repetitions 1 \ + --mutation-checkpoint-dir /var/lib/stack-bench/results/postgres-l2-checkpoints \ + --out /var/lib/stack-bench/results/postgres-l2-mutations.json +``` + +If the first batch is incomplete, run the same command again with the same +checkpoint directory and a new output file. Set +`--mutation-max-runtime-minutes N` from 1 through 120 to change the batch limit. +`--timeout-minutes` must be at least 20 minutes longer than the batch limit. +It caps the complete clean-plus-mutation repetition and cannot exceed 180 +minutes for mutation qualification. + +The worker count reserves consecutive run slots starting at `--run-index`. +The command fails before launch if those slots exceed the supported range or +if the worker count exceeds the number of selected mutations. Keep top-level +qualifications on non-overlapping slot ranges when several stacks run at once. + +Before the first qualification of a recipe whose packs still have unmeasured +runtime budgets, run each `budgetPreparation.commands` entry printed by +`qualification status` through the same Compose prefix shown above. Those +commands collect pristine references for every supported stack and then run +`pack-budget recommend`. The recommendation command verifies the exact recipe, +calibration, fixture, engine, stack coverage, repetitions, controller +environment, retained raw runs, and component arithmetic. Its policy takes the +largest observed pack runtime, doubles it, and rounds upward to the next second. +It writes a review artifact and never edits pack definitions. + +Review the recommendation, apply the accepted bounds to the pack definitions, +commit them, and build a new exact controller image. The budget-measurement +artifacts are inputs to that source change; they are not the final qualification +evidence because the executable identity changes when the bounds are added. +On the new image, require a green preflight and run every command in the +`commands` array through the Compose prefix. Only those post-budget reference, +mutation, and null artifacts can be bound to promotion. + +Before launching the official repetitions, inspect the exact go/no-go record: + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + qualification status --track ecommerce --level 2 +``` + +It is read-only. The JSON separates launch blockers, required evidence and +commands, promotion blockers, and the governance states promotion would change. +It never supplies a missing runtime budget or treats an absent artifact as a +pass. + +To run one already-decided attempt directly: + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + run --backend postgres --track ecommerce --levels 1-2 --run-index 0 +``` + +The report records the exact levels, packs, checks, stack adapter, agent +adapter, model, image content identity, and preflight evidence. It does not +infer a special label from a partial selection; it states what was run. + +## Cleanup and recovery boundary + +Results remain under `/var/lib/stack-bench/results` after the controller exits. +This includes archived model transcripts and the SpacetimeDB friction log; they +must not depend on the CLI's 30-day live-cache retention. +Do not delete that directory until its artifact manifest has been verified and +copied off the runner. Every run writes a public recovery status and keeps +private authenticated recovery authority until exact-owned cleanup succeeds. +Follow [`RECOVERY.md`](RECOVERY.md) for interruption, quarantine, safe retry, and +intentional retention. Dependency and database volume destruction remains an +operator action after verified result export; no run recursively deletes the +shared state root. diff --git a/tools/stack-bench/appliance/RECOVERY.md b/tools/stack-bench/appliance/RECOVERY.md new file mode 100644 index 00000000000..ecf413d98af --- /dev/null +++ b/tools/stack-bench/appliance/RECOVERY.md @@ -0,0 +1,67 @@ +# Interruption and recovery + +Stack Bench never guesses that a container, listener, lock, database, or data +directory is safe to delete. Normal teardown authenticates the run's private +lease, compares exact container IDs and listener PIDs, and releases only locks +whose owner record still matches that lease. + +Every appliance run keeps two different records: + +- `results/.../recovery.json` is public, contains no ownership token, and says + whether cleanup is `clean`, intentionally `retained`, or `quarantined`; +- `controller-home/supervisor/.json` is private recovery authority. It + contains the lease token and must remain readable only by the appliance + operator. Normal cleanup deletes it. Refused cleanup deliberately preserves + it. + +## If a run is interrupted + +1. Preserve the result directory and private supervisor-state file. +2. Read `recovery.json`. Do not publish an attempt whose status is + `quarantined`. +3. Do not start another run using any lock key listed in that artifact. +4. Retry authenticated cleanup from the controller: + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + recover /var/lib/stack-bench/controller-home/supervisor/.json +``` + +On success the command changes `recovery.json` to `clean`, releases the exact +owned resources, and removes the private supervisor state. It is idempotent +when public lease evidence already proves that an earlier cleanup completed. + +If the parent process ended before it retained a supervisor file, recover from +the private runtime lease instead. Supply a durable output directory outside +the private runtime directory: + +```sh +docker compose --env-file /var/lib/stack-bench/operator.env \ + -f appliance/docker-compose.yaml run --rm controller \ + recover-lease /var/lib/stack-bench/controller-home/runtime//backend-lease.json \ + --out /var/lib/stack-bench/results/recovery/ +``` + +This path uses the same ownership token, container ID, listener PID, and lock +checks. It refuses an output directory inside the runtime directory because a +successful recovery removes that directory. + +## If recovery refuses + +Refusal is the safety behavior. It means a live resource does not match the +lease or its identity could not be proven. The command leaves the private state, +lease, lock records, and public quarantine artifact intact. + +Compare the live container ID and listener PIDs with `recovery.json` and the +private lease before manual action. Never delete a same-name container, kill a +port's current listener, remove another lock, or recursively clear the shared +state root merely because its name resembles Stack Bench. Escalate with the +complete result directory and private state stored separately from public +artifacts. + +## Intentional retention + +`--retain-backend` is inspection mode, not successful cleanup. It writes +`status: "retained"` and preserves private recovery authority. No other run may +reuse the listed locks until the recovery command completes. diff --git a/tools/stack-bench/appliance/RELEASE.md b/tools/stack-bench/appliance/RELEASE.md new file mode 100644 index 00000000000..ab1c8932711 --- /dev/null +++ b/tools/stack-bench/appliance/RELEASE.md @@ -0,0 +1,102 @@ +# Release assembly and verification + +Stack Bench uses two deliberately different release states. + +- A `candidate` has exact image digests, checksummed files, and digest-bound + SPDX SBOMs. It is useful for inspecting and testing a proposed bundle, but it + is unsigned and cannot be called qualified. +- A `qualified` release adds a bundled public key, a detached Sigstore bundle + covering `release.json`, and registry signatures for every image. Verification + must use a public key obtained outside the release bundle. + +Schema v2 is the only accepted format. Pre-release schema v1 required candidate +bundles to contain placeholder signature files and could not verify a qualified +release. It has no compatibility reader because no v1 bundle was published. + +## Build a candidate + +Publish the first-party images, resolve every first- and third-party image to an +exact single-platform `linux/amd64` manifest reference, then generate one SPDX +SBOM for each exact reference. Do not use a multi-architecture index digest: +Docker Scout correctly reports the selected child-manifest digest, so an index +digest cannot satisfy the one-image/one-SBOM identity contract. + +```sh +node dist/src/releases/release-bundle.js sbom registry.example/controller@sha256:DIGEST \ + --output bundle/sbom/controller.spdx.json +``` + +The command uses registry resolution, refuses mutable references and existing +output, and checks that Docker Scout's SPDX 2.3 document contains the requested +image digest. A successful tool exit without that digest binding is rejected. + +Create a strict release specification with `state: "candidate"`, +`signing: null`, and `files` entries containing only `path` and `role`. Place +every input below the bundle root, then materialize immutable size and SHA-256 +metadata: + +```sh +node dist/src/releases/release-bundle.js assemble release-spec.json \ + --root bundle --output bundle/release.json +node dist/src/releases/release-manifest.js verify bundle/release.json --root bundle +``` + +Candidate verification reports `candidate-file-integrity`. It validates all +declared files and all four image-to-SBOM digest bindings. Candidate manifests +must use `signing: null` and cannot include a public signing key. + +## Sign and qualify + +Signing keys are external CI inputs. Never copy a private key, registry token, +or signing password into the source tree, image, bundle, Compose environment, +or command transcript. Sign each exact registry image with Cosign. The +authoritative image-signature evidence stays attached to the registry object +and is checked directly during verification; the release does not preserve a +redundant unverified export. Add the public half of the signing key as +`signing/cosign.pub` with the `public-key` role. + +Change the specification to `state: "qualified"` and declare: + +```json +{ + "signing": { + "scheme": "cosign-public-key-v1", + "publicKeyPath": "signing/cosign.pub", + "manifestBundlePath": "signing/release-manifest.sigstore.json" + } +} +``` + +Assemble `release.json` only after all other evidence exists, then sign that +exact file with a detached Cosign bundle: + +```sh +cosign sign-blob --yes --key "$COSIGN_KEY" \ + --bundle bundle/signing/release-manifest.sigstore.json bundle/release.json +``` + +The detached bundle is intentionally not checksummed by `release.json`: a file +cannot contain the hash of its own signature. Cosign authenticates it instead. + +Verify with the trusted public key copied to a path outside the downloaded +bundle: + +```sh +node dist/src/releases/release-manifest.js verify bundle/release.json --root bundle \ + --trusted-key /operator/trust/stack-bench-cosign.pub +``` + +Qualified verification refuses an absent or bundle-local trust key, requires +it to equal the public key bound by the signed manifest, verifies the detached +manifest signature, and runs `cosign verify` against every exact registry image +reference. A failed or unavailable Cosign invocation is a failed release; there +is no downgrade to candidate verification. The controller image includes +checksum-pinned Cosign 3.1.3 so this command is available in the delivered +appliance rather than depending on an untracked host installation. + +## Trust distribution + +The release bundle cannot establish trust in its own key. Publish the expected +public key and its SHA-256 fingerprint through a separately controlled channel. +The operator must compare that fingerprint before verification. Key rotation +requires a new release and an explicit trust-distribution update. diff --git a/tools/stack-bench/appliance/campaign.ecommerce-progression-reference.json b/tools/stack-bench/appliance/campaign.ecommerce-progression-reference.json new file mode 100644 index 00000000000..cde95f63a2c --- /dev/null +++ b/tools/stack-bench/appliance/campaign.ecommerce-progression-reference.json @@ -0,0 +1,97 @@ +{ + "schemaVersion": 5, + "kind": "campaign-manifest", + "id": "ecommerce-progression-reference", + "version": "2.0.0", + "state": "draft", + "title": "Ecommerce progression reference pilot", + "track": "ecommerce", + "mode": { "id": "dependency", "version": "3.0.0", "repairSelection": "feature", + "strikes": { "default": 1, "levels": {} } }, + "levels": [1, 2, 3, 4, 5, 6], + "featureCatalog": "ecommerce.questlines@2.0.1", + "selection": { + "levels": [ + { "level": 1, "recipe": "ecommerce.progression-catalog@2.0.1" }, + { "level": 2, "recipe": "ecommerce.progression-catalog@2.0.1" }, + { "level": 3, "recipe": "ecommerce.progression-catalog@2.0.1" }, + { "level": 4, "recipe": "ecommerce.progression-catalog@2.0.1" }, + { "level": 5, "recipe": "ecommerce.progression-catalog@2.0.1" }, + { "level": 6, "recipe": "ecommerce.progression-catalog@2.0.1" } + ] + }, + "stacks": [ + { "id": "mongodb", "adapterVersion": "1.2.0" }, + { "id": "postgres", "adapterVersion": "1.3.0" }, + { "id": "spacetime", "adapterVersion": "1.0.0" } + ], + "agents": [ + { + "adapter": "reference-fixture", + "adapterVersion": "1.3.0", + "model": "reference-fixture" + } + ], + "conditions": [ + { + "id": "reference-pilot", + "version": "1.0.0", + "guidanceProfile": "neutral@1.7.0", + "repairPolicy": "scored-only@1.0.0" + } + ], + "repetitions": 1, + "parallelism": 3, + "ordering": { + "method": "balanced-rotation", + "seed": "ecommerce-progression-reference-1" + }, + "budgets": { + "fixRounds": 0, + "attemptTimeoutMinutes": 180, + "maxCostUsdPerAttempt": null + }, + "attemptPolicy": { + "retries": 0, + "retryOn": [], + "excludeFromAnalysis": [ + "contaminated", + "harness_failure", + "inconclusive", + "ungraded" + ] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-08-25T00:00:00.000Z", + "source": "Reference fixtures make no provider calls.", + "models": { + "reference-fixture": { + "input": 0, + "output": 0, + "cacheWrite5m": 0, + "cacheWrite1h": 0, + "cacheRead": 0 + } + } + }, + "analysis": { + "primaryMetric": "finalScoreRate", + "secondaryMetrics": [ + "firstBuildScoreRate", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/campaign.example.json b/tools/stack-bench/appliance/campaign.example.json new file mode 100644 index 00000000000..27544b9f10e --- /dev/null +++ b/tools/stack-bench/appliance/campaign.example.json @@ -0,0 +1,115 @@ +{ + "schemaVersion": 5, + "kind": "campaign-manifest", + "id": "ecommerce-l1-example", + "version": "2.0.0", + "state": "draft", + "title": "Ecommerce L1 model-free example", + "track": "ecommerce", + "mode": { "id": "sequential", "version": "1.0.0" }, + "levels": [1], + "selection": { + "levels": [ + { + "level": 1, + "recipe": "ecommerce.sequential-l1@2.5.0", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin" + ], + "checks": [] + } + ] + }, + "stacks": [ + { "id": "spacetime", "adapterVersion": "1.0.0" }, + { "id": "postgres", "adapterVersion": "1.3.0" }, + { "id": "mongodb", "adapterVersion": "1.2.0" } + ], + "agents": [ + { + "adapter": "deterministic", + "adapterVersion": "1.2.0", + "model": "deterministic" + } + ], + "conditions": [ + { + "id": "prescribed", + "version": "1.1.0", + "guidanceProfile": "prescribed@1.1.0", + "repairPolicy": "scored-only@1.0.0", + "specifications": { + "levels": [ + { + "level": 1, + "requested": [ + "ecommerce.spec.access-control@1.2.0", + "ecommerce.spec.concurrency-safety@1.3.0", + "ecommerce.spec.external-data-sync@1.1.0", + "ecommerce.spec.live-state@1.2.0", + "ecommerce.spec.state-durability@1.1.0", + "ecommerce.spec.transactional-integrity@1.3.0" + ], + "expected": [], + "observed": [] + } + ] + } + } + ], + "repetitions": 3, + "parallelism": 1, + "ordering": { + "method": "balanced-rotation", + "seed": "replace-before-measurement" + }, + "budgets": { + "fixRounds": 3, + "attemptTimeoutMinutes": 240, + "maxCostUsdPerAttempt": null + }, + "attemptPolicy": { + "retries": 1, + "retryOn": ["harness_failure", "inconclusive"], + "excludeFromAnalysis": ["contaminated", "harness_failure", "inconclusive", "ungraded"] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-08-12T00:00:00.000Z", + "source": "deterministic adapter makes no billable provider calls", + "models": { + "deterministic": { + "input": 0, + "output": 0, + "cacheWrite5m": 0, + "cacheWrite1h": 0, + "cacheRead": 0 + } + } + }, + "analysis": { + "primaryMetric": "firstBuildScoreRate", + "secondaryMetrics": [ + "finalScoreRate", + "totalCostUsd", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/campaign.product-brief-reference.json b/tools/stack-bench/appliance/campaign.product-brief-reference.json new file mode 100644 index 00000000000..a8ee49efef6 --- /dev/null +++ b/tools/stack-bench/appliance/campaign.product-brief-reference.json @@ -0,0 +1,114 @@ +{ + "schemaVersion": 5, + "kind": "campaign-manifest", + "id": "ecommerce-l1-product-brief-reference", + "version": "2.0.0", + "state": "draft", + "title": "Ecommerce L1 product brief and quality validation", + "track": "ecommerce", + "mode": { "id": "sequential", "version": "1.0.0" }, + "levels": [1], + "selection": { + "levels": [ + { + "level": 1, + "recipe": "ecommerce.sequential-l1@2.5.0", + "features": [ + "ecommerce.feature.accounts", + "ecommerce.feature.cart-checkout", + "ecommerce.feature.catalog", + "ecommerce.feature.purchasing", + "ecommerce.feature.reviews", + "ecommerce.feature.warehouse-admin" + ], + "checks": [] + } + ] + }, + "stacks": [ + { "id": "spacetime", "adapterVersion": "1.0.0" }, + { "id": "postgres", "adapterVersion": "1.3.0" }, + { "id": "mongodb", "adapterVersion": "1.2.0" } + ], + "agents": [ + { + "adapter": "reference-fixture", + "adapterVersion": "1.3.0", + "model": "reference-fixture" + } + ], + "conditions": [ + { + "id": "product-brief-quality", + "version": "1.2.0", + "guidanceProfile": "neutral@1.7.0", + "repairPolicy": "scored-only@1.0.0", + "specifications": { + "levels": [ + { + "level": 1, + "requested": [], + "expected": [ + "ecommerce.spec.access-control@1.2.0", + "ecommerce.spec.concurrency-safety@1.3.0", + "ecommerce.spec.external-data-sync@1.1.0", + "ecommerce.spec.live-state@1.2.0", + "ecommerce.spec.state-durability@1.1.0", + "ecommerce.spec.transactional-integrity@1.3.0" + ], + "observed": [] + } + ] + } + } + ], + "repetitions": 2, + "ordering": { + "method": "balanced-rotation", + "seed": "product-brief-quality-reference-1" + }, + "budgets": { + "fixRounds": 0, + "attemptTimeoutMinutes": 60, + "maxCostUsdPerAttempt": null + }, + "attemptPolicy": { + "retries": 1, + "retryOn": ["harness_failure", "inconclusive"], + "excludeFromAnalysis": ["contaminated", "harness_failure", "inconclusive", "ungraded"] + }, + "runtime": { + "releaseManifestSha256": null, + "controllerImage": null, + "buildImage": null, + "platform": "linux/amd64" + }, + "pricing": { + "currency": "USD", + "unit": "USD-per-million-tokens", + "capturedAt": "2026-08-16T00:00:00.000Z", + "source": "reference fixture adapter makes no billable provider calls", + "models": { + "reference-fixture": { + "input": 0, + "output": 0, + "cacheWrite5m": 0, + "cacheWrite1h": 0, + "cacheRead": 0 + } + } + }, + "analysis": { + "primaryMetric": "firstBuildScoreRate", + "secondaryMetrics": [ + "finalScoreRate", + "totalCostUsd", + "totalDurationMs", + "invalidAttemptRate" + ], + "dispersion": "median-iqr", + "invalidAttempts": "report-separately", + "missingData": "no-imputation", + "comparisonUnit": "stack-agent-condition-recipe" + } +} diff --git a/tools/stack-bench/appliance/controller.ts b/tools/stack-bench/appliance/controller.ts new file mode 100644 index 00000000000..4b916d6bad8 --- /dev/null +++ b/tools/stack-bench/appliance/controller.ts @@ -0,0 +1,151 @@ +#!/usr/bin/env node + +import { spawn } from 'node:child_process'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +const RUNTIME_ROOT = join(STACK_BENCH_ROOT, 'dist'); + +const COMMANDS = Object.freeze({ + 'init-deps': [join(RUNTIME_ROOT, 'appliance', 'dependency-volume.js'), 'init'], + 'verify-deps': [join(RUNTIME_ROOT, 'appliance', 'dependency-volume.js'), 'verify'], + 'preflight': [join(RUNTIME_ROOT, 'commands', 'preflight.js')], + 'qualify-reference': [join(RUNTIME_ROOT, 'src', 'references', 'reference-live.js')], + 'qualify-null': [join(RUNTIME_ROOT, 'commands', 'null-control.js')], + 'qualification': [join(RUNTIME_ROOT, 'commands', 'qualification-cli.js')], + 'pack-budget': [join(RUNTIME_ROOT, 'src', 'composition', 'pack-budget.js')], + 'campaign': [join(RUNTIME_ROOT, 'commands', 'campaign-cli.js')], + 'dashboard': [join(RUNTIME_ROOT, 'dashboard', 'dashboard-server.js')], + 'repair': [join(RUNTIME_ROOT, 'commands', 'repair-cli.js')], + 'run': [join(RUNTIME_ROOT, 'commands', 'bench.js')], + 'verify-release': [join(RUNTIME_ROOT, 'src', 'releases', 'release-manifest.js'), 'verify'], + 'recover': [join(RUNTIME_ROOT, 'commands', 'recovery.js'), 'recover'], + 'recover-lease': [join(RUNTIME_ROOT, 'commands', 'recovery.js'), 'recover-lease'], +} satisfies Record); + +const COMMANDS_REQUIRING_AGENT_AUTH = new Set(['preflight', 'dashboard', 'run']); + +export function controllerCommandRequiresAgentAuth(command: string | undefined, + args: string[] = []): boolean { + if (command && COMMANDS_REQUIRING_AGENT_AUTH.has(command)) return true; + return command === 'campaign' && args[0] === 'run'; +} + +export interface ResolvedControllerCommand { + executable: string; + args: string[]; +} + +export function resolveControllerCommand(argv: string[]): ResolvedControllerCommand | null { + const [command, ...rest] = argv; + if (!command || command === '--help' || command === 'help') return null; + if (!Object.hasOwn(COMMANDS, command)) { + throw new Error(`unknown controller command ${JSON.stringify(command)}`); + } + return { executable: process.execPath, + args: [...COMMANDS[command as keyof typeof COMMANDS], ...rest] }; +} + +export function controllerChildEnvironment(source: NodeJS.ProcessEnv = process.env, + { requireAgentAuth = true }: { requireAgentAuth?: boolean } = {}): NodeJS.ProcessEnv { + const env = { ...source }; + delete env.ANTHROPIC_API_KEY; + delete env.ANTHROPIC_API_KEY_FILE; + delete env.CLAUDE_CODE_OAUTH_TOKEN; + delete env.CLAUDE_CODE_OAUTH_TOKEN_FILE; + if (!requireAgentAuth) return env; + const mode = source.STACK_BENCH_AGENT_AUTH ?? 'subscription-token'; + if (!['subscription-token', 'api-key'].includes(mode)) { + throw new Error('STACK_BENCH_AGENT_AUTH must be subscription-token or api-key'); + } + if (mode === 'api-key') { + const path = source.STACK_BENCH_ANTHROPIC_API_KEY_FILE?.trim(); + if (!path) throw new Error('api-key auth requires STACK_BENCH_ANTHROPIC_API_KEY_FILE'); + env.ANTHROPIC_API_KEY_FILE = path; + } else { + const path = source.STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE?.trim(); + if (!path) { + throw new Error('subscription-token auth requires STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE'); + } + env.CLAUDE_CODE_OAUTH_TOKEN_FILE = path; + } + return env; +} + +interface SignalChild { + kill(signal: NodeJS.Signals): unknown; +} + +interface SignalSource { + on(signal: NodeJS.Signals, listener: () => void): unknown; + off(signal: NodeJS.Signals, listener: () => void): unknown; +} + +export function forwardControllerSignals(child: SignalChild, + source: SignalSource = process): () => void { + const signals: NodeJS.Signals[] = ['SIGINT', 'SIGTERM']; + const listeners = new Map void>(signals.map(signal => + [signal, () => { child.kill(signal); }])); + for (const [signal, listener] of listeners) source.on(signal, listener); + return () => { + for (const [signal, listener] of listeners) source.off(signal, listener); + }; +} + +function help(): void { + process.stdout.write('Stack Bench controller\n\n' + + 'Commands:\n' + + ' preflight verify the runner without a model call\n' + + ' qualify-reference run a pristine or mutation reference gate\n' + + ' --mutation-workers N split one mutation gate across 1 to 8 isolated workers\n' + + ' qualify-null run the exact null-oracle gate\n' + + ' qualification status show exact launch and promotion blockers\n' + + ' pack-budget recommend derive reviewable bounds from exact reference evidence\n' + + ' campaign validate|show compile the exact comparison plan without running it\n' + + ' campaign prepare|trial --out checkpoint or exercise a model-free draft\n' + + ' campaign run --out execute a frozen plan\n' + + ' campaign reconcile --out prove cleanup for interrupted work\n' + + ' campaign status inspect exact durable campaign state\n' + + ' campaign report regenerate deterministic JSON and static HTML\n' + + ' dashboard [--port N] serve the local operator dashboard\n' + + ' repair status --level N inspect whether a failed level can continue\n' + + ' repair grant --level N --rounds N add one finite repair budget\n' + + ' run execute and retain one requested run\n' + + ' verify-release verify candidate files or a qualified signed release\n' + + ' recover retry authenticated cleanup or retain quarantine\n' + + ' recover-lease --out recover when parent state was not retained\n' + + ' init-deps | verify-deps initialize or verify the release dependency volume\n'); +} + +interface ChildOutcome { + code: number | null; + signal: NodeJS.Signals | null; +} + +async function main(argv: string[]): Promise { + const command = argv[2]; + const resolved = resolveControllerCommand(argv.slice(2)); + if (!resolved) { help(); return; } + const child = spawn(resolved.executable, resolved.args, + { stdio: 'inherit', env: controllerChildEnvironment(process.env, + { requireAgentAuth: controllerCommandRequiresAgentAuth(command, argv.slice(3)) }) }); + const stopForwardingSignals = forwardControllerSignals(child); + let outcome: ChildOutcome; + try { + outcome = await new Promise((resolveExit, reject) => { + child.once('error', reject); + child.once('exit', (code, signal) => { resolveExit({ code, signal }); }); + }); + } finally { stopForwardingSignals(); } + if (outcome.signal) process.kill(process.pid, outcome.signal); + process.exitCode = outcome.code ?? 1; +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + main(process.argv).catch((error: unknown) => { + console.error(`stack-bench-controller: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/appliance/dependency-volume.ts b/tools/stack-bench/appliance/dependency-volume.ts new file mode 100644 index 00000000000..94cdbc9ce97 --- /dev/null +++ b/tools/stack-bench/appliance/dependency-volume.ts @@ -0,0 +1,165 @@ +#!/usr/bin/env node + +import { createHash } from 'node:crypto'; +import { + chmodSync, copyFileSync, existsSync, lstatSync, mkdirSync, readFileSync, readdirSync, + renameSync, rmSync, writeFileSync, +} from 'node:fs'; +import { dirname, join, relative, resolve, sep } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +const MARKER = '.stack-bench-release-deps.json'; + +export interface DependencyManifestFile { + path: string; + size: number; + mode: number; + sha256: string; +} + +export interface DependencyManifest { + schemaVersion: 1; + files: DependencyManifestFile[]; +} + +interface DependencyVerification { + manifestSha256: string; + files: number; +} + +interface DependencyInitialization extends DependencyVerification { + initialized: boolean; +} + +function sha256Bytes(bytes: string | NodeJS.ArrayBufferView): string { + return createHash('sha256').update(bytes).digest('hex'); +} + +function normalizedRelative(root: string, path: string): string { + const value = relative(root, path).split(sep).join('/'); + if (!value || value.startsWith('../') || value === '..') throw new Error(`path escapes dependency root: ${path}`); + return value; +} + +function walk(root: string, current = root): string[] { + const files: string[] = []; + for (const entry of readdirSync(current, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) { + const path = join(current, entry.name); + if (entry.isSymbolicLink()) throw new Error(`dependency tree cannot contain symlinks: ${normalizedRelative(root, path)}`); + if (entry.isDirectory()) files.push(...walk(root, path)); + else if (entry.isFile()) files.push(path); + else throw new Error(`dependency tree contains unsupported entry: ${normalizedRelative(root, path)}`); + } + return files; +} + +export function createDependencyManifest(root: string): DependencyManifest { + const absolute = resolve(root); + if (!existsSync(absolute) || !lstatSync(absolute).isDirectory()) { + throw new Error(`dependency source is not a directory: ${absolute}`); + } + const files = walk(absolute).map(path => { + const bytes = readFileSync(path); + return { path: normalizedRelative(absolute, path), size: bytes.length, + mode: lstatSync(path).mode & 0o777, sha256: sha256Bytes(bytes) }; + }); + if (!files.length) throw new Error('dependency source is empty'); + return { schemaVersion: 1, files }; +} + +export function manifestSha256(manifest: DependencyManifest): string { + return sha256Bytes(`${JSON.stringify(manifest)}\n`); +} + +export function verifyDependencyTree(root: string, manifest: DependencyManifest, + { allowMarker = false }: { allowMarker?: boolean } = {}): DependencyVerification { + if (!manifest || manifest.schemaVersion !== 1 || !Array.isArray(manifest.files) || !manifest.files.length) { + throw new Error('dependency manifest is invalid'); + } + const absolute = resolve(root); + const actual = createDependencyManifest(absolute); + if (allowMarker) actual.files = actual.files.filter(file => file.path !== MARKER); + if (JSON.stringify(actual.files) !== JSON.stringify(manifest.files)) { + throw new Error(`dependency tree does not match manifest ${manifestSha256(manifest)}`); + } + return { manifestSha256: manifestSha256(manifest), files: manifest.files.length }; +} + +export function initializeDependencyVolume({ source, target, manifest }: + { source: string; target: string; manifest: DependencyManifest }): DependencyInitialization { + const sourceRoot = resolve(source); + const targetRoot = resolve(target); + const verified = verifyDependencyTree(sourceRoot, manifest); + mkdirSync(targetRoot, { recursive: true, mode: 0o755 }); + const markerPath = join(targetRoot, MARKER); + const existing = readdirSync(targetRoot); + if (existing.length) { + if (!existsSync(markerPath)) throw new Error('dependency volume is non-empty but has no release marker'); + const marker = JSON.parse(readFileSync(markerPath, 'utf8')); + if (marker.schemaVersion !== 1 || marker.manifestSha256 !== verified.manifestSha256) { + throw new Error('dependency volume belongs to a different release'); + } + verifyDependencyTree(targetRoot, manifest, { allowMarker: true }); + return { ...verified, initialized: false }; + } + + const staging = join(targetRoot, `.staging-${process.pid}`); + mkdirSync(staging, { mode: 0o700 }); + try { + for (const file of manifest.files) { + const from = join(sourceRoot, ...file.path.split('/')); + const to = join(staging, ...file.path.split('/')); + mkdirSync(dirname(to), { recursive: true }); + copyFileSync(from, to); + chmodSync(to, file.mode); + } + for (const entry of readdirSync(staging)) renameSync(join(staging, entry), join(targetRoot, entry)); + rmSync(staging, { recursive: true, force: true }); + writeFileSync(markerPath, `${JSON.stringify({ schemaVersion: 1, + manifestSha256: verified.manifestSha256 })}\n`, { flag: 'wx', mode: 0o444 }); + verifyDependencyTree(targetRoot, manifest, { allowMarker: true }); + return { ...verified, initialized: true }; + } catch (error) { + rmSync(staging, { recursive: true, force: true }); + throw error; + } +} + +function option(argv: string[], name: string): string | undefined { + const index = argv.indexOf(name); + if (index === -1) return undefined; + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`${name} requires a value`); + return value; +} + +function main(argv: string[]): void { + const command = argv[2]; + const source = option(argv, '--source') ?? '/opt/stack-bench-embedded-deps'; + const target = option(argv, '--target') ?? '/opt/stack-bench-release-deps'; + const manifestPath = option(argv, '--manifest') ?? '/opt/stack-bench/dependency-manifest.json'; + if (command === 'manifest') { + const output = option(argv, '--out'); + if (!output) throw new Error('manifest requires --out'); + writeFileSync(resolve(output), `${JSON.stringify(createDependencyManifest(source), null, 2)}\n`, { flag: 'wx' }); + return; + } + const manifest: DependencyManifest = JSON.parse(readFileSync(resolve(manifestPath), 'utf8')); + if (command === 'init') { + process.stdout.write(`${JSON.stringify(initializeDependencyVolume({ source, target, manifest }))}\n`); + return; + } + if (command === 'verify') { + process.stdout.write(`${JSON.stringify(verifyDependencyTree(target, manifest, { allowMarker: true }))}\n`); + return; + } + throw new Error('usage: dependency-volume manifest|init|verify [options]'); +} + +if (import.meta.url === pathToFileURL(process.argv[1] ?? '').href) { + try { main(process.argv); } + catch (error) { + console.error(`dependency-volume: ${error instanceof Error ? error.message : String(error)}`); + process.exit(2); + } +} diff --git a/tools/stack-bench/appliance/docker-compose.yaml b/tools/stack-bench/appliance/docker-compose.yaml new file mode 100644 index 00000000000..1edcead0ca5 --- /dev/null +++ b/tools/stack-bench/appliance/docker-compose.yaml @@ -0,0 +1,114 @@ +name: stack-bench-appliance + +services: + deps-init: + image: ${STACK_BENCH_CONTROLLER_IMAGE:?set STACK_BENCH_CONTROLLER_IMAGE to the manifest digest reference} + platform: linux/amd64 + command: ["init-deps"] + read_only: true + cap_drop: ["ALL"] + security_opt: ["no-new-privileges:true"] + volumes: + - type: volume + source: release-deps + target: /opt/stack-bench-release-deps + tmpfs: + - /tmp:size=64m,mode=1777 + + controller: + image: ${STACK_BENCH_CONTROLLER_IMAGE:?set STACK_BENCH_CONTROLLER_IMAGE to the manifest digest reference} + platform: linux/amd64 + init: true + network_mode: host + read_only: true + cap_drop: ["ALL"] + security_opt: ["no-new-privileges:true"] + depends_on: + deps-init: + condition: service_completed_successfully + postgres: + condition: service_healthy + mongodb: + condition: service_healthy + environment: + STACK_BENCH_CONTROLLER_IMAGE: ${STACK_BENCH_CONTROLLER_IMAGE:?set STACK_BENCH_CONTROLLER_IMAGE to the manifest digest reference} + STACK_BENCH_IMAGE: ${STACK_BENCH_BUILD_IMAGE:?set STACK_BENCH_BUILD_IMAGE to the manifest digest reference} + STACK_BENCH_RELEASE_MANIFEST: ${STACK_BENCH_RELEASE_MANIFEST:-} + STACK_BENCH_APPLIANCE: "1" + STACK_BENCH_COMPOSE_FILE: /opt/stack-bench/appliance/docker-compose.yaml + STACK_BENCH_WORK_DIR: /var/lib/stack-bench/work + STACK_BENCH_RESULTS_DIR: /var/lib/stack-bench/results + STACK_BENCH_SUPERVISOR_DIR: /var/lib/stack-bench/controller-home/supervisor + STACK_BENCH_RUNTIME_DIR: /var/lib/stack-bench/controller-home/runtime + STACK_BENCH_RESOURCE_LOCK_DIR: /var/lib/stack-bench/controller-home/resource-locks + STACK_BENCH_RELEASE_DEPS_VOLUME: stack-bench-release-deps + STACK_BENCH_LINUX_CLI: /opt/stack-bench-release-deps/spacetimedb-cli + STDB_PACKAGE: /opt/stack-bench-release-deps/bindings-typescript + SPACETIME_BIN: /opt/stack-bench-release-deps/spacetimedb-cli + STACK_BENCH_AGENT_AUTH: ${STACK_BENCH_AGENT_AUTH:-subscription-token} + STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE: ${STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE:-/var/lib/stack-bench/secrets/claude_subscription_token} + STACK_BENCH_ANTHROPIC_API_KEY_FILE: ${STACK_BENCH_ANTHROPIC_API_KEY_FILE:-} + HOME: /var/lib/stack-bench/controller-home + volumes: + - type: bind + source: /var/run/docker.sock + target: /var/run/docker.sock + - type: bind + source: /var/lib/stack-bench + target: /var/lib/stack-bench + - type: volume + source: release-deps + target: /opt/stack-bench-release-deps + read_only: true + tmpfs: + - /tmp:size=1g,mode=1777 + command: ["--help"] + + dashboard: + extends: + service: controller + profiles: ["dashboard"] + network_mode: bridge + environment: + STACK_BENCH_DASHBOARD_CONTROL_SECRET_FILE: ${STACK_BENCH_DASHBOARD_CONTROL_SECRET_FILE:-/var/lib/stack-bench/secrets/dashboard_control_secret} + ports: + - "127.0.0.1:7331:7331" + command: ["dashboard", "--host", "0.0.0.0", "--port", "7331", "--allow-container-bind"] + + postgres: + image: postgres:16@sha256:219341e4cedb06c8634f80af40851da3425b41b76603fd890272f58e37e139f7 + platform: linux/amd64 + container_name: stack-bench-postgres + ports: ["127.0.0.1:6532:5432"] + environment: + POSTGRES_USER: appuser + POSTGRES_PASSWORD: local-app-password + POSTGRES_DB: app + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U appuser -d app"] + interval: 5s + timeout: 5s + retries: 12 + + mongodb: + image: mongo:7@sha256:554a9bb1ec6e00c40ba078a41974a834d1a9a8ab1772645b69142afecc87f082 + platform: linux/amd64 + container_name: stack-bench-mongodb + ports: ["127.0.0.1:6537:27017"] + volumes: + - mongodata:/data/db + healthcheck: + test: ["CMD", "mongosh", "--quiet", "--eval", "db.runCommand({ping:1})"] + interval: 5s + timeout: 5s + retries: 12 + +volumes: + release-deps: + name: stack-bench-release-deps + pgdata: + name: stack-bench-appliance-pgdata + mongodata: + name: stack-bench-appliance-mongodata diff --git a/tools/stack-bench/appliance/operator.env.example b/tools/stack-bench/appliance/operator.env.example new file mode 100644 index 00000000000..ad900f63759 --- /dev/null +++ b/tools/stack-bench/appliance/operator.env.example @@ -0,0 +1,20 @@ +# Subscription billing is the default. Generate a dedicated long-lived Claude +# setup token, write only the token to this mode-0600 file, and never commit it. +STACK_BENCH_AGENT_AUTH=subscription-token +STACK_BENCH_CLAUDE_OAUTH_TOKEN_FILE=/var/lib/stack-bench/secrets/claude_subscription_token + +# To bill through an API key instead, set the mode to api-key and provide an +# absolute path below /var/lib/stack-bench containing only that key. +# STACK_BENCH_ANTHROPIC_API_KEY_FILE=/var/lib/stack-bench/secrets/anthropic_api_key + +# The dashboard reads this separate operator secret from a mode-0600 file. +# The file must contain at least 32 random characters on one line. +STACK_BENCH_DASHBOARD_CONTROL_SECRET_FILE=/var/lib/stack-bench/secrets/dashboard_control_secret + +# Both image values must be registry references ending in @sha256:<64 hex chars>. +STACK_BENCH_CONTROLLER_IMAGE=registry.example/stack-bench-controller@sha256:replace-with-release-digest +STACK_BENCH_BUILD_IMAGE=registry.example/stack-bench-build@sha256:replace-with-release-digest + +# Optional for internal campaigns and required for a distributed release +# campaign. The file must be below the appliance state root. +STACK_BENCH_RELEASE_MANIFEST=/var/lib/stack-bench/release/release.json diff --git a/tools/stack-bench/backends/minimal/mongodb-1.5.md b/tools/stack-bench/backends/minimal/mongodb-1.5.md new file mode 100644 index 00000000000..8b9e91fbdba --- /dev/null +++ b/tools/stack-bench/backends/minimal/mongodb-1.5.md @@ -0,0 +1,19 @@ +# MongoDB + +Use MongoDB for the application data. Choose the libraries, architecture, and +project structure. + +## Connection + +| Setting | Value | +|---|---| +| `DATABASE_URL` | `` | +| Web application | `http://localhost:` | + +The MongoDB service is already running. Use the exact `DATABASE_URL`. Do not +start another MongoDB server, connect to another instance, or create another +database. Serve the complete application on ``. +Create `/app/start.sh`. From a clean source checkout, it must install +dependencies, build the complete application, and start it on ``. +The script must not change source files. Leave the application running when the +work is complete. diff --git a/tools/stack-bench/backends/minimal/postgres-1.5.md b/tools/stack-bench/backends/minimal/postgres-1.5.md new file mode 100644 index 00000000000..4a9124a88d2 --- /dev/null +++ b/tools/stack-bench/backends/minimal/postgres-1.5.md @@ -0,0 +1,19 @@ +# PostgreSQL + +Use PostgreSQL for the application data. Choose the libraries, architecture, +and project structure. + +## Connection + +| Setting | Value | +|---|---| +| `DATABASE_URL` | `` | +| Web application | `http://localhost:` | + +The PostgreSQL service is already running. Use the exact `DATABASE_URL`. Do not +start another PostgreSQL server, connect to another instance, or create another +database. Serve the complete application on ``. +Create `/app/start.sh`. From a clean source checkout, it must install +dependencies, build the complete application, and start it on ``. +The script must not change source files. Leave the application running when the +work is complete. diff --git a/tools/stack-bench/backends/minimal/spacetime-1.6.md b/tools/stack-bench/backends/minimal/spacetime-1.6.md new file mode 100644 index 00000000000..4d686b4d547 --- /dev/null +++ b/tools/stack-bench/backends/minimal/spacetime-1.6.md @@ -0,0 +1,28 @@ +# SpacetimeDB + +Use SpacetimeDB for the application data. Put the TypeScript module in the +required directory below. Choose the schema, libraries, architecture, and the +rest of the project structure. + +## Connection + +Use the connection settings below. + +| Setting | Value | +|---|---| +| Server URI | `` | +| Module name | `` | +| SpacetimeDB CLI | `` | +| TypeScript SDK package | `` | +| Module source directory | `/app/backend/spacetimedb` | +| Web application | `http://localhost:` | + +Publish only the named module to the exact server URI. Local publish and +development commands must use `--yes`. Do not pipe confirmation input, publish +anonymously, or use the hosted service. Create `/app/start.sh`. From a clean +source checkout, it must install dependencies, build the complete application, +and start it on ``. The script must not change source files. Leave +the application running when the work is complete. + +The included TypeScript core SDK reference describes the available core API syntax. +CLI `--help` is available for command syntax. diff --git a/tools/stack-bench/backends/model-free-stub.md b/tools/stack-bench/backends/model-free-stub.md new file mode 100644 index 00000000000..c450d2d44b5 --- /dev/null +++ b/tools/stack-bench/backends/model-free-stub.md @@ -0,0 +1,7 @@ +# Internal model-free fixture + +Use the test service supplied by Stack Bench. Leave the app running on the +assigned client port when the work is complete. + +This fixture is only for internal harness tests. It is not a product stack and +must not be used in stack comparisons. diff --git a/tools/stack-bench/backends/mongodb.md b/tools/stack-bench/backends/mongodb.md new file mode 100644 index 00000000000..5b5ee1468b9 --- /dev/null +++ b/tools/stack-bench/backends/mongodb.md @@ -0,0 +1,64 @@ +# Backend: MongoDB + +An Express API server with Socket.io for live updates, Mongoose over MongoDB, +and a React client. + +## Layout + +``` +/ + server/ + package.json express, socket.io, mongoose, dotenv, tsx + .env DATABASE_URL and PORT + src/models.ts Mongoose schemas and models + src/index.ts Express routes, Socket.io handlers + client/ + package.json react, react-dom, vite, socket.io-client + vite.config.ts server.port , proxy /api and /socket.io to + index.html + src/main.tsx + src/App.tsx +``` + +## Deploy + +```bash +cd server && npm install && npm run dev # on +cd client && npm install && npm run dev # on +``` + +Mongoose creates collections on first write; there is no migration step. + +The server prints to the terminal running `npm run dev`; it restarts on save, +so a code change is live without redeploying. + +Keep existing application data when you change the schema. Do not drop +collections during upgrades or repairs. + +## Branding & Styling + +- App title: **"MongoDB "** +- Dark theme using official MongoDB brand colors: + - Primary: `#00ED64` (MongoDB green) + - Primary hover: `#00C957` (darker green) + - Secondary: `#00684A` (MongoDB forest green) + - Background: `#001E2B` (MongoDB dark slate) + - Surface: `#023430` (deep green-slate) + - Border: `#1C2D38` (muted slate border) + - Text: `#E8EDEB` (light gray) + - Text muted: `#889397` (MongoDB gray) + - Accent: `#00ED64` (MongoDB green) + - Success: `#00ED64` (green for online indicators) + - Warning: `#FFC010` (MongoDB amber) + - Danger: `#FF4F4F` (MongoDB red) + + +## Configuration + +| Setting | Value | +|---|---| +| `DATABASE_URL` | `` | +| API server port | `` | +| Client dev server | `` | + +Use this exact `DATABASE_URL`. Do not point at another MongoDB instance. diff --git a/tools/stack-bench/backends/postgres.md b/tools/stack-bench/backends/postgres.md new file mode 100644 index 00000000000..7bc7b7e259c --- /dev/null +++ b/tools/stack-bench/backends/postgres.md @@ -0,0 +1,66 @@ +# Backend: PostgreSQL + +An Express API server with Socket.io for live updates, Drizzle ORM over +PostgreSQL, and a React client. + +## Layout + +``` +/ + server/ + package.json express, socket.io, drizzle-orm, pg, dotenv, tsx + .env DATABASE_URL and PORT + drizzle.config.ts + src/schema.ts Drizzle table definitions + src/index.ts Express routes, Socket.io handlers + client/ + package.json react, react-dom, vite, socket.io-client + vite.config.ts server.port , proxy /api and /socket.io to + index.html + src/main.tsx + src/App.tsx +``` + +## Deploy + +```bash +cd server && npm install && npx drizzle-kit push && npm run dev # on +cd client && npm install && npm run dev # on +``` + +Re-run `npx drizzle-kit push` after any schema change. + +The server prints to the terminal running `npm run dev`; it restarts on save, +so a code change is live without redeploying. + +Keep existing application data when you change the schema. Do not drop or +recreate tables during upgrades or repairs. + +## Branding & Styling + +- App title: **"PostgreSQL "** +- Dark theme using official PostgreSQL brand colors: + - Primary: `#336791` (PostgreSQL blue) + - Primary hover: `#008bb9` (lighter PostgreSQL blue) + - Secondary: `#0064a5` (dark PostgreSQL blue) + - Background: `#1a1a2e` (dark navy) + - Surface: `#16213e` (slightly lighter) + - Border: `#2a2a4a` (muted border) + - Text: `#e8e8e8` (light gray) + - Text muted: `#848484` (PostgreSQL light grey) + - Accent: `#008bb9` (PostgreSQL light blue) + - Success: `#27ae60` (green for online indicators) + - Warning: `#f26522` (PostgreSQL light orange) + - Danger: `#cc3b03` (PostgreSQL dark orange/red) + + +## Configuration + +| Setting | Value | +|---|---| +| `DATABASE_URL` | `` | +| API server port | `` | +| Client dev server | `` | + +Use this exact `DATABASE_URL`. Do not point at another PostgreSQL instance and do +not create databases outside it. diff --git a/tools/stack-bench/backends/spacetime.md b/tools/stack-bench/backends/spacetime.md new file mode 100644 index 00000000000..37a11633139 --- /dev/null +++ b/tools/stack-bench/backends/spacetime.md @@ -0,0 +1,99 @@ +# Backend: SpacetimeDB + +The database runs your server logic. There is no separate API server and no ORM: +tables and reducers are a WASM module you publish, and the client subscribes to +tables and calls reducers over a live connection. + +## Layout + +``` +/ + backend/spacetimedb/ + package.json { "type": "module", dependencies: { "spacetimedb": "" }, + devDependencies: { "typescript": "~5.6.2" } } ← required; the build runs tsc from node_modules + tsconfig.json + src/schema.ts tables and indexes + src/index.ts reducers and lifecycle hooks + client/ + package.json react, react-dom, vite, and "spacetimedb": "" + vite.config.ts server.port must be + index.html + src/config.ts MODULE_NAME and SPACETIMEDB_URI + src/main.tsx React entry + src/App.tsx + src/module_bindings/ generated; never edit by hand +``` + +## Deploy + +Publish the module, then regenerate the client bindings from it: + +```bash + publish --module-path backend/spacetimedb -s --yes + generate --lang typescript --out-dir client/src/module_bindings --module-path backend/spacetimedb +``` + +**While iterating, run development mode instead of republishing by hand.** It +watches the module and automatically rebuilds, publishes, and regenerates the +client bindings on every save: + +```bash + dev --module-path backend/spacetimedb -s --yes +``` + +Leave it running in the background while you work. The manual commands below +are for one-off publishes and for the first deploy. + +Republish after any server change, and regenerate after any schema change. + +Keep existing application data when you change the schema. + +Always use `--yes` for local publish and development commands. It selects the +CLI's non-interactive authentication flow for the target server. Do not pipe +`y` into the command and do not publish anonymously. Use the same local +identity for every publish to the named module. + +Then start the client: + +```bash +cd client && npm install && npm run dev +``` + +` logs -s ` shows module output, including reducer errors. + +To inspect stored data while debugging: + +```bash + sql "SELECT * FROM item LIMIT 5" -s +``` + +## Branding & Styling + +- App title: **"SpacetimeDB "** +- Dark theme using official SpacetimeDB brand colors: + - Primary: `#4cf490` (SpacetimeDB green) + - Primary hover: `#4cf490bf` (green 75% opacity) + - Secondary: `#a880ff` (SpacetimeDB purple) + - Background: `#0d0d0e` (shade2 — near black) + - Surface: `#141416` (shade1 — slightly lighter) + - Border: `#202126` (n6) + - Text: `#e6e9f0` (n1 — light gray) + - Text muted: `#6f7987` (n4) + - Accent: `#02befa` (SpacetimeDB blue) + - Success: `#4cf490` (green — same as primary) + - Warning: `#fbdc8e` (SpacetimeDB yellow) + - Danger: `#ff4c4c` (SpacetimeDB red) + - Gradient (optional, for headers): `linear-gradient(266deg, #4cf490 0%, #8a38f5 100%)` (green to purple) + + +## Configuration + +| Setting | Value | +|---|---| +| Server URI | `` | +| Module name | `` | +| Client dev server | `` | + +The SDK reference for writing modules and clients is in the skill documents +included with these instructions. Follow them for API specifics: import paths, +type builders, accessors and context typing. diff --git a/tools/stack-bench/commands/agent.ts b/tools/stack-bench/commands/agent.ts new file mode 100644 index 00000000000..98746c2100d --- /dev/null +++ b/tools/stack-bench/commands/agent.ts @@ -0,0 +1,979 @@ +#!/usr/bin/env node +// Drives one headless coding session: build a level, upgrade to the next, or fix +// reported bugs. Self-contained — no dependency on the sequential-upgrade tool. +// +// Cost and token usage come from the CLI's own JSON result, so there is no +// telemetry collector to run. +// +// Usage: +// node dist/commands/agent.js --mode build --backend spacetime --level 1 --app +// node dist/commands/agent.js --mode upgrade --backend spacetime --level 2 --app +// node dist/commands/agent.js --mode fix --backend spacetime --app +// +// Prints a JSON line: { appDir, costUsd, tokens, durationMs, sessionId, ok } + +import { execFileSync } from 'node:child_process'; +import type { ExecFileSyncOptionsWithStringEncoding } from 'node:child_process'; +import { readFileSync, writeFileSync, mkdirSync, existsSync, realpathSync, + openSync, readSync, closeSync, readdirSync } from 'node:fs'; +import { join, dirname, resolve, relative, isAbsolute, sep } from 'node:path'; +import { homedir } from 'node:os'; +import { fileURLToPath } from 'node:url'; +import { loadTrack, levelPrompt, appendix, suitesFor, dbName, moduleName, portsFor, DEFAULT_TRACK } from '../src/composition/tracks.js'; +import type { Track, TrackDefinition } from '../src/composition/tracks.js'; +import type { BackendLease } from '../src/runtime/backend-lease.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { resolveDefaultGuidanceForStack, type ResolvedSkills } + from '../src/campaigns/condition-compiler.js'; +import type { ExactRecipeRequest, RecipeBinding } from '../src/composition/recipe-release.js'; +import { createBoundRecipeTaskRequest, resolveBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import { agentVisibleContractText } from '../src/composition/agent-visible-contract.js'; +import { leaseFromEnv } from '../src/runtime/backend-lease.js'; +import { resolveContainerImage } from '../src/runtime/container-image.js'; +import { hashDirectory, sessionProvenance, sha256 } from '../src/evidence/provenance.js'; +import { executeStackCapability } from '../src/stacks/stack-adapter-contract.js'; +import type { StackRunPorts } from '../src/stacks/stack-adapter-contract.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { dockerMountArguments } from '../src/runtime/container-mount.js'; +import { normalizePromptText, readAgentSkillDocuments, selectAgentSkills } from '../src/agents/agent-materials.js'; +import { codingSessionFailure, DEFAULT_THROTTLE_MAX_WAIT_MS, providerSessionFailure, + runCodingSessionWithRecovery } from '../src/agents/coding-session-recovery.js'; +import type { CodingSessionRecoveryResult } from '../src/agents/coding-session-recovery.js'; +import { AGENT_PROCESS_TIMEOUT_MS } from '../src/agents/coding-session-timeouts.js'; +import { assertNewOrEmptyDirectory } from '../src/runtime/path-safety.js'; +import { claudeRatesForModel } from '../src/evidence/claude-usage-cost.js'; +import { PRICING_UNIT, validatePricingAuthority } + from '../src/evidence/pricing-authority.js'; +import type { PricingAuthority } from '../src/evidence/pricing-authority.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const REPO = resolve(ROOT, '..', '..'); +const CONTROL_COMMAND_TIMEOUT_MS = 120_000; +const DEFAULT_CODING_INTERRUPTION_RETRIES = 2; + +type UnknownRecord = Record; +interface GuidanceDocument { + path: string; + sha256: string; + bytes: number; + applicationInterface: 'http' | 'reducer'; +} + +interface PromptMaterials { + skillsText?: string; + requirementText?: string; + contractText?: string; + startingCatalog?: string; +} + +type RecipeTaskRequest = Parameters[1] & { + recipe?: Exclude; +}; + +interface AgentArgs { + mode: string; + backend: string; + app: string; + level: number; + runIndex: number; + model: string; + guidance: string; + track: string; + pricing: Readonly | null; + guidanceDocument?: GuidanceDocument; + credentialAliases?: Readonly>; + recipe?: string; + recipeTask?: RecipeTaskRequest; + thinking?: string; + maxBudgetUsd?: number; + skills?: string[]; + skillIdentity?: ResolvedSkills; + apiKey?: string; + printPrompt?: boolean; +} + +interface ThinkingVolume { + blocks: number; + signatureBytes: number; + bytesPerBlock: number; +} + +interface SessionUsage { + input_tokens?: number; + output_tokens?: number; + cache_creation_input_tokens?: number; + cache_read_input_tokens?: number; +} + +const isRecord = (value: unknown): value is UnknownRecord => + value !== null && typeof value === 'object' && !Array.isArray(value); + +const stringValue = (value: unknown): string | null => typeof value === 'string' ? value : null; + +function sessionUsage(value: unknown): SessionUsage { + return isRecord(value) ? { + input_tokens: typeof value.input_tokens === 'number' ? value.input_tokens : undefined, + output_tokens: typeof value.output_tokens === 'number' ? value.output_tokens : undefined, + cache_creation_input_tokens: typeof value.cache_creation_input_tokens === 'number' + ? value.cache_creation_input_tokens : undefined, + cache_read_input_tokens: typeof value.cache_read_input_tokens === 'number' + ? value.cache_read_input_tokens : undefined, + } : {}; +} + +// The benchmark runs its own SpacetimeDB host so that measurements describe the +// module under test rather than whatever else is published on a shared machine, +// and so restarting it for durability tests cannot take somebody else down. +const STDB_URI = process.env.STACK_BENCH_STDB_URI ?? 'http://127.0.0.1:3210'; + +// Build the app against the SpacetimeDB in THIS repository, not a published +// release: otherwise a change to the host, the CLI or the module SDK is not +// under test, and a result describes software nobody is working on. +const LOCAL_CLI = join(REPO, 'target', 'release', 'spacetimedb-cli.exe'); +const STDB_BIN = process.env.SPACETIME_BIN ?? (existsSync(LOCAL_CLI) ? LOCAL_CLI : 'spacetime'); +const LOCAL_PKG = process.env.STDB_PACKAGE ?? join(REPO, 'crates', 'bindings-typescript'); + +// Both shells and npm read a Windows backslash as an escape, so paths handed to +// the model are written the one way every tool agrees on. +const fwd = (path: string): string => path.split('\\').join('/'); + +// Keep the provider's default thinking budget unless an experiment selects one +// explicitly. The run records observed reasoning volume so default changes are +// visible in the evidence. +const THINKING_TOKENS = process.env.STACK_BENCH_THINKING ?? null; + +// Pin and record effort instead of inheriting an ambient CLI setting. +const EFFORT = process.env.STACK_BENCH_EFFORT ?? 'high'; + +// Coding sessions always run in Docker; there is no host execution path. The +// build container cannot see benchmark definitions, and the model-free +// container test verifies its lifecycle and SpacetimeDB publish/log path. +const IMAGE = process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE; + +// Set once in main(), because the addresses a build is TOLD to use depend on +// where the build runs. +// +// Host services — the databases and the SpacetimeDB host — stay on +// the machine, and `localhost` inside a container is the container. Those +// addresses are rewritten to Docker's host alias. Dev-server ports are NOT +// rewritten: those servers start inside the container and are published back +// out, so the grader on the host still reaches them at localhost. +export function hostServiceAddress(env: NodeJS.ProcessEnv = process.env): string { + return env.STACK_BENCH_HOST_ALIAS + ?? (env.STACK_BENCH_APPLIANCE === '1' ? '127.0.0.1' : 'host.docker.internal'); +} + +const HOST_ADDR = hostServiceAddress(); +const hostUrl = (url: string): string => url.replace(/127\.0\.0\.1|localhost/g, HOST_ADDR); + +// Where the two artifacts under test are mounted. Container paths also keep the +// repository root out of the prompt. +const C_PKG = '/deps/spacetimedb.tgz'; +const C_BIN = '/deps/spacetimedb-cli'; + +// The Linux build of the repository's CLI, which is what a container mounts at +// C_BIN. Built by container/build-linux-cli.sh; target/release holds the +// Windows binary and the two must not be confused for each other. +const LINUX_CLI = process.env.STACK_BENCH_LINUX_CLI + ?? join(ROOT, 'container', 'bin', 'spacetimedb-cli'); + +// Measure observed reasoning from the session transcript. The text of a +// thinking block is redacted at the wire level -- `thinking` is +// an empty string and the content survives only as an opaque signature -- so +// this counts blocks and signature bytes. Neither is a token count, but both +// move with reasoning volume, and they are what the transcript actually has. +function thinkingVolume(appDir: string, sessionId: string | null | undefined): ThinkingVolume | null { + if (!sessionId) return null; + try { + const store = join(homedir(), '.claude', 'projects'); + if (!existsSync(store)) return null; + const want = resolve(appDir).replace(/[\\/:]/g, '-').toLowerCase(); + const dir = readdirSync(store).find(d => { + const n = d.toLowerCase(); + return n === want || n === want.replace(/^-+/, ''); + }); + const file = dir && join(store, dir, `${sessionId}.jsonl`); + if (!file || !existsSync(file)) return null; + + let blocks = 0, bytes = 0; + for (const line of readFileSync(file, 'utf8').split('\n')) { + if (!line.includes('"thinking"')) continue; // cheap filter before parsing + let record: unknown; + try { record = JSON.parse(line); } catch { continue; } + if (!isRecord(record) || !isRecord(record.message) + || !Array.isArray(record.message.content)) continue; + for (const content of record.message.content) { + if (!isRecord(content) || content.type !== 'thinking') continue; + blocks++; + bytes += stringValue(content.signature)?.length ?? 0; + } + } + return { blocks, signatureBytes: bytes, + bytesPerBlock: blocks ? Math.round(bytes / blocks) : 0 }; + } catch { return null; } +} + +function combinedThinkingVolume(appDir: string, sessionIds: readonly (string | null | undefined)[]): ThinkingVolume | null { + const volumes: ThinkingVolume[] = [...new Set(sessionIds.filter((id): id is string => Boolean(id)))] + .map(id => thinkingVolume(appDir, id)).filter((item): item is ThinkingVolume => item !== null); + if (!volumes.length) return null; + const blocks = volumes.reduce((sum, item) => sum + item.blocks, 0); + const signatureBytes = volumes.reduce((sum, item) => sum + item.signatureBytes, 0); + return { blocks, signatureBytes, + bytesPerBlock: blocks ? Math.round(signatureBytes / blocks) : 0 }; +} + + +// Record the Linux CLI executed by the container. The host and container +// binaries can change independently and must not share an identity. +function linuxSpacetimeVersion(image: string): { commit: string | null; binarySha256: string | null; raw: string } { + try { + const releaseVolume = process.env.STACK_BENCH_RELEASE_DEPS_VOLUME?.trim() || null; + const mountArgs = releaseVolume + ? dockerMountArguments({ kind: 'volume', source: releaseVolume, + target: '/release-deps', readOnly: true }) + : ['-v', `${LINUX_CLI}:/deps/spacetimedb-cli:ro`]; + const entrypoint = releaseVolume ? '/release-deps/spacetimedb-cli' : '/deps/spacetimedb-cli'; + const out = execFileSync('docker', + ['run', '--rm', ...mountArgs, '--entrypoint', entrypoint, image, '--version'], + { encoding: 'utf8', stdio: 'pipe', env: { ...process.env, MSYS_NO_PATHCONV: '1' }, + timeout: CONTROL_COMMAND_TIMEOUT_MS }); + const commit = out.match(/Commit:\s*([0-9a-f]+)/i)?.[1] ?? null; + return { commit, binarySha256: sha256(readFileSync(LINUX_CLI)), + raw: out.trim().split(/\r?\n/).slice(0, 2).join(' ') }; + } catch { return { commit: null, binarySha256: null, raw: 'unknown' }; } +} + +function bindingsIdentity(pkgDir: string): { package: string; sourceSha256: string | null; sourceFiles: number } { + try { + const p = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')); + const source = hashDirectory(pkgDir, { exclude: name => + /(^|\/)(node_modules|dist|target)(\/|$)/.test(name) }); + return { package: `${p.name}@${p.version}`, sourceSha256: source.sha256, + sourceFiles: source.files.length }; + } catch { return { package: 'unknown', sourceSha256: null, sourceFiles: 0 }; } +} + +// The CLI version inside the build image. Read by running it, not by trusting +// the tag: the image is pinned by ARG and a tag can be moved. +function imageCliVersion(image: string): string { + try { + return execFileSync('docker', ['run', '--rm', '--entrypoint', 'claude', image, '--version'], + { encoding: 'utf8', stdio: 'pipe', env: { ...process.env, MSYS_NO_PATHCONV: '1' }, + timeout: CONTROL_COMMAND_TIMEOUT_MS }).trim(); + } catch { return 'unknown'; } +} + +function imageNodeVersion(image: string): string { + try { + return execFileSync('docker', ['run', '--rm', '--entrypoint', 'node', image, '--version'], + { encoding: 'utf8', stdio: 'pipe', env: { ...process.env, MSYS_NO_PATHCONV: '1' }, + timeout: CONTROL_COMMAND_TIMEOUT_MS }).trim(); + } catch { return 'unknown'; } +} + +function containerImage(name: string): { reference: string; imageId: string | null | undefined } { + try { + const out = execFileSync('docker', ['inspect', '-f', '{{.Config.Image}} {{.Image}}', name], + { encoding: 'utf8', stdio: 'pipe', timeout: CONTROL_COMMAND_TIMEOUT_MS }).trim(); + const [reference, imageId] = out.split(/\s+/, 2); + return { reference: reference ?? '', imageId }; + } catch { return { reference: 'unknown', imageId: null }; } +} + +// Record ambient provider configuration that can change model behaviour while +// replacing credential values with presence markers. +function ambientEnv(): Record { + const seen: Record = {}; + for (const [k, v] of Object.entries(process.env)) { + if (!/^(CLAUDE|ANTHROPIC|MAX_THINKING|DISABLE_AUTOUPDATER|FORCE_PROMPT)/.test(k)) continue; + // Never record a credential, only that one was present. + seen[k] = /KEY|TOKEN|SECRET/i.test(k) ? '' : v; + } + return seen; +} + +function normalizeGuidance(value: string): 'neutral' | 'prescribed' { + if (value === 'neutral' || value === 'prescribed') return value; + throw new Error(`guidance must be neutral or prescribed, received ${JSON.stringify(value)}`); +} + +function parseArgs(argv: readonly string[]): AgentArgs { + const a: Partial & Pick = { level: 1, runIndex: 0, model: 'claude-sonnet-5', guidance: 'prescribed', + track: DEFAULT_TRACK }; + for (let i = 2; i < argv.length; i++) { + switch (argv[i]) { + case '--mode': a.mode = argv[++i] ?? ''; break; + case '--track': a.track = argv[++i] ?? ''; break; + case '--backend': a.backend = argv[++i] ?? ''; break; + case '--level': a.level = parseInt(argv[++i] ?? '', 10); break; + case '--app': a.app = argv[++i] ?? ''; break; + case '--run-index': a.runIndex = parseInt(argv[++i] ?? '', 10); break; + case '--model': a.model = argv[++i] ?? ''; break; + case '--pricing-json': a.pricing = JSON.parse(argv[++i] ?? ''); break; + case '--guidance': a.guidance = normalizeGuidance(argv[++i] ?? ''); break; + case '--guidance-document-json': a.guidanceDocument = JSON.parse(argv[++i] ?? ''); break; + case '--credential-aliases-json': a.credentialAliases = JSON.parse(argv[++i] ?? ''); break; + case '--recipe': a.recipe = argv[++i] ?? ''; break; + case '--recipe-task-json': a.recipeTask = JSON.parse(argv[++i] ?? ''); break; + case '--thinking': a.thinking = argv[++i] ?? ''; break; + case '--max-budget-usd': a.maxBudgetUsd = Number(argv[++i]); break; + // Comma-separated skill directories to inline, e.g. + // --skills typescript-server,typescript-client,cli + case '--skills': a.skills = (argv[++i] ?? '').split(',').map((skill: string) => skill.trim()).filter(Boolean); break; + case '--skills-json': a.skills = JSON.parse(argv[++i] ?? ''); break; + case '--skill-identity-json': a.skillIdentity = validateSkillIdentity( + JSON.parse(argv[++i] ?? '')); break; + // Container or nothing. For sweeps whose claim is that no build could + // reach the grader — there, a silent fallback would be a false claim. + // An API key, when supplied, is used instead of the mounted plan + // credential — it keeps a rotating token off the build's filesystem. + case '--api-key': a.apiKey = argv[++i] ?? ''; break; + case '--print-prompt': a.printPrompt = true; break; + default: console.error(`Unknown argument: ${argv[i]}`); process.exit(2); + } + } + if (!a.mode || !a.backend || !a.app) { + console.error('Usage: node dist/commands/agent.js --mode build|upgrade|fix --backend --app [--level N]'); + process.exit(2); + } + if (a.maxBudgetUsd !== undefined && (!Number.isFinite(a.maxBudgetUsd) || a.maxBudgetUsd <= 0)) { + throw new Error('--max-budget-usd must be a positive number'); + } + if (a.pricing !== undefined) { + a.pricing = validatePricingAuthority(a.pricing, { at: '--pricing-json' }); + } else if (a.maxBudgetUsd !== undefined) { + const rates = claudeRatesForModel(a.model); + if (!rates) throw new Error(`no default pricing is recorded for model ${a.model}`); + a.pricing = validatePricingAuthority({ unit: PRICING_UNIT, rates }, + { at: 'default pricing' }); + } else { + a.pricing = null; + } + return { mode: a.mode, backend: a.backend, app: a.app, level: a.level, + runIndex: a.runIndex, model: a.model, guidance: a.guidance, track: a.track, + pricing: a.pricing ?? null, ...(a.guidanceDocument ? { guidanceDocument: a.guidanceDocument } : {}), + ...(a.credentialAliases ? { credentialAliases: a.credentialAliases } : {}), + ...(a.recipe ? { recipe: a.recipe } : {}), ...(a.recipeTask ? { recipeTask: a.recipeTask } : {}), + ...(a.thinking ? { thinking: a.thinking } : {}), + ...(a.maxBudgetUsd !== undefined ? { maxBudgetUsd: a.maxBudgetUsd } : {}), + ...(a.skills ? { skills: a.skills } : {}), + ...(a.skillIdentity ? { skillIdentity: a.skillIdentity } : {}), + ...(a.apiKey ? { apiKey: a.apiKey } : {}), + ...(a.printPrompt ? { printPrompt: true } : {}) }; +} + +const dbUrl = (backend: string, runIndex: number, dbPort: number | null, track: Track): string | null => { + const url = executeStackCapability(STACK_ADAPTER_REGISTRY.get(backend), 'agent', 'connection-url', + { dbPort, database: dbName(track, runIndex), hostUrl }); + return typeof url === 'string' ? url : null; +}; + +// Create the leased database before the app connects. A build clears its schema. +// A reset between suites preserves the schema required by the running app. +interface DatabasePreparationLease { + runId: string; + track: string; + runIndex: number; + resources: Pick; +} + +type DatabaseCommandOptions = Pick; + +type DatabaseCommandExecutor = (command: string, args: readonly string[], + options: DatabaseCommandOptions) => string; + +const databaseCommandExecutor: DatabaseCommandExecutor = (command, args, options) => + String(execFileSync(command, args, { ...options, encoding: 'utf8' })); + +interface DatabasePreparationOptions { + exec?: DatabaseCommandExecutor; + stdbBin?: string; + lease?: DatabasePreparationLease; +} + +export function ensureDatabase(backend: string, runIndex: number, dbPort: number | null, + track: Pick, wipe = false, + { exec = databaseCommandExecutor, stdbBin = STDB_BIN, lease: suppliedLease }: DatabasePreparationOptions = {}) { + const lease = suppliedLease ?? leaseFromEnv(process.env, { backend, active: true }).lease; + if (lease.runIndex !== runIndex || lease.track !== track.name) { + throw new Error(`backend lease ${lease.runId} belongs to ${lease.track}/run${lease.runIndex}, ` + + `not ${track.name}/run${runIndex}`); + } + const expectedName = dbName(track, runIndex); + const name = lease.resources.database ?? expectedName; + return executeStackCapability(STACK_ADAPTER_REGISTRY.get(backend), 'database', 'prepare', { + lease, name, expectedName, wipe, exec, cli: stdbBin, + expectedServerUri: STDB_URI, expectedModule: moduleName(track, runIndex), dbPort, + }); +} + +// Prescribed guidance chooses an implementation stack. Neutral guidance gives +// only stack access facts and the selected API references. +export function readBackendGuidanceDocument(document: GuidanceDocument | undefined, fallbackRelativePath: string): string { + if (typeof fallbackRelativePath !== 'string' || !fallbackRelativePath) { + throw new Error('backend guidance fallback path is required'); + } + if (document !== undefined) { + const fields = new Set(['path', 'sha256', 'bytes', 'applicationInterface']); + if (!document || typeof document !== 'object' || Array.isArray(document) + || Object.keys(document).some(field => !fields.has(field)) + || typeof document.path !== 'string' || !document.path || isAbsolute(document.path) + || document.path.includes('\\') + || !/^[a-f0-9]{64}$/.test(document.sha256) + || !Number.isSafeInteger(document.bytes) || document.bytes < 0 + || !['http', 'reducer'].includes(document.applicationInterface)) { + throw new Error('campaign guidance document identity is invalid'); + } + } + const root = realpathSync(ROOT); + const candidate = resolve(root, document?.path ?? fallbackRelativePath); + const candidateRel = relative(root, candidate); + if (candidateRel === '..' || candidateRel.startsWith(`..${sep}`) || isAbsolute(candidateRel)) { + throw new Error('campaign guidance document escapes the Stack Bench root'); + } + const selectedPath = realpathSync(candidate); + const resolvedRel = relative(root, selectedPath); + if (resolvedRel === '..' || resolvedRel.startsWith(`..${sep}`) || isAbsolute(resolvedRel)) { + throw new Error('campaign guidance document resolves outside the Stack Bench root'); + } + const bytes = Buffer.from(normalizePromptText(readFileSync(selectedPath, 'utf8')), 'utf8'); + if (document && (sha256(bytes) !== document.sha256 || bytes.length !== document.bytes)) { + throw new Error(`campaign guidance document changed after compilation: ${document.path}`); + } + return bytes.toString('utf8'); +} + +function validateSkillIdentity(value: unknown): ResolvedSkills { + const fields = new Set(['ids', 'sha256', 'bytes']); + if (!isRecord(value) || Object.keys(value).some(field => !fields.has(field)) + || !Array.isArray(value.ids) || new Set(value.ids).size !== value.ids.length + || value.ids.some(id => typeof id !== 'string' || !/^[a-z][a-z0-9-]*$/.test(id)) + || typeof value.sha256 !== 'string' || !/^[a-f0-9]{64}$/.test(value.sha256) + || !Number.isSafeInteger(value.bytes) || Number(value.bytes) < 0) { + throw new Error('campaign skill identity is invalid'); + } + return { ids: value.ids as string[], sha256: value.sha256, bytes: Number(value.bytes) }; +} + +function backendDoc(args: AgentArgs, p: StackRunPorts, track: Track): string { + const defaultGuidance = resolveDefaultGuidanceForStack(args.guidance, args.backend); + let defaultPath = defaultGuidance?.documents[args.backend]?.path; + if (!defaultPath && args.guidance === 'neutral') { + throw new Error(`neutral guidance has no document for ${args.backend}`); + } + defaultPath ??= join('backends', `${args.backend}.md`); + const raw = readBackendGuidanceDocument(args.guidanceDocument, defaultPath); + return raw + .replaceAll('', String(p.vite)) + .replaceAll('', String(p.express ?? '')) + .replaceAll('', track.title) + .replaceAll('', moduleName(track, args.runIndex)) + .replaceAll('', p.dbPort ? dbUrl(args.backend, args.runIndex, p.dbPort, track) ?? '' : '') + .replaceAll('', hostUrl(STDB_URI)) + .replaceAll('', C_BIN) + .replaceAll('', `file:${C_PKG}`); +} + +// SpacetimeDB is young enough that models have little of it in training data; +// the skill documents are its API reference, equivalent to what the other stacks +// get from having been on the internet for a decade. +// Which documents are inlined is the variable under test in the cost work, so +// it is a flag rather than an edit: both arms of a comparison then come from +// one binary, and run.json records which arm produced each number. +// Whether a containerised run can actually work, checked before anything is +// spent rather than discovered an hour in as a build error the model gets +// blamed for. Returns a reason instead of throwing, because the default and the +// explicit request are answered differently — see resolveIsolation. +// +// SpacetimeDB needs a CLI the container can execute. `target/release/ +// spacetimedb-cli.exe` is a Windows PE binary, so container/build-linux-cli.sh +// compiles a Linux one from this same checkout. The benchmark deliberately +// tests the CLI in THIS repository rather than a published release, so falling +// back to a `spacetime` from the image would measure different software and +// report it under the same name. +function containerBlocker(backend: string): string | null { + try { + execFileSync('docker', ['image', 'inspect', IMAGE], + { stdio: 'pipe', timeout: CONTROL_COMMAND_TIMEOUT_MS }); + } catch (error: unknown) { + const detail = error instanceof Error ? error.message.split('\n')[0] : String(error).split('\n')[0]; + return `cannot verify isolation image ${IMAGE}: ${detail} — ` + + `build it with docker build -t ${IMAGE} ${fwd(join(ROOT, 'container'))}`; + } + if (!executeStackCapability(STACK_ADAPTER_REGISTRY.get(backend), + 'agent', 'linux-cli-required')) return null; + if (!existsSync(LINUX_CLI)) { + return `no Linux SpacetimeDB CLI at ${fwd(LINUX_CLI)} — ` + + 'bash tools/stack-bench/container/build-linux-cli.sh'; + } + // ELF magic, checked rather than assumed: the path existing says nothing + // about which platform the file was built for, and a Windows binary copied + // there would fail deep inside a build as an unexplained publish error. + const magic = Buffer.alloc(4); + try { + const fd = openSync(LINUX_CLI, 'r'); + try { readSync(fd, magic, 0, 4, 0); } finally { closeSync(fd); } + } catch { + return `cannot read the Linux SpacetimeDB CLI at ${fwd(LINUX_CLI)}`; + } + if (magic.toString('binary') !== '\x7fELF') { + return `${fwd(LINUX_CLI)} is not a Linux binary; rebuild it with ` + + 'container/build-linux-cli.sh'; + } + return null; +} + +// There is one coding-session topology to secure and clean up: Docker or fail. +function decideIsolation(args: AgentArgs): { container: true; reason: null } { + const blocker = containerBlocker(args.backend); + if (!blocker) return { container: true, reason: null }; + console.error(`agent.js: isolated build unavailable: ${blocker}`); + console.error(' benchmark coding sessions require the isolation container'); + process.exit(2); +} + +// Every round must retain the container pin written by the first round. A +// missing or non-container marker is ambiguous prior state, never authority to +// choose a second execution topology. +// +// So the decision is made once, at `build`, and recorded beside the app rather +// than inside it (the app directory is what gets copied into source/ and +// audited). Later modes read it back and refuse ambiguous prior state. +// +// The marker lives outside the app because `build` wipes the app directory. +// An app with benchmark state and no marker predates this invariant; refuse it +// rather than guessing where its earlier rounds executed. +function resolveIsolation(args: AgentArgs): { container: true; reason: null } { + const marker = resolve(args.app, '..', '.stack-bench-isolation'); + const backendMarker = resolve(args.app, '..', '.stack-bench-backend'); + + if (args.mode === 'build') { + const decided = decideIsolation(args); + if (!args.printPrompt) { + mkdirSync(dirname(marker), { recursive: true }); + writeFileSync(marker, 'container'); + } + return decided; + } + + if (existsSync(marker)) { + const pinned = readFileSync(marker, 'utf8').trim(); + if (pinned !== 'container') { + console.error(`agent.js: unsupported isolation marker ${JSON.stringify(pinned)}; expected "container"`); + process.exit(2); + } + const blocker = containerBlocker(args.backend); + if (blocker) { + console.error(`agent.js: this run's build ran in a container, but ${blocker}`); + console.error(' refusing to run this round in a different environment'); + process.exit(2); + } + return { container: true, reason: null }; + } + + // No isolation marker. The sibling backend marker is written by every agent + // round, so its presence means a previous round already worked here without + // recording its topology. (A seeded upgrade does not + // have it: restoreSource copies source directories, not dotfiles, so the + // first round of a new run still decides fresh.) + if (existsSync(backendMarker)) { + console.error('agent.js: app has prior benchmark state but no isolation marker'); + console.error(' refusing to guess where earlier rounds ran; start a clean run'); + process.exit(2); + } + const decided = decideIsolation(args); + if (!args.printPrompt) { + mkdirSync(dirname(marker), { recursive: true }); + writeFileSync(marker, 'container'); + } + return decided; +} + +export function buildPrompt(args: AgentArgs, p: StackRunPorts, track: Track, + materials: PromptMaterials = {}): string { + const applicationInterface = args.guidanceDocument?.applicationInterface + ?? resolveDefaultGuidanceForStack(args.guidance, args.backend) + ?.documents[args.backend]?.applicationInterface; + if (applicationInterface !== 'http' && applicationInterface !== 'reducer') { + throw new Error(`stack ${args.backend} has no application interface`); + } + const common = [ + 'Build the app in /app.', + // Container runs only, and identical for every backend so no stack gets + // more guidance than another. + // + // Vite binds `localhost` by default. Inside a container that is the + // container's own loopback, which a published port does not forward to — + // measured: a server on 127.0.0.1 in a container is unreachable from the + // host through -p, the same server on 0.0.0.0 answers 200. Without this + // every containerised run would build a working app the grader cannot + // reach, and fail for a reason that has nothing to do with the database. + '', + 'The web application must listen on 0.0.0.0, not localhost, so it is reachable ' + + 'outside its process.', + '', + '## Stack', + '', + agentVisibleContractText(backendDoc(args, p, track), args.credentialAliases, + applicationInterface), + ]; + const skills = materials.skillsText ?? readAgentSkillDocuments(REPO, args.skills ?? []); + if (skills) common.push('', '## Selected API reference', '', skills); + + if (args.mode === 'resume') { + return [ + 'Restore the existing application to a runnable state.', + '', + 'This is a saved application from an earlier completed run. Install its', + 'dependencies and start its existing database module, server, and web client', + 'as needed. Do not implement features or fix application behavior. Do not', + 'change source files. The saved source must remain byte-for-byte identical.', + '', + 'Output RESUME_COMPLETE when the existing app is running.', + '', + ...common, + ].join('\n'); + } + + if (args.mode === 'fix') { + return [ + 'Fix the reported application bugs.', + '', + 'Read BUG_REPORT.md in the app directory. Each entry says what was expected', + 'and what actually happened. Fix the app so the behaviour matches, redeploy,', + 'and make sure the dev server is running.', + '', + 'Change only what is needed. Do not alter behaviour that is already correct.', + '', + 'Output FIX_COMPLETE when done.', + '', + ...common, + ].join('\n'); + } + + const verb = args.mode === 'upgrade' + ? [ + `Add the level ${args.level} features below to the existing app.`, + '', + 'Keep completed features working. Add only the current features below.', + ] + : [`Build the application described below and leave it running.`]; + const startingCatalog = args.mode === 'build' && materials.startingCatalog + ? ['', '## Starting catalog', '', 'Use exactly this starting data:', '', + '```json', materials.startingCatalog, '```'] : []; + + return [ + ...verb, + '', + `After the web application is running, reply with ${args.mode === 'upgrade' + ? 'UPGRADE_COMPLETE' : 'DEPLOY_COMPLETE'}.`, + '', + ...common, + '', + agentVisibleContractText(materials.requirementText ?? levelPrompt(track, args.level), + args.credentialAliases, applicationInterface), + ...startingCatalog, + '', + '## Application interface', + '', + agentVisibleContractText(materials.contractText ?? appendix(track, args.level), + args.credentialAliases, applicationInterface), + ].join('\n'); +} + +export function agentScenarioPaths(track: Track, level: number, + recipeBinding: RecipeBinding | null = null): string[] { + const execution = recipeBinding?.execution; + if (execution) return execution.map(entry => resolve(track.dir, entry.source ?? '')); + return suitesFor(track, level).map(suite => suite.spec); +} + +export function agentRecipeRequest(explicitRecipe: string | null = null, + recipeTask: RecipeTaskRequest | null = null): ExactRecipeRequest | null { + const bound = recipeTask?.recipe; + if (!bound) return explicitRecipe; + const identity = `${bound.id}@${bound.version}`; + if (explicitRecipe && explicitRecipe !== identity) { + throw new Error(`agent recipe ${explicitRecipe} does not match bound task ${identity}`); + } + return bound; +} + +// A build must not be able to read the thing that grades it. +// The sandbox probe imports the same deny list used here. + +async function main() { + const args = parseArgs(process.argv); + const track = loadTrack(args.track); + const p = portsFor(track, args.backend, args.runIndex); + const adapter = STACK_ADAPTER_REGISTRY.get(args.backend); + const profileSkills = resolveDefaultGuidanceForStack(args.guidance, args.backend) + ?.skills[args.backend]?.ids; + const adapterSkills = executeStackCapability(adapter, 'agent', 'default-skills'); + if (!profileSkills && (!Array.isArray(adapterSkills) + || adapterSkills.some(skill => typeof skill !== 'string'))) { + throw new Error(`stack adapter ${args.backend} returned invalid default skills`); + } + const defaultSkills = profileSkills ?? adapterSkills as string[]; + const selectedSkills = selectAgentSkills(defaultSkills, + args.skillIdentity?.ids ?? args.skills ?? null); + const skillsText = readAgentSkillDocuments(REPO, selectedSkills); + if (args.skillIdentity && (sha256(skillsText) !== args.skillIdentity.sha256 + || Buffer.byteLength(skillsText) !== args.skillIdentity.bytes)) { + throw new Error('campaign skill material changed after compilation'); + } + const recipeBinding = resolveRecipeRelease(track, args.level, + agentRecipeRequest(args.recipe ?? null, args.recipeTask ?? null)); + if (args.recipeTask && !recipeBinding) { + throw new Error(`L${args.level} has no recipe release for the requested task`); + } + const selectedTask = recipeBinding + ? (args.recipeTask + ? resolveBoundRecipeTaskRequest(recipeBinding, args.recipeTask) + : createBoundRecipeTaskRequest(recipeBinding)) + : null; + const requirementText = selectedTask?.task.requirementText ?? levelPrompt(track, args.level); + const contractText = selectedTask?.task.contractText ?? appendix(track, args.level); + const taskMode = isRecord(args.recipeTask?.task) ? args.recipeTask.task.mode : null; + const startingCatalog = recipeBinding && taskMode === 'fresh' ? JSON.stringify({ + warehouses: recipeBinding.plan.fixture.warehouses, + items: recipeBinding.plan.fixture.items, + }, null, 2) : undefined; + + // Renders what the model would be given, without spending anything or + // touching the app directory. The regression gate for harness changes is a + // diff of this against a saved copy — captured with the same isolation flags, + // since host and container prompts differ by design. + if (args.printPrompt) { + process.stdout.write(buildPrompt(args, p, track, + { skillsText, requirementText, contractText, startingCatalog })); + return; + } + if (args.mode === 'build') { + assertNewOrEmptyDirectory(args.app, 'build application directory'); + } + // Printed prompts describe the one supported coding topology but are a pure + // review operation: they must work on a release-review machine that does not + // have Docker or the build image. A paid session, by contrast, proves the + // image and Linux CLI boundary immediately before touching the app. + resolveIsolation(args); + const imageIdentity = resolveContainerImage(IMAGE); + // Only a build wipes: an upgrade or a fix must find the data the previous + // level left, which is the whole point of a cumulative ladder. + // Called for every backend, not just those with a dbPort: spacetime has no + // database port and would have skipped the pre-build wipe entirely, which is + // exactly the backend whose leftover module causes the migration abort. + ensureDatabase(args.backend, args.runIndex, p.dbPort, track, args.mode === 'build'); + // A build must start from a new or empty directory. Never turn a caller- + // supplied path into permission to recursively erase an existing tree. + mkdirSync(args.app, { recursive: true }); + writeFileSync(resolve(args.app, '..', '.stack-bench-backend'), args.backend); + + const prompt = buildPrompt(args, p, track, + { skillsText, requirementText, contractText, startingCatalog }); + const bugReportPath = join(args.app, 'BUG_REPORT.md'); + const bugReportText = args.mode === 'fix' && existsSync(bugReportPath) + ? readFileSync(bugReportPath, 'utf8') : null; + const provenance = sessionProvenance({ prompt, skillsText, contractText, bugReportText, + scenarioPaths: agentScenarioPaths(track, args.level, recipeBinding), + trackDir: track.dir, trackManifestPath: join(track.dir, 'track.json') }); + const started = Date.now(); + const retryLimitRaw = process.env.STACK_BENCH_CODING_INTERRUPTION_RETRIES + ?? String(DEFAULT_CODING_INTERRUPTION_RETRIES); + const retryLimit = Number(retryLimitRaw); + if (!Number.isInteger(retryLimit) || retryLimit < 0 || retryLimit > 3) { + throw new Error('STACK_BENCH_CODING_INTERRUPTION_RETRIES must be an integer from 0 to 3'); + } + // The benchmark runner stops this process at the adapter deadline. + // A wait budget past that point would promise time the + // supervisor never grants. DEFAULT_THROTTLE_MAX_WAIT_MS is sized to + // span one full subscription usage-window reset. + const throttleWaitRaw = process.env.STACK_BENCH_PROVIDER_THROTTLE_MAX_WAIT_MINUTES + ?? String(DEFAULT_THROTTLE_MAX_WAIT_MS / 60_000); + const throttleMaxWaitMinutes = Number(throttleWaitRaw); + if (!Number.isInteger(throttleMaxWaitMinutes) || throttleMaxWaitMinutes < 0 + || throttleMaxWaitMinutes > DEFAULT_THROTTLE_MAX_WAIT_MS / 60_000) { + throw new Error('STACK_BENCH_PROVIDER_THROTTLE_MAX_WAIT_MINUTES must be an integer from 0 to ' + + `${DEFAULT_THROTTLE_MAX_WAIT_MS / 60_000}`); + } + // Concurrent campaign slots must not wake and retry as one burst. This + // stable offset keeps retries reproducible while spreading them over 45s. + const throttleJitterMs = parseInt(sha256(Buffer.from( + `${args.backend}:${args.runIndex}:${args.level}:${args.mode}`)).slice(0, 8), 16) % 45_001; + let coding: CodingSessionRecoveryResult; + try { + // The prompt goes in on stdin, not as an argument. Windows caps a command + // line at 32767 characters, and the SpacetimeDB prompt carries the skill + // documents on top of the level spec — it crossed that line as soon as L1 + // grew, and the spawn fails with a bare ENOENT that reads as "CLI missing". + const cliEnv = { ...process.env, + // Absent unless deliberately overridden — see THINKING_TOKENS above. + ...((args.thinking ?? THINKING_TOKENS) + ? { MAX_THINKING_TOKENS: String(args.thinking ?? THINKING_TOKENS) } + : {}), + // A CLI that updates itself mid-series changes the thing under test + // between one backend and the next. The sequential harness has frozen + // it since April; this one had not. + DISABLE_AUTOUPDATER: '1', + // Cache reads are ~69% of a run's bill, so cache TTL moves cost more + // than anything else measured here. Unpinned, runs were getting the + // 1-hour tier: a second run of the same backend within the hour reads + // a prefix the first one paid to create and looks cheaper for reasons + // that have nothing to do with the database. That is fatal for n=5 + // parallel trials, which would ALL share one warm prefix, and the + // effect differs per backend because the prompts differ in size. The + // 5-minute tier makes each trial pay its own way. The sequential + // harness has pinned this since April for the same reason. + FORCE_PROMPT_CACHING_5M: '1' }; + + // Same CLI arguments, run somewhere the harness does not exist. The + // container path is a separate script so that what a build can reach is + // stated in one place — see the compiled container/run-build.js entry point. + coding = runCodingSessionWithRecovery({ prompt, retryLimit, maxBudgetUsd: args.maxBudgetUsd, + throttleMaxWaitMs: throttleMaxWaitMinutes * 60_000, + throttleJitterMs, + invoke: ({ input, maxBudgetUsd, resumeSession, recoverStoppedContainer }) => + execFileSync(process.execPath, [ + compiledEntrypoint('container', 'run-build.js'), + '--app', args.app, + '--backend', args.backend, + '--image', imageIdentity.id, + '--effort', EFFORT, + '--model', args.model, + ...(args.pricing ? ['--pricing-json', JSON.stringify(args.pricing)] : []), + '--completion-marker', args.mode === 'fix' ? 'FIX_COMPLETE' + : args.mode === 'upgrade' ? 'UPGRADE_COMPLETE' + : args.mode === 'resume' ? 'RESUME_COMPLETE' : 'DEPLOY_COMPLETE', + ...(maxBudgetUsd != null ? ['--max-budget-usd', String(maxBudgetUsd)] : []), + '--ports', [p.vite, p.express].filter(Boolean).join(','), + ...(resumeSession ? ['--resume-session', resumeSession] : []), + ...(recoverStoppedContainer ? ['--recover-stopped-container'] : []), + ], { input, encoding: 'utf8', maxBuffer: 256 * 1024 * 1024, + env: { ...cliEnv, ...(args.apiKey ? { STACK_BENCH_AGENT_API_KEY: args.apiKey } : {}) }, + timeout: AGENT_PROCESS_TIMEOUT_MS }), + }); + } catch (err: unknown) { + coding = { raw: '', spawnError: codingSessionFailure(isRecord(err) ? err : {}), sessionResults: [], + interruptions: [], result: { total_cost_usd: 0, num_turns: 0, + usage: { input_tokens: 0, output_tokens: 0, cache_creation_input_tokens: 0, + cache_read_input_tokens: 0 }, stack_bench_cost_receipts: [] }, + throttle: { waits: 0, waitedMs: 0, maxWaitMs: throttleMaxWaitMinutes * 60_000, + jitterMs: throttleJitterMs } }; + } + + const { raw, spawnError, sessionResults, interruptions, result, throttle } = coding; + const noOutput = !result.session_id && !raw.trim(); + const failed = Boolean(spawnError || noOutput); + const providerFailure = providerSessionFailure(result); + const usage = sessionUsage(result.usage); + const input = usage.input_tokens ?? 0; + const output = usage.output_tokens ?? 0; + const cacheWrite = usage.cache_creation_input_tokens ?? 0; + const cacheRead = usage.cache_read_input_tokens ?? 0; + const turns = result.num_turns ?? 0; + + // A single total cannot say WHY one backend cost more, and the answer is + // almost never the database. Cost here is roughly (turns × prompt size): + // cache reads dominate the token count and are re-paid every turn, so a + // backend handed a bigger guidance pack pays more for identical work. Keep + // the parts, so a cost difference can be attributed instead of assumed. + const setupMetadata = executeStackCapability(adapter, 'agent', 'setup-metadata', { + imageId: imageIdentity.id, + localPackage: LOCAL_PKG, + env: process.env, + helpers: { linuxSpacetimeVersion, bindingsIdentity, containerImage }, + }); + const out = { + appDir: args.app, + mode: args.mode, + level: args.level, + track: args.track, + backend: args.backend, + model: args.model, + guidance: args.guidance, + // The setup that produced this number. A score whose reasoning budget, + // permission mode or CLI version is unknown cannot be compared with a later + // one — the run would look identical in the record and not be. + setup: { + thinkingTokens: (args.thinking ?? THINKING_TOKENS) ? Number(args.thinking ?? THINKING_TOKENS) : 'cli default', + permissionMode: 'acceptEdits', + effort: EFFORT, + // Which reference documents the model was handed. The cost work varies + // this deliberately, so a number is meaningless without it. + skills: selectedSkills, + // Recorded because it materially changes cost, and because a figure whose + // cache tier is unknown cannot be compared with one taken later. + cacheTier: '5m', + autoUpdater: 'disabled', + codingInterruptionRetries: { limit: retryLimit, + used: interruptions.filter(item => item.kind !== 'provider-throttled').length }, + // Waiting out a provider throttle is time the model did not work; a + // duration compared without this figure blames the backend for the + // account's usage window. + providerThrottle: { maxWaitMinutes: throttleMaxWaitMinutes, + waits: throttle?.waits ?? 0, waitedMs: throttle?.waitedMs ?? 0, + jitterMs: throttle?.jitterMs ?? throttleJitterMs }, + // In a container the host CLI is not the one that ran, so the version is + // read from the image. Reporting the host's would attribute a number to + // software that took no part in producing it. + cliVersion: imageCliVersion(imageIdentity.id), + // Both the human-facing reference and immutable content identity are + // recorded; the latter is what Docker actually executes. + isolation: { mode: 'container', image: imageIdentity.reference, + imageId: imageIdentity.id, hostAlias: HOST_ADDR }, + // Whether the run billed to a key or to the plan. Cost figures from the + // two are not the same measurement. + auth: (args.apiKey ?? process.env.ANTHROPIC_API_KEY) ? 'api-key' + : (process.env.CLAUDE_CODE_OAUTH_TOKEN || process.env.CLAUDE_CODE_OAUTH_TOKEN_FILE) + ? 'subscription-token' : 'not-selected', + // What is actually being benchmarked, not just what drove it. + ...(isRecord(setupMetadata) ? setupMetadata : {}), + // Ambient variables that could have influenced the model, recorded so the + // question "what settings produced this" has an answer. + env: ambientEnv(), + // The orchestrator and coding session are separate runtimes. Recording + // only the host Node version would attribute the model's tool execution + // to a binary that never entered its container. + node: { orchestrator: process.version, codingContainer: imageNodeVersion(imageIdentity.id) }, + platform: process.platform, + }, + costUsd: Number((result.total_cost_usd ?? 0).toFixed(6)), + costReceipts: result.stack_bench_cost_receipts ?? [], + tokens: input + output + cacheWrite + cacheRead, + outputTokens: output, + usage: { input, output, cacheWrite, cacheRead }, + provenance, + turns, + // What the model was handed before it did anything — the denominator for + // every per-turn cost, and the axis the benchmark is least fair on. + promptBytes: Buffer.byteLength(prompt), + tokensPerTurn: turns ? Math.round((input + output + cacheWrite + cacheRead) / turns) : null, + // Reasoning volume, measured rather than assumed — the budget is unpinned, + // so this is how a change in the CLI default becomes visible in the record. + thinking: combinedThinkingVolume(args.app, sessionResults.map(item => item.session_id)), + durationMs: Date.now() - started, + sessionId: result.session_id ?? null, + ok: !failed && result.is_error === false, + providerMetadata: { failureCode: failed + ? String(spawnError ?? '').startsWith('provider stayed throttled') + ? 'provider-throttle-exhausted' + : providerFailure?.code ?? (noOutput ? 'coding-session-no-output' : 'coding-session-failed') + : result.is_error === true ? 'provider-session-error' : null, + failure: failed ? { + providerStatus: result.api_error_status ?? null, + waitedMs: throttle?.waitedMs ?? 0, + waits: throttle?.waits ?? 0, + } : null, + interruptions, invocations: sessionResults.length, + terminalRecovery: isRecord(result) ? result.terminal_recovery ?? null : null, + credentialBroker: result.stack_bench_credential_broker ?? null, + sessionIds: [...new Set(sessionResults.map(item => item.session_id).filter(Boolean))] }, + }; + console.log(JSON.stringify(out)); +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch(err => { console.error(err); process.exit(1); }); +} diff --git a/tools/stack-bench/commands/archive-transcripts.ts b/tools/stack-bench/commands/archive-transcripts.ts new file mode 100644 index 00000000000..67e024e0eb2 --- /dev/null +++ b/tools/stack-bench/commands/archive-transcripts.ts @@ -0,0 +1,126 @@ +#!/usr/bin/env node + +import { copyFileSync, existsSync, mkdirSync, readdirSync, statSync } from 'node:fs'; +import { homedir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { operationalOutputRoot } from '../src/runtime/operational-paths.js'; + +interface ArchiveArgs { + results: string; + out: string; + app?: string; + label?: string; +} + +export interface ArchiveResult { + copied: number; + missing: number; + outputDirectory: string; +} + +function optionValue(argv: string[], index: number): string { + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error(`${argv[index]} requires a value`); + return value; +} + +export function parseArchiveArgs(argv: string[], env: NodeJS.ProcessEnv = process.env): ArchiveArgs { + const operationalRoot = operationalOutputRoot(STACK_BENCH_ROOT, env); + const values: { results?: string; out?: string; app?: string; label?: string } = {}; + const seen = new Set(); + for (let index = 2; index < argv.length; index += 2) { + const flag = argv[index]; + if (!flag || !['--results', '--out', '--app', '--label'].includes(flag)) { + throw new Error(`unknown option ${String(flag)}`); + } + if (seen.has(flag)) throw new Error(`duplicate option ${flag}`); + seen.add(flag); + const value = optionValue(argv, index); + if (flag === '--results') values.results = value; + else if (flag === '--out') values.out = value; + else if (flag === '--app') values.app = value; + else values.label = value; + } + const app = values.app ? resolve(values.app) : undefined; + return { + results: resolve(values.results ?? join(STACK_BENCH_ROOT, 'results')), + out: resolve(values.out ?? join(operationalRoot, 'transcripts')), + ...(app ? { app, label: values.label ?? 'run' } : {}), + }; +} + +function encodedStoreName(directory: string): string { + return resolve(directory).replace(/[\\/:]/g, '-').toLowerCase(); +} + +function transcriptStoreFor(appDirectory: string, storeRoot: string): string | null { + if (!existsSync(storeRoot)) return null; + const expected = encodedStoreName(appDirectory); + const match = readdirSync(storeRoot).find(directory => { + const normalized = directory.toLowerCase(); + return normalized === expected || normalized === expected.replace(/^-+/, ''); + }); + return match ? join(storeRoot, match) : null; +} + +function collectTranscripts(directory: string, output: string[] = []): string[] { + for (const entry of readdirSync(directory, { withFileTypes: true })) { + const path = join(directory, entry.name); + if (entry.isDirectory()) collectTranscripts(path, output); + else if (entry.isFile() && entry.name.endsWith('.jsonl')) output.push(path); + } + return output; +} + +function archiveJobs(args: ArchiveArgs): Array<{ label: string; appDirectory: string }> { + if (args.app) return [{ label: args.label ?? 'run', appDirectory: args.app }]; + if (!existsSync(args.results)) throw new Error(`no results directory at ${args.results}`); + return readdirSync(args.results, { withFileTypes: true }) + .filter(entry => entry.isDirectory()) + .map(entry => ({ label: entry.name, + appDirectory: join(args.results, entry.name, 'app') })) + .filter(job => existsSync(job.appDirectory)); +} + +export function archiveTranscripts(args: ArchiveArgs, + storeRoot = join(homedir(), '.claude', 'projects')): ArchiveResult { + mkdirSync(args.out, { recursive: true }); + let copied = 0; + let missing = 0; + for (const job of archiveJobs(args)) { + const store = transcriptStoreFor(job.appDirectory, storeRoot); + if (!store) { + console.log(` ${job.label}: NO TRANSCRIPT in the CLI store — already pruned, or never run`); + missing += 1; + continue; + } + const destination = join(args.out, job.label); + mkdirSync(destination, { recursive: true }); + for (const source of collectTranscripts(store)) { + const relativeName = source.slice(store.length + 1).replace(/[\\/]/g, '__'); + const target = join(destination, relativeName); + const sourceSize = statSync(source).size; + if (existsSync(target) && statSync(target).size >= sourceSize) continue; + copyFileSync(source, target); + copied += 1; + console.log(` ${job.label}/${relativeName} (${(sourceSize / 1024 / 1024).toFixed(1)} MB)`); + } + } + console.log(`\n${copied} transcript(s) archived to ${args.out}`); + if (missing) console.log(`${missing} run(s) had no transcript left to archive.`); + return { copied, missing, outputDirectory: args.out }; +} + +function main(): void { + try { + archiveTranscripts(parseArchiveArgs(process.argv)); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + } +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/bench-arguments.ts b/tools/stack-bench/commands/bench-arguments.ts new file mode 100644 index 00000000000..e7a15e3ad90 --- /dev/null +++ b/tools/stack-bench/commands/bench-arguments.ts @@ -0,0 +1,331 @@ +import { dirname, resolve } from 'node:path'; +import { readArtifact } from '../src/evidence/artifacts.js'; +import { canonicalDefinitionJson } from '../src/composition/definition-plan.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { DEFAULT_TRACK } from '../src/composition/tracks.js'; +import { validateCompiledCampaignPlan } from '../src/campaigns/campaign-compiler.js'; +import type { CampaignAttemptPlan, CampaignSelection } + from '../src/campaigns/campaign-compiler.js'; +import { campaignAdmissionSmokeReuse, readCampaignAdmission } + from '../src/campaigns/campaign-admission.js'; +import type { CampaignAdmissionSmokeReuse } from '../src/campaigns/campaign-admission.js'; +import { compileProgressionInput, dependencyRuntimeDefinition, progressionLevels, + validateFeatureCatalogInput, validateProgressionInput } + from '../src/progression/progression-definition.js'; +import type { CompiledDependencyPolicyDefinition, CompiledProgressionDefinition, + ProgressionInput } from '../src/progression/progression-definition.js'; +import type { PricingAuthority } from '../src/evidence/pricing-authority.js'; + +type StudyCondition = CampaignAttemptPlan['condition']; +type UnknownRecord = Record; + +export interface BenchArguments { + backend?: string; + track: string; + levels: string; + levelsProvided: boolean; + levelList: number[]; + model: string | null; + agentAdapter: string; + pricing?: PricingAuthority | null; + fixRounds: number; + maxStalledRepairs: number; + maxBudgetUsd?: number; + runIndex: number; + out?: string; + app?: string; + url?: string; + media: boolean; + retainBackend?: boolean; + behavioralReview: boolean; + guidance: string; + guidanceDocument?: unknown; + condition?: StudyCondition; + selectionRequest?: CampaignSelection; + taskMode?: string; + packIds: string[]; + checkKeys: string[]; + featureIds: string[]; + requestedSpecifications: string[]; + expectedSpecifications: string[]; + observedSpecifications: string[]; + skipProbe?: boolean; + skills?: string[]; + apiKey?: string; + apiKeyFile?: string; + mutations?: string; + mutationShardIndex?: number; + mutationShardCount?: number; + mutationResumeFrom?: string; + mutationCheckpointOut?: string; + mutationBaselineBundle?: string; + expectedMutationCalibration?: unknown; + mutationMaxRuntimeMinutes: number; + referenceMutationOnly?: boolean; + seedFrom?: string; + parentAttemptId?: string; + repairFrom?: string; + repairLevel?: number; + recipe?: string; + campaignFile?: string; + featureCatalogSha256?: string; + dependencyPolicySha256?: string; + campaignSha256?: string; + campaignAttemptId?: string; + campaignAdmissionId?: string; + progressionResumeFrom?: string; + experimentIdentity?: { id: string; version: string; sha256: string; state: string }; + campaignAdmission?: { id: string } & CampaignAdmissionSmokeReuse; + runMode?: CampaignAttemptPlan['mode']; + featureCatalog?: ProgressionInput; + dependencyPolicy?: ProgressionInput; + progression?: ProgressionInput; + progressionOwner?: UnknownRecord; +} + +function normalizeGuidance(value: string): 'neutral' | 'prescribed' { + if (value === 'neutral' || value === 'prescribed') return value; + throw new Error(`guidance must be neutral or prescribed, received ${JSON.stringify(value)}`); +} + +export function parseBenchArguments(argv: readonly string[]): BenchArguments { + const args: BenchArguments = { model: null, agentAdapter: 'claude-code', + fixRounds: 10, runIndex: 0, levels: '1', levelsProvided: false, media: true, + levelList: [], maxStalledRepairs: 3, guidance: 'prescribed', track: DEFAULT_TRACK, + packIds: [], checkKeys: [], featureIds: [], requestedSpecifications: [], + expectedSpecifications: [], observedSpecifications: [], behavioralReview: false, + mutationMaxRuntimeMinutes: 60 }; + for (let i = 2; i < argv.length; i++) { + const option = argv[i]; + if (!option) continue; + const value = (): string => { + const next = argv[++i]; + if (next === undefined) throw new Error(`${option} requires a value`); + return next; + }; + switch (option) { + case '--backend': args.backend = value(); break; + case '--track': args.track = value(); break; + case '--levels': args.levels = value(); args.levelsProvided = true; break; + case '--campaign-file': args.campaignFile = resolve(value()); break; + case '--feature-catalog-sha256': args.featureCatalogSha256 = value(); break; + case '--dependency-policy-sha256': args.dependencyPolicySha256 = value(); break; + case '--campaign-sha256': args.campaignSha256 = value(); break; + case '--campaign-attempt-id': args.campaignAttemptId = value(); break; + case '--campaign-admission-id': args.campaignAdmissionId = value(); break; + case '--progression-resume-from': args.progressionResumeFrom = resolve(value()); break; + case '--recipe': args.recipe = value(); break; + case '--pack': args.packIds.push(...value().split(',').filter(Boolean)); break; + case '--check': args.checkKeys.push(...value().split(',').filter(Boolean)); break; + case '--model': args.model = value(); break; + case '--pricing-json': args.pricing = JSON.parse(value()); break; + case '--fix-rounds': args.fixRounds = Number(value()); break; + case '--max-stalled-repairs': args.maxStalledRepairs = Number(value()); break; + case '--max-budget-usd': args.maxBudgetUsd = Number(value()); break; + case '--run-index': args.runIndex = parseInt(value(), 10); break; + case '--out': args.out = value(); break; + case '--app': args.app = value(); break; + case '--url': args.url = value(); break; + case '--agent-adapter': args.agentAdapter = value(); break; + case '--no-media': args.media = false; break; + case '--retain-backend': args.retainBackend = true; break; + case '--behavioral-review': args.behavioralReview = true; break; + case '--guidance': args.guidance = normalizeGuidance(value()); break; + case '--guidance-document-json': args.guidanceDocument = JSON.parse(value()); break; + case '--condition-json': args.condition = JSON.parse(value()); break; + case '--selection-json': args.selectionRequest = JSON.parse(value()); break; + case '--task-mode': args.taskMode = value(); break; + case '--feature-module': args.featureIds.push(...value().split(',').filter(Boolean)); break; + case '--request-spec': args.requestedSpecifications.push(...value().split(',').filter(Boolean)); break; + case '--expect-spec': args.expectedSpecifications.push(...value().split(',').filter(Boolean)); break; + case '--observe-spec': args.observedSpecifications.push(...value().split(',').filter(Boolean)); break; + case '--skip-probe': args.skipProbe = true; break; + case '--skills': args.skills = value().split(',').filter(Boolean); break; + case '--skills-json': args.skills = JSON.parse(value()); break; + case '--api-key': args.apiKey = value(); break; + case '--api-key-file': args.apiKeyFile = resolve(value()); break; + case '--mutations': args.mutations = resolve(value()); break; + case '--mutation-shard-index': args.mutationShardIndex = Number(value()); break; + case '--mutation-shard-count': args.mutationShardCount = Number(value()); break; + case '--mutation-resume-from': args.mutationResumeFrom = resolve(value()); break; + case '--mutation-checkpoint-out': args.mutationCheckpointOut = resolve(value()); break; + case '--mutation-baseline-bundle': args.mutationBaselineBundle = resolve(value()); break; + case '--expected-mutation-calibration-json': + args.expectedMutationCalibration = JSON.parse(value()); break; + case '--mutation-max-runtime-minutes': args.mutationMaxRuntimeMinutes = Number(value()); break; + case '--reference-mutation-only': args.referenceMutationOnly = true; break; + case '--seed-from': args.seedFrom = value(); break; + case '--parent-attempt-id': args.parentAttemptId = value(); break; + case '--repair-from': args.repairFrom = resolve(value()); break; + case '--repair-level': args.repairLevel = Number(value()); break; + default: console.error(`Unknown argument: ${option}`); process.exit(2); + } + } + if (!args.backend && !args.repairFrom) { + console.error('Usage: npm run bench -- --backend --levels 1-3 [--fix-rounds 10] [--run-index N]'); + process.exit(2); + } + if ((args.mutationResumeFrom || args.mutationCheckpointOut || args.mutationBaselineBundle) + && !args.mutations) { + throw new Error('mutation control options require --mutations'); + } + if (args.expectedMutationCalibration && !args.mutations) { + throw new Error('--expected-mutation-calibration-json requires --mutations'); + } + if (!Number.isFinite(args.mutationMaxRuntimeMinutes) || args.mutationMaxRuntimeMinutes < 1 + || args.mutationMaxRuntimeMinutes > 120) { + throw new Error('--mutation-max-runtime-minutes must be from 1 through 120'); + } + if (args.referenceMutationOnly && (!args.mutations || args.agentAdapter !== 'reference-fixture' + || args.fixRounds !== 0 || !args.app || args.campaignFile)) { + throw new Error('--reference-mutation-only requires a mutation-bound reference fixture run'); + } + if (args.mutationBaselineBundle && !args.referenceMutationOnly) { + throw new Error('--mutation-baseline-bundle is an internal reference mutation option'); + } + if (args.repairFrom && (args.repairLevel === undefined + || !Number.isSafeInteger(args.repairLevel) || args.repairLevel < 1)) { + throw new Error('--repair-from requires --repair-level with a positive integer'); + } + if (args.campaignFile && (!args.campaignSha256 || !args.campaignAttemptId)) { + throw new Error('--campaign-file requires --campaign-sha256 and --campaign-attempt-id'); + } + if (!args.campaignFile && (args.campaignSha256 || args.campaignAttemptId + || args.campaignAdmissionId || args.featureCatalogSha256 || args.dependencyPolicySha256)) { + throw new Error('campaign binding requires --campaign-file'); + } + if (args.progressionResumeFrom && !args.campaignFile) { + throw new Error('--progression-resume-from requires a compiled campaign'); + } + if (args.campaignFile) bindCampaign(args); + if (args.progression) { + if (args.levelsProvided) throw new Error('--levels cannot be combined with progression input'); + args.progression = validateProgressionInput(args.progression); + args.levelList = progressionLevels(args.progression); + args.levels = `${args.levelList[0]}-${args.levelList.at(-1)}`; + } else { + const [fromText, toText] = args.levels.split('-'); + const from = Number(fromText); + const to = toText === undefined ? from : Number(toText); + if (!Number.isSafeInteger(from) || from < 1 || !Number.isSafeInteger(to) || to < from) { + throw new Error('--levels must be one positive level or an ascending range'); + } + args.levelList = Array.from({ length: (to ?? from) - from + 1 }, (_, index) => from + index); + } + if (args.recipe && args.levelList.length !== 1) { + throw new Error('--recipe requires exactly one requested level'); + } + if (!Number.isInteger(args.fixRounds) || args.fixRounds < 0 || args.fixRounds > 20) { + throw new Error('--fix-rounds must be an integer from 0 through 20'); + } + if (!Number.isInteger(args.maxStalledRepairs) || args.maxStalledRepairs < 0 + || args.maxStalledRepairs > 20) { + throw new Error('--max-stalled-repairs must be an integer from 0 through 20'); + } + if (args.maxBudgetUsd !== undefined + && (!Number.isFinite(args.maxBudgetUsd) || args.maxBudgetUsd <= 0)) { + throw new Error('--max-budget-usd must be a positive number'); + } + if ((args.mutationShardIndex === undefined) !== (args.mutationShardCount === undefined)) { + throw new Error('--mutation-shard-index and --mutation-shard-count must be supplied together'); + } + return args; +} + +function bindCampaign(args: BenchArguments): void { + if (!args.campaignFile) throw new Error('campaign file is required'); + const unsupported = [ + [args.maxStalledRepairs !== 3, '--max-stalled-repairs'], + [args.skipProbe === true, '--skip-probe'], + [args.behavioralReview === true, '--behavioral-review'], + [args.mutations !== undefined || args.mutationShardIndex !== undefined + || args.mutationShardCount !== undefined, '--mutations'], + [args.seedFrom !== undefined, '--seed-from'], + [args.repairFrom !== undefined || args.repairLevel !== undefined, '--repair-from'], + [args.app !== undefined, '--app'], [args.url !== undefined, '--url'], + [args.retainBackend === true, '--retain-backend'], + [args.apiKey !== undefined || args.apiKeyFile !== undefined, 'credential override'], + [args.recipe !== undefined || args.packIds.length > 0 || args.checkKeys.length > 0 + || args.featureIds.length > 0 || args.requestedSpecifications.length > 0 + || args.expectedSpecifications.length > 0 || args.observedSpecifications.length > 0, + 'direct recipe selection'], + ].filter(([changed]) => changed).map(([, name]) => name); + if (unsupported.length) { + throw new Error(`campaign progression input cannot override ${unsupported.join(', ')}`); + } + const artifact = readArtifact(args.campaignFile, { expectedKind: 'campaign_plan' }); + const plan = validateCompiledCampaignPlan(artifact.payload); + if (plan.contentSha256 !== args.campaignSha256) { + throw new Error('--campaign-sha256 does not match the compiled campaign plan'); + } + const attempt = plan.attempts.find(item => item.id === args.campaignAttemptId); + if (attempt) { + args.condition ??= structuredClone(attempt.condition); + args.skills ??= structuredClone(attempt.skills); + args.selectionRequest ??= structuredClone(plan.definition.selection); + args.guidanceDocument ??= structuredClone( + attempt.condition?.guidance?.documents?.[attempt.stack]); + } + if (!attempt || attempt.stack !== args.backend || attempt.agentAdapter !== args.agentAdapter + || attempt.model !== args.model + || canonicalDefinitionJson(attempt.pricing) !== canonicalDefinitionJson(args.pricing) + || plan.definition.track !== args.track || attempt.guidance !== args.guidance + || canonicalDefinitionJson(attempt.condition) !== canonicalDefinitionJson(args.condition) + || canonicalDefinitionJson(attempt.skills) !== canonicalDefinitionJson(args.skills) + || canonicalDefinitionJson(plan.definition.selection) + !== canonicalDefinitionJson(args.selectionRequest) + || canonicalDefinitionJson(attempt.condition?.guidance?.documents?.[attempt.stack]) + !== canonicalDefinitionJson(args.guidanceDocument) + || plan.definition.budgets.fixRounds !== args.fixRounds + || plan.definition.budgets.maxCostUsdPerAttempt !== (args.maxBudgetUsd ?? null) + || args.parentAttemptId !== attempt.id || args.media !== false) { + throw new Error('--campaign-attempt-id does not match the requested campaign attempt'); + } + args.experimentIdentity = { + id: plan.id, version: plan.version, sha256: plan.contentSha256, state: plan.state, + }; + if (args.campaignAdmissionId) { + const admission = readCampaignAdmission(dirname(args.campaignFile), + args.campaignAdmissionId, plan); + const image = plan.definition.runtime.buildImage + ?? process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE; + args.campaignAdmission = { + id: args.campaignAdmissionId, + ...campaignAdmissionSmokeReuse(admission, { + agentAdapter: args.agentAdapter, + runIndex: args.runIndex, + backend: args.backend, + image, + }), + }; + } + args.runMode = structuredClone(attempt.mode); + if (plan.featureCatalog) { + args.featureCatalog = validateFeatureCatalogInput(plan.featureCatalog); + if (args.featureCatalog.identity.sha256 !== args.featureCatalogSha256 + || canonicalDefinitionJson(attempt.featureCatalog) + !== canonicalDefinitionJson(args.featureCatalog.identity)) { + throw new Error('--feature-catalog-sha256 does not match the compiled campaign plan'); + } + } else if (args.featureCatalogSha256 !== undefined || attempt.featureCatalog !== undefined) { + throw new Error('campaign attempt has an unexpected feature catalog'); + } + if (attempt.mode.id === 'dependency') { + if (!plan.dependencyPolicy || !args.featureCatalog) { + throw new Error('dependency campaign requires a feature catalog and dependency policy'); + } + if (plan.dependencyPolicy.identity.sha256 !== args.dependencyPolicySha256 + || canonicalDefinitionJson(attempt.dependencyPolicy) + !== canonicalDefinitionJson(plan.dependencyPolicy.identity)) { + throw new Error('--dependency-policy-sha256 does not match the compiled campaign plan'); + } + args.dependencyPolicy = plan.dependencyPolicy; + args.progression = compileProgressionInput(dependencyRuntimeDefinition( + args.featureCatalog, args.dependencyPolicy)); + args.progressionOwner = { schemaVersion: 1, + campaign: { id: plan.id, version: plan.version, sha256: plan.contentSha256 }, + attempt: { id: attempt.id, track: plan.definition.track, stack: attempt.stack, + agentAdapter: attempt.agentAdapter, model: attempt.model, + conditionSha256: attempt.condition.sha256 } }; + } +} diff --git a/tools/stack-bench/commands/bench.ts b/tools/stack-bench/commands/bench.ts new file mode 100644 index 00000000000..ccfd4b7c8cd --- /dev/null +++ b/tools/stack-bench/commands/bench.ts @@ -0,0 +1,2375 @@ +#!/usr/bin/env node +// Stack Bench: run the whole benchmark for one backend, unattended. +// +// For each level: build (or upgrade), grade, and if anything failed hand the +// agent a behavioural bug report and let it fix — up to --fix-rounds times — +// re-grading after each attempt. Records score, cost, time and fix rounds per +// level, then writes a summary. +// +// Usage: +// node dist/commands/bench.js --backend spacetime --levels 1-5 [--model claude-sonnet-5] +// [--fix-rounds 10] [--run-index 0] [--out ] +// [--retain-backend] [--no-media] +// +// The benchmark runs its own SpacetimeDB host (STACK_BENCH_STDB_URI, default +// 127.0.0.1:3210, data in .spacetime-data) rather than a machine-wide one, so +// resource measurements describe the module under test and a durability restart +// cannot disturb anything else. It is started if absent and stopped at the end +// unless --retain-backend. + +import { execFile, execFileSync } from 'node:child_process'; +import type { ChildProcess, ExecFileSyncOptionsWithStringEncoding } from 'node:child_process'; +import { readFileSync, writeFileSync, mkdirSync, existsSync, cpSync, rmSync } from 'node:fs'; +import { join, dirname, resolve } from 'node:path'; +import { tmpdir } from 'node:os'; +import { pathToFileURL } from 'node:url'; +import { loadTrack, resultsName, portsFor, workDirFor, assertNoPortCollisions, + moduleName, dbName, suitesFor } from '../src/composition/tracks.js'; +import { parseBenchArguments } from './bench-arguments.js'; +import type { BenchArguments } from './bench-arguments.js'; +import { killTree } from '../src/runtime/platform.js'; +import { formatRepairProgress } from '../src/evidence/scoring.js'; +import { emptyArtifactIdentities, readArtifact, readArtifactPayload, writeArtifact, writeRunJson } from '../src/evidence/artifacts.js'; +import { aggregateRunOutcome, classifyBundle, ladderMayAdvance, ladderMayContinue, + mutationControlEligible, runExitCode } from '../src/evidence/outcomes.js'; +import { summarizeSessions } from '../src/evidence/session-metrics.js'; +import { hashDirectory, sha256 } from '../src/evidence/provenance.js'; +import { createBackendLease, newRunId, publicBackendLease, readBackendLease, + acquireResourceLocks, backendResourceLockKeys, releaseResourceLocks, resourceLockScope, + writeBackendLease } from '../src/runtime/backend-lease.js'; +import { captureBackendDiagnostics, controlApplication } from '../src/runtime/backend-control.js'; +import { releaseBackendLease } from '../src/runtime/backend-teardown.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { createAgentVisibleTaskRequest, createBoundRecipeTaskRequest } + from '../src/composition/recipe-selection.js'; +import { criterionEvidence, evidencePassed } from '../src/evidence/check-evidence.js'; +import { executeStackCapability } from '../src/stacks/stack-adapter-contract.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { agentRecipeIdentity, agentRequestArgv, agentSessionFailure, + validateAgentResult } from '../src/agents/agent-adapter-contract.js'; +import { AGENT_ADAPTER_REGISTRY, agentAdapterIdentity } from '../src/agents/agent-adapters.js'; +import { runPreflight } from '../src/runtime/preflight.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { SUPERVISOR_STATE_VERSION, writeRecoveryArtifact } from '../src/runtime/recovery.js'; +import { resolveAgentCredential } from '../src/agents/agent-credentials.js'; +import { sandboxProbeMode } from '../src/runtime/sandbox.js'; +import { hashAppSource, resetAppToSource, seedAppSource, snapshotAppSource } from '../src/runtime/source-snapshot.js'; +import { preserveLevelCheckpoint } from '../src/runtime/source-checkpoint.js'; +import { compareRepairBaseline, createRepairGrant } from '../src/runtime/repair-grant.js'; +import { canonicalDefinitionJson } from '../src/composition/definition-plan.js'; +import { contractControlIds } from '../src/composition/agent-visible-contract.js'; +import { repairEvidenceDecision } from '../src/evidence/repair-evidence.js'; +import { mutationControlArgv, mutationControlTimeoutMs } from '../src/evidence/mutation-control.js'; +import type { MutationControlArgs } from '../src/evidence/mutation-control.js'; +import { progressionEngine } from '../src/progression/progression-engine.js'; +import { DEPENDENCY_MODE_VERSION } from '../src/progression/dependency-definition.js'; +import { resolveProgressionRecipeAction, resolveProgressionRecipeLevelSelection } + from '../src/progression/progression-recipe-selection.js'; +import { createLiveProgressionExecution } + from '../src/progression/live-progression.js'; +import type { CampaignSelection } from '../src/campaigns/campaign-compiler.js'; +import { gradingRunTimeoutMs, selectedGradingSourceCount } + from '../src/runtime/grading-timeout.js'; +import { claudeRatesForModel } from '../src/evidence/claude-usage-cost.js'; +import { PRICING_UNIT, validatePricingAuthority } + from '../src/evidence/pricing-authority.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import type { BoundRecipeTaskRequestResult } from '../src/composition/recipe-selection.js'; +import type { RecipeBinding } from '../src/composition/recipe-release.js'; +import type { ProgressionInput } from '../src/progression/progression-definition.js'; +import type { RepairGrantResolution, RepairOutcome } from '../src/runtime/repair-grant.js'; +import type { AgentAdapter, AgentMode, AgentRequest, ValidatedAgentResult } + from '../src/agents/agent-adapter-contract.js'; +import type { Track } from '../src/composition/tracks.js'; +import type { RunOutcome } from '../src/evidence/outcomes.js'; +import type { GradeBundlePayload, BenchmarkRunRecord, RunLevelRecord, + RunContinuation, RunTotals } from '../src/evidence/benchmark-run.js'; +import { addCostUsd, finalizeRunTotals, formatLevelSummary, runSessionRecord } + from '../src/evidence/benchmark-run.js'; +import type { ProgressionAction } from '../src/progression/progression-engine.js'; +import type { ProgressionAttempt, ProgressionState } from '../src/progression/progression-state.js'; +import type { ProgressionRecipeAction, ProgressionRecipeSelections } + from '../src/progression/progression-recipe-selection.js'; +import type { BackendLease, BackendLeaseContainer } from '../src/runtime/backend-lease.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const COMMAND_TIMEOUT_MS = 20 * 60_000; + +type UnknownRecord = Record; +type RepairProgress = { score: number | null; fingerprint: string; stalledRounds: number }; +type ContaminationAudit = { kind: 'contaminated' | 'harness_failure'; evidence: string[]; + verdict: string }; +type LeakAuditEntry = { hits: Array<{ kind: string; path: string }> }; +type CommandFailure = Error & { stdout?: string | Buffer; stderr?: string | Buffer; + status?: number | null; signal?: NodeJS.Signals | null }; +type GradeOptions = { observation?: 'scored' | 'observed'; out?: string | null; + sourceSha256?: string | null; applicationFailure?: RunOutcome | null }; +type RestartSpec = { backend: string; app: string; port: number | null; probe: string }; +type MutationControlResult = UnknownRecord & { ok: boolean; artifact?: string; + skipped?: boolean; processError?: string | null; outcome: RunOutcome | null }; +type RecipeTask = (BoundRecipeTaskRequestResult | ProgressionRecipeSelections['grader']) & { agentRequest?: UnknownRecord; + progressionAction?: ProgressionAction }; +type BenchArgs = BenchArguments & { + recipeTasks: Map; + recipeBindings: Map; + repairGrant?: RepairGrantResolution; + mutationImageId?: string; + spentBudgetUsd?: number; +}; +type StudyCondition = NonNullable; +type StackRuntimeConfig = { + environment: Record; + lease: { serverUri: string | null }; + lifecycle: UnknownRecord; + windowsEnvironmentBridge: string[]; +}; +type PreparedLease = { + lease: { + serverUri: string | null; + database: string | null; + module: string | null; + dataDir: string | null; + container: Pick | null; + }; + lockKeys: string[]; +}; +type ProgressionWorkRecipeAction = ProgressionRecipeSelections & { + action: Exclude; +}; +type FirstBuildRecord = { + score: number | null; + max: number | null; + regression: NonNullable['regression'] | null; + contractPass: boolean | null; + outcome: RunOutcome; + source: { sha256: string; files: number } | null; + missed: string[]; + observations?: UnknownRecord; +}; +type RepairStatus = 'not-needed' | 'corrected' | 'budget-exhausted' | 'incomplete' | 'ungraded'; +type ProgressionFailure = { kind?: string; reason?: string }; + +const object = (value: unknown): value is UnknownRecord => + value !== null && typeof value === 'object' && !Array.isArray(value); + +const errorMessage = (error: unknown): string => + error instanceof Error ? error.message : String(error); + +function commandFailure(error: unknown): CommandFailure { + if (error instanceof Error) return error; + throw error; +} + +function parseLeakAudit(value: string): LeakAuditEntry[] { + const parsed: unknown = JSON.parse(value); + if (!Array.isArray(parsed)) throw new Error('contamination audit output must be an array'); + return parsed.map((entry, index) => { + if (!object(entry) || !Array.isArray(entry.hits)) { + throw new Error(`contamination audit output[${index}] is invalid`); + } + const hits = entry.hits.map((hit, hitIndex) => { + if (!object(hit) || typeof hit.kind !== 'string' || typeof hit.path !== 'string') { + throw new Error(`contamination audit output[${index}].hits[${hitIndex}] is invalid`); + } + return { kind: hit.kind, path: hit.path }; + }); + return { hits }; + }); +} + +function nullableString(value: unknown, at: string): string | null { + if (value === null) return null; + if (typeof value !== 'string') throw new Error(`${at} must be a string or null`); + return value; +} + +function stringRecord(value: unknown, at: string): Record { + if (!object(value)) throw new Error(`${at} must be an object`); + const result: Record = {}; + for (const [key, entry] of Object.entries(value)) { + if (typeof entry !== 'string') throw new Error(`${at}.${key} must be a string`); + result[key] = entry; + } + return result; +} + +function stringArray(value: unknown, at: string): string[] { + if (!Array.isArray(value) || value.some(entry => typeof entry !== 'string')) { + throw new Error(`${at} must be an array of strings`); + } + return [...value]; +} + +function booleanResult(value: unknown, at: string): boolean { + if (typeof value !== 'boolean') throw new Error(`${at} must return a boolean`); + return value; +} + +function campaignSelection(value: unknown, at: string): CampaignSelection { + if (!object(value)) throw new Error(`${at} must be an object`); + const optionalStrings = (field: 'packs' | 'checks'): string[] | undefined => { + const entry = value[field]; + if (entry === undefined) return undefined; + return stringArray(entry, `${at}.${field}`); + }; + let levels: CampaignSelection['levels']; + if (value.levels !== undefined) { + if (!Array.isArray(value.levels)) throw new Error(`${at}.levels must be an array`); + levels = value.levels.map((entry, index) => { + if (!object(entry)) throw new Error(`${at}.levels[${index}] is invalid`); + const level = entry.level; + const recipe = entry.recipe; + if (typeof level !== 'number' || !Number.isSafeInteger(level) || typeof recipe !== 'string') { + throw new Error(`${at}.levels[${index}] is invalid`); + } + return { level, recipe, + ...(entry.features === undefined ? {} : { features: stringArray(entry.features, + `${at}.levels[${index}].features`) }), + ...(entry.checks === undefined ? {} : { checks: stringArray(entry.checks, + `${at}.levels[${index}].checks`) }) }; + }); + } + return { ...(optionalStrings('packs') === undefined ? {} : { packs: optionalStrings('packs') }), + ...(optionalStrings('checks') === undefined ? {} : { checks: optionalStrings('checks') }), + ...(levels === undefined ? {} : { levels }) }; +} + +function stackRuntimeConfig(value: unknown): StackRuntimeConfig { + if (!object(value)) throw new Error('stack orchestrator config must be an object'); + if (!object(value.lease)) throw new Error('stack orchestrator config.lease must be an object'); + if (!object(value.lifecycle)) throw new Error('stack orchestrator config.lifecycle must be an object'); + if (!Array.isArray(value.windowsEnvironmentBridge) + || value.windowsEnvironmentBridge.some(entry => typeof entry !== 'string')) { + throw new Error('stack orchestrator config.windowsEnvironmentBridge must be strings'); + } + return { environment: stringRecord(value.environment, 'stack orchestrator config.environment'), + lease: { serverUri: nullableString(value.lease.serverUri, + 'stack orchestrator config.lease.serverUri') }, + lifecycle: value.lifecycle, + windowsEnvironmentBridge: [...value.windowsEnvironmentBridge] }; +} + +function preparedLeaseResult(value: unknown): PreparedLease { + if (!object(value) || !object(value.lease) || !Array.isArray(value.lockKeys) + || value.lockKeys.some(key => typeof key !== 'string')) { + throw new Error('stack lease preparation result is invalid'); + } + const lease = value.lease; + const container = lease.container; + if (container === null) { + return { lease: { serverUri: nullableString(lease.serverUri, 'stack lease serverUri'), + database: nullableString(lease.database, 'stack lease database'), + module: nullableString(lease.module, 'stack lease module'), + dataDir: nullableString(lease.dataDir, 'stack lease dataDir'), container: null }, + lockKeys: [...value.lockKeys] }; + } + if (!object(container) || typeof container.name !== 'string' || typeof container.id !== 'string') { + throw new Error('stack lease preparation result.container is invalid'); + } + return { lease: { serverUri: nullableString(lease.serverUri, 'stack lease serverUri'), + database: nullableString(lease.database, 'stack lease database'), + module: nullableString(lease.module, 'stack lease module'), + dataDir: nullableString(lease.dataDir, 'stack lease dataDir'), + container: { name: container.name, id: container.id } }, + lockKeys: [...value.lockKeys] }; +} + +function isProgressionWorkRecipeAction(value: ProgressionRecipeAction): + value is ProgressionWorkRecipeAction { + return value.action.type !== 'terminal'; +} + +function repairReportArgs(value: ProgressionRecipeAction | null): string[] { + if (!value || !isProgressionWorkRecipeAction(value) || value.action.type !== 'repair') return []; + if (!object(value.action.prompt) || !Array.isArray(value.action.prompt.nodeIds) + || !object(value.action.grading) || !Array.isArray(value.action.grading.checks)) { + throw new Error('dependency repair action has invalid prompt or grading selections'); + } + const promptNodeIds = new Set(value.action.prompt.nodeIds.map(nodeId => { + if (typeof nodeId !== 'string' || !nodeId) { + throw new Error('dependency repair action has an invalid prompt node'); + } + return nodeId; + })); + const checks = value.action.grading.checks.flatMap(check => { + if (!object(check) || typeof check.id !== 'string' || !check.id + || typeof check.nodeId !== 'string' || !check.nodeId) { + throw new Error('dependency repair action has an invalid grading check'); + } + return promptNodeIds.has(check.nodeId) ? [check.id] : []; + }); + if (checks.length === 0) throw new Error('dependency repair action selects no repair checks'); + const controls = contractControlIds(value.agent.task.contractText); + return ['--checks-json', JSON.stringify(checks), + '--controls-json', JSON.stringify(controls)]; +} + +function requireProgressionState(state: ProgressionState | null): ProgressionState { + if (!state) throw new Error('live dependency progression has no active state'); + return state; +} + +function requireContinuation(run: BenchmarkRunRecord): RunContinuation { + if (!run.continuation) throw new Error('repair continuation has no continuation record'); + return run.continuation; +} + +function requireRunTotals(run: BenchmarkRunRecord): RunTotals { + if (!run.totals) throw new Error('benchmark run totals are not available'); + return run.totals; +} + +function repairOutcome(outcome: RunOutcome): RepairOutcome { + return { kind: outcome.kind, appFailures: [...(outcome.appFailures ?? [])], + inconclusive: [...(outcome.inconclusive ?? [])], + harnessFailures: [...(outcome.harnessFailures ?? [])] }; +} + +function progressionFailure(outcome: RunOutcome): ProgressionFailure { + return { kind: outcome.kind, ...(outcome.reason === null || outcome.reason === undefined + ? {} : { reason: outcome.reason }) }; +} + +function mutationControlArgs(args: BenchArgs): MutationControlArgs { + if (!args.out || !args.mutations || !args.backend || !args.parentAttemptId) { + throw new Error('mutation control has incomplete run identity'); + } + return { levelList: args.levelList, out: args.out, recipe: args.recipe, + recipeTasks: args.recipeTasks, mutations: args.mutations, backend: args.backend, + track: args.track, runIndex: args.runIndex, parentAttemptId: args.parentAttemptId, + mutationShardIndex: args.mutationShardIndex, mutationShardCount: args.mutationShardCount, + mutationResumeFrom: args.mutationResumeFrom, mutationCheckpointOut: args.mutationCheckpointOut, + mutationBaselineBundle: args.mutationBaselineBundle, + expectedMutationCalibration: args.expectedMutationCalibration, + mutationMaxRuntimeMinutes: args.mutationMaxRuntimeMinutes, + mutationImageId: args.mutationImageId }; +} + +function mutationOutcome(value: unknown): RunOutcome | null { + if (value === null || value === undefined) return null; + if (!object(value) || typeof value.kind !== 'string') { + throw new Error('mutation control artifact outcome is invalid'); + } + return { kind: value.kind, ...(typeof value.phase === 'string' ? { phase: value.phase } : {}), + ...(typeof value.reason === 'string' ? { reason: value.reason } : {}) }; +} + +function recipeRequestIdentity(value: unknown): { recipeSha256: string; selectionSha256: string; + taskPacks: unknown; taskSha256: string } { + if (!object(value) || !object(value.recipe) || !object(value.selection) || !object(value.task) + || typeof value.recipe.contentSha256 !== 'string' || typeof value.selection.sha256 !== 'string' + || typeof value.task.sha256 !== 'string') { + throw new Error('recipe task request has no complete identity'); + } + return { recipeSha256: value.recipe.contentSha256, selectionSha256: value.selection.sha256, + taskPacks: value.selection.taskPacks, taskSha256: value.task.sha256 }; +} + +// Grading writes private evidence into the mounted app directory because the +// grader and app share one runtime. A repair session may receive only the +// behavioural BUG_REPORT.md produced from that evidence. Remove the raw +// bundle, scenario names, screenshots, and grader output before the coding +// model starts. +export function clearPrivateGradingEvidence(appDir: string): void { + rmSync(join(resolve(appDir), 'stack-bench'), { recursive: true, force: true }); +} + +export function repairProgressState( + previous: RepairProgress | null, + bundle: GradeBundlePayload | null, +): RepairProgress { + const outcome = classifyBundle(bundle); + const score = bundle?.totals?.score ?? null; + const fingerprint = canonicalDefinitionJson({ + kind: outcome.kind, + phase: outcome.phase ?? null, + appFailures: [...(outcome.appFailures ?? [])].sort(), + inconclusive: [...(outcome.inconclusive ?? [])].sort(), + harnessFailures: [...(outcome.harnessFailures ?? [])].sort(), + contractFailures: (bundle?.suites?.lint?.results ?? []) + .filter(result => result.status === 'FAIL') + .map(result => ({ id: result.id, detail: result.detail ?? null })), + }); + const stalledRounds = previous && score !== null && previous.score !== null + && score <= previous.score && fingerprint === previous.fingerprint + ? previous.stalledRounds + 1 : 0; + return { score, fingerprint, stalledRounds }; +} + +export function repairHistoryEntry( + round: number, + before: GradeBundlePayload | null, + after: GradeBundlePayload | null, + result: string, +) { + const failureKeys = (bundle: GradeBundlePayload | null): string[] => { + const outcome = classifyBundle(bundle); + const contract = (bundle?.suites?.lint?.results ?? []) + .filter(item => item.status === 'FAIL').map(item => `testing-interface/${item.id}`); + return [...new Set([...(outcome.appFailures ?? []).filter(key => key !== 'contract-lint'), + ...contract])].sort(); + }; + return { + round, + beforeScore: before?.totals?.score ?? null, + beforeMax: before?.totals?.max ?? null, + afterScore: after?.totals?.score ?? null, + afterMax: after?.totals?.max ?? null, + result, + remainingFailures: failureKeys(after), + }; +} + +export function levelGradeIsUsable( + bundleOutcome: RunOutcome, + progressionAttempt: Pick | null = null, +): boolean { + if (progressionAttempt) return progressionAttempt.outcome === 'conclusive'; + return !['provider_failure', 'ungraded', 'harness_failure'].includes(bundleOutcome.kind); +} + +export function dependencyRepairBudget( + action: unknown, + completedRepairRounds: number, + initialGradePending = false, +): number { + if (!object(action) || action.type === 'terminal' || !object(action.strikes) + || action.strikes.scope !== 'feature' + || !Number.isSafeInteger(action.strikes.maxRemaining) + || Number(action.strikes.maxRemaining) < 0 + || !Number.isSafeInteger(completedRepairRounds) || completedRepairRounds < 0) { + throw new Error('dependency repair budget requires one valid feature-strike action'); + } + return Math.max(0, completedRepairRounds + Number(action.strikes.maxRemaining) + - (initialGradePending ? 1 : 0)); +} + +interface DependencyStrikeState { + definition: { nodes: Array<{ id: string; level: number }> }; + nodes: Record; +} + +export function dependencyStrikeRecords( + state: DependencyStrikeState, + level: number, + includedNodeIds: ReadonlySet | readonly string[] = [], +) { + const included = new Set(includedNodeIds); + return state.definition.nodes + .filter(node => node.level === level + || state.nodes[node.id]?.exhaustedAtLevel === level + || included.has(node.id)) + .map(node => { + const nodeState = state.nodes[node.id]; + if (!nodeState) throw new Error(`progression state is missing node ${node.id}`); + return { nodeId: node.id, + initialBudget: nodeState.strikes.initialBudget, + granted: nodeState.strikes.granted, + budget: nodeState.strikes.budget, + used: nodeState.strikes.used, + remaining: nodeState.strikes.budget - nodeState.strikes.used, + exhaustionReason: nodeState.exhaustionReason }; + }) + .sort((left, right) => left.nodeId.localeCompare(right.nodeId)); +} + +function snapshotSource(appDir: string, to: string): void { + snapshotAppSource(appDir, to); +} + +export function preserveFinalPackageEvidence( + { appDir, outputDir }: { appDir: string; outputDir: string }, +): { + source: { directory: string; sha256: string; files: number }; + grading: { directory: string; artifact: string; sourceSha256: string }; +} { + const failures: string[] = []; + let source: { directory: string; sha256: string; files: number } | null = null; + let grading: { directory: string; artifact: string; sourceSha256: string } | null = null; + + try { + const live = hashAppSource(appDir); + const destination = join(outputDir, 'source'); + snapshotSource(appDir, destination); + const saved = hashDirectory(destination); + if (saved.sha256 !== live.sha256 || saved.files.length !== live.files.length) { + throw new Error('preserved final source differs from the live application source'); + } + source = { directory: 'source', sha256: saved.sha256, files: saved.files.length }; + } catch (error) { + failures.push(`source: ${errorMessage(error).split(/\r?\n/)[0]}`); + } + + try { + const from = join(appDir, 'stack-bench'); + const destination = join(outputDir, 'grading'); + if (!existsSync(join(from, 'bundle.json'))) { + throw new Error('final grader produced no bundle.json'); + } + rmSync(destination, { recursive: true, force: true }); + cpSync(from, destination, { + recursive: true, + filter: path => !/[\\/]media([\\/]|$)/.test(path), + }); + const bundle = readArtifactPayload(join(destination, 'bundle.json'), { + expectedKind: 'grade_bundle', + }); + if (!source || bundle.source?.sha256 !== source.sha256) { + throw new Error('final grading bundle does not match the preserved application source'); + } + grading = { directory: 'grading', artifact: 'grading/bundle.json', + sourceSha256: bundle.source.sha256 }; + } catch (error) { + failures.push(`grading: ${errorMessage(error).split(/\r?\n/)[0]}`); + } + + if (failures.length) { + throw new Error(`could not preserve mandatory result package evidence: ${failures.join('; ')}`); + } + if (!source || !grading) { + throw new Error('could not preserve mandatory result package evidence'); + } + return { source, grading }; +} + +export function sourceBoundFirstBuildOutcome( + bundle: GradeBundlePayload | null, + source: object | null, +): RunOutcome { + if (source) return classifyBundle(bundle); + const reason = 'the first-build source could not be preserved and verified'; + return { kind: 'harness_failure', phase: 'first-build-source', reason, + appFailures: [], inconclusive: [], harnessFailures: [reason] }; +} + +export async function materializeAcceptedSource( + sourcePath: string, + appDir: string, + application: RestartSpec, + lifecycle: typeof controlApplication = controlApplication, +): Promise { + const accepted = hashDirectory(sourcePath); + await lifecycle(application, 'stop'); + resetAppToSource(sourcePath, appDir); + if (!existsSync(join(appDir, 'start.sh'))) { + throw Object.assign(new Error('accepted application source has no /app/start.sh'), + { code: 'generated_app_start_contract_missing' }); + } + let startFailure: unknown = null; + try { + await lifecycle(application, 'start'); + } catch (error) { + startFailure = error; + } + const restoreAcceptedSource = async (): Promise => { + let cleanupFailure: unknown = null; + try { + await lifecycle(application, 'stop'); + } catch (error) { + cleanupFailure = error; + } + try { + resetAppToSource(sourcePath, appDir); + } catch (error) { + cleanupFailure ??= error; + } + if (cleanupFailure) { + throw new Error('could not stop and restore an application after startup', + { cause: cleanupFailure }); + } + }; + const materialized = hashAppSource(appDir); + if (materialized.sha256 !== accepted.sha256 + || materialized.files.length !== accepted.files.length) { + await restoreAcceptedSource(); + throw Object.assign( + new Error('materialized application source differs from its accepted snapshot'), + { code: 'generated_app_source_changed' }); + } + if (startFailure) { + await restoreAcceptedSource(); + throw startFailure; + } +} + +export function materializationAppFailure( + error: unknown, +): RunOutcome { + const code = error && typeof error === 'object' && 'code' in error ? error.code : null; + if (code === 'generated_app_source_changed' + || code === 'generated_app_start_contract_missing' + || code === 'generated_app_not_restartable') { + const reason = code === 'generated_app_source_changed' + ? 'application startup changed the accepted source' + : code === 'generated_app_not_restartable' + ? `application did not start from clean source: ${redactCredentials(errorMessage(error)) + .replace(/\s+/g, ' ').slice(0, 600)}` + : 'accepted application source has no /app/start.sh'; + return { kind: 'app_failure', phase: 'application-restart', reason, + appFailures: ['application-restart'], inconclusive: [], harnessFailures: [] }; + } + throw error; +} + +// Check contamination after every coding session. File-tool permissions do not +// govern shell reads, so the transcript audit remains a separate hard gate. +function auditContamination(appDir: string): ContaminationAudit | null { + const args = [join(ROOT, 'dist', 'commands', 'leak-audit.js'), '--app', appDir, '--json']; + let firstFailure: unknown = null; + for (let attempt = 1; attempt <= 2; attempt++) { + try { + const audit = sh('node', args, { stdio: 'pipe' }); + const escapes = parseLeakAudit(audit).flatMap(entry => entry.hits); + const serious = escapes.filter(h => /GRADER|CONTRACT|BENCHMARK NOTES|PROMPTS/.test(h.kind)); + if (firstFailure) { + console.error(` warning: contamination audit passed on retry after: ${auditFailureSummary(firstFailure)}`); + } + if (!serious.length) return null; + return { kind: 'contaminated', + evidence: [...new Set(serious.map(h => `${h.kind}: ${h.path.split('/').slice(-2).join('/')}`))].slice(0, 8), + verdict: 'SCORES NOT USABLE — the build read the harness that grades it.' }; + } catch (error) { + firstFailure ??= error; + if (attempt === 2) { + // An audit that could not run is not a pass. Keep the process details so + // the failure can be repaired without another paid reproduction. + return { kind: 'harness_failure', + evidence: [`audit did not run after retry: ${auditFailureSummary(error)}`], + verdict: 'SCORES NOT USABLE — nothing verified this build stayed inside its directory.' }; + } + } + } + return null; +} + +export function auditFailureSummary(error: unknown): string { + const failure = object(error) ? error : {}; + const message = errorMessage(error).split(/\r?\n/)[0] ?? ''; + const stderr = String(failure.stderr ?? '').trim().split(/\r?\n/)[0]; + const details = [ + Number.isInteger(failure.status) ? `exit ${String(failure.status)}` : null, + failure.signal ? `signal ${String(failure.signal)}` : null, + stderr ? `stderr: ${stderr}` : null, + ].filter(Boolean); + return details.length ? `${message} (${details.join('; ')})` : message; +} + +function containerIdentity(name: string): { name: string; id: string } { + try { + const id = execFileSync('docker', ['inspect', '--format', '{{.Id}}', name], + { encoding: 'utf8', stdio: 'pipe', timeout: 120_000 }).trim(); + if (!id) throw new Error('empty container id'); + return { name, id }; + } catch (error) { + throw new Error(`cannot lease ${name}: ${errorMessage(error).split('\n')[0]}`); + } +} + +const sh = (cmd: string, args: readonly string[], + opts: Omit = {}): string => + execFileSync(cmd, [...args], { + encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: COMMAND_TIMEOUT_MS, ...opts, + }); + +let activeAgentChild: ChildProcess | null = null; +// Set once a run owns resources. The top-level rejection handler invokes this +// directly; relying only on process 'exit' made cleanup best-effort precisely +// when an awaited build rejected unexpectedly. +let emergencyTeardown: (() => void) | null = null; + +function runAgent( + args: BenchArgs, + adapter: AgentAdapter, + mode: AgentMode, + level: number, + appDir: string, +): Promise { + if (!args.backend || !args.model) throw new Error('agent run requires backend and model'); + const remainingBudget = args.maxBudgetUsd == null ? null + : addCostUsd(args.maxBudgetUsd, -(args.spentBudgetUsd ?? 0)); + if (remainingBudget !== null && remainingBudget <= 0) { + throw new Error(`attempt cost cap of $${args.maxBudgetUsd} was exhausted before ${mode} L${level}`); + } + if (remainingBudget !== null && adapter.costLimit === 'unsupported') { + throw new Error(`agent adapter ${adapter.id} cannot enforce --max-budget-usd`); + } + const recipeTask = args.recipeTasks?.get(level)?.agentRequest + ?? args.recipeTasks?.get(level)?.request ?? null; + const request: AgentRequest = { mode, level, app: appDir, backend: args.backend, track: args.track, + runIndex: args.runIndex, model: args.model, guidance: args.guidance, skills: args.skills, + ...(adapter.usesStackSkills + ? { skillIdentity: args.condition?.guidance.skills[args.backend] } : {}), + recipe: agentRecipeIdentity(args.recipe, recipeTask), + guidanceDocument: args.guidanceDocument, + credentialAliases: args.condition?.guidance?.credentialAliases ?? {}, + recipeTask, pricing: args.pricing, + maxBudgetUsd: remainingBudget, adapterCostLimit: adapter.costLimit }; + const argv = agentRequestArgv(adapter, request); + if (args.apiKey && !adapter.apiKeyEnvironmentVariable) { + throw new Error(`agent adapter ${adapter.id} does not accept an API key`); + } + const env = { ...process.env }; + if (args.apiKey) { + const credentialName = adapter.apiKeyEnvironmentVariable; + if (!credentialName) throw new Error(`agent adapter ${adapter.id} does not accept an API key`); + env[credentialName] = args.apiKey; + } + return new Promise((resolveRun, rejectRun) => { + const child = execFile('node', argv, { + encoding: 'utf8', maxBuffer: 64 * 1024 * 1024, timeout: adapter.deadlineMs, + env, + }, + (error, stdout, stderr) => { + if (activeAgentChild === child) activeAgentChild = null; + if (error) { + error.stdout = stdout; + error.stderr = stderr; + rejectRun(error); + return; + } + try { + const resultLine = stdout.trim().split('\n').pop(); + if (!resultLine) throw new Error('agent returned no result line'); + const result = validateAgentResult(JSON.parse(resultLine), request); + args.spentBudgetUsd = addCostUsd(args.spentBudgetUsd, result.costUsd); + resolveRun(result); + } + catch (parseError) { + // Preserve bounded output tails when the agent result is malformed; + // teardown may remove the container that produced them. + const stdoutTail = stdout.trim().slice(-2000) || ''; + const stderrTail = stderr.trim().slice(-4000) || ''; + rejectRun(new Error(`agent returned invalid JSON: ${errorMessage(parseError)}\n` + + `agent stdout tail:\n${stdoutTail}\nagent stderr tail:\n${stderrTail}`)); + } + }); + activeAgentChild = child; + }); +} + +interface GradeCheck { + stableKey: string; + executionId?: string; + source?: string; +} + +interface GradeRecipeTask { + request: UnknownRecord; + selection: { checks: readonly GradeCheck[] } + | { scoredChecks: readonly GradeCheck[]; observedChecks?: readonly GradeCheck[] }; +} + +function checksForGrade(task: GradeRecipeTask | undefined, observation: GradeOptions['observation']): + readonly GradeCheck[] { + if (!task) return []; + if ('scoredChecks' in task.selection) { + return observation === 'observed' + ? task.selection.observedChecks ?? [] : task.selection.scoredChecks; + } + return task.selection.checks; +} + +type GradeArguments = Pick & { + recipeTasks?: ReadonlyMap; + progression?: { identity: { policy?: string } }; + condition?: { guidance?: { credentialAliases?: Record } }; +}; + +export function gradeArgv( + args: GradeArguments, + appDir: string, + url: string, + label: string, + level: number, + track: Track, + parentAttemptId: string, + { observation = 'scored', out = null, sourceSha256 = null, + applicationFailure = null }: GradeOptions = {}, +): string[] { + if (!args.backend) throw new Error('grading requires a backend'); + const restartSpec = restartSpecFor(args, appDir, track); + const task = args.recipeTasks?.get(level); + return [compiledEntrypoint('commands', 'run-suite.js'), '--app', appDir, '--url', url, + '--backend', args.backend, '--label', label, '--level', String(level), + '--track', args.track, + '--run-index', String(args.runIndex), + '--parent-attempt-id', parentAttemptId, + '--observation', observation, + ...(out ? ['--out', out] : []), + ...(sourceSha256 ? ['--source-sha256', sourceSha256] : []), + ...(args.recipe ? ['--recipe', args.recipe] : []), + ...(task ? ['--recipe-task-json', JSON.stringify(task.request)] : []), + ...(args.condition?.guidance?.credentialAliases + ? ['--credential-aliases-json', JSON.stringify( + args.condition.guidance.credentialAliases)] : []), + ...(applicationFailure + ? ['--application-failure-json', JSON.stringify(applicationFailure)] : []), + ...(observation === 'scored' && args.recipeTasks && !args.progression + ? ['--regression-checks-json', JSON.stringify([...args.recipeTasks.entries()] + .filter(([priorLevel]) => priorLevel < level) + .flatMap(([, priorTask]) => checksForGrade(priorTask, 'scored') + .map(check => check.stableKey)))] : []), + ...(args.media && observation === 'scored' ? [] : ['--no-media']), + ...(!executeStackCapability(STACK_ADAPTER_REGISTRY.get(args.backend), + 'run-policy', 'reset-enabled') + ? ['--no-reset'] + : ['--restart-spec', JSON.stringify(restartSpec)])]; +} + +function grade( + args: BenchArgs, + appDir: string, + url: string, + label: string, + level: number, + track: Track, + parentAttemptId: string, + options: GradeOptions = {}, +): GradeBundlePayload | null { + const { out = null } = options; + const source = hashAppSource(appDir); + const argv = gradeArgv(args, appDir, url, label, level, track, parentAttemptId, { + ...options, sourceSha256: options.sourceSha256 ?? source.sha256, + }); + const bundle = join(out ?? join(appDir, 'stack-bench'), 'bundle.json'); + rmSync(bundle, { force: true }); + const task = args.recipeTasks?.get(level); + const currentChecks = checksForGrade(task, options.observation); + const regressionChecks = options.observation === 'observed' || args.progression + ? [] + : [...(args.recipeTasks?.entries() ?? [])] + .filter(([priorLevel]) => priorLevel < level) + .flatMap(([, priorTask]) => checksForGrade(priorTask, 'scored')); + const sourceCount = task + ? selectedGradingSourceCount(currentChecks, regressionChecks) + : suitesFor(track, level).length; + try { + sh('node', argv, { stdio: 'inherit', timeout: gradingRunTimeoutMs(sourceCount) }); + } catch { /* a current bundle may still explain a scored failure */ } + return existsSync(bundle) + ? readArtifactPayload(bundle, { expectedKind: 'grade_bundle' }) : null; +} + +function restartSpecFor(args: Pick, + appDir: string, track: Track): RestartSpec { + if (!args.backend) throw new Error('restart specification requires a backend'); + const port = portsFor(track, args.backend, args.runIndex).vite ?? null; + return { backend: args.backend, app: appDir, port: port == null ? null : Number(port), + probe: '' }; +} + +export function pristineMutationBaselinePath( + args: Pick, + exists: (path: string) => boolean = existsSync, +): string | null { + if (args.referenceMutationOnly) return args.mutationBaselineBundle ?? null; + if (args.mutationBaselineBundle) return args.mutationBaselineBundle; + const level = args.levelList?.at(-1); + if (typeof level !== 'number' || !Number.isSafeInteger(level) || level < 1 || !args.out) return null; + const candidate = join(args.out, `first-build-l${level}-grading`, 'bundle.json'); + return exists(candidate) ? candidate : null; +} + +function runMutationControl( + args: BenchArgs, + appDir: string, + url: string, + track: Track, + imageId: string | null, +): MutationControlResult { + if (!args.out || !args.mutations) throw new Error('mutation control requires output and manifest paths'); + const output = join(args.out, 'mutation-control.json'); + if (!args.mutationResumeFrom || resolve(args.mutationResumeFrom) !== resolve(output)) { + rmSync(output, { force: true }); + } + if (imageId) args.mutationImageId = imageId; + else delete args.mutationImageId; + const manifest = JSON.parse(readFileSync(args.mutations, 'utf8')); + const argv = mutationControlArgv(mutationControlArgs(args), appDir, url, track); + let processError = null; + try { sh(process.execPath, argv, { + stdio: 'inherit', timeout: mutationControlTimeoutMs(manifest, + args.mutationMaxRuntimeMinutes), + }); } + catch (error) { processError = errorMessage(error).split('\n')[0] ?? null; } + if (!existsSync(output)) { + return { ok: false, artifact: output, processError, + outcome: { kind: 'harness_failure', phase: 'mutation-control', + reason: processError ?? 'mutation runner produced no artifact' } }; + } + const artifact = readArtifactPayload(output, { expectedKind: 'mutation_control' }); + return { ok: artifact.ok === true && !processError, artifact: output, + processError, summary: artifact.summary ?? null, outcome: mutationOutcome(artifact.outcome), + results: artifact.results ?? [] }; +} + +function validateMutationInput(args: BenchArgs): void { + if (!args.mutations) return; + if (!args.app) throw new Error('--mutations requires an explicit pristine --app'); + const manifest = JSON.parse(readFileSync(args.mutations, 'utf8')); + if (!/^[a-f0-9]{64}$/.test(manifest.fixtureSha256 ?? '')) { + throw new Error('mutation manifest has no valid fixtureSha256'); + } + const fixture = hashDirectory(args.app); + if (fixture.sha256 !== manifest.fixtureSha256) { + throw new Error(`mutation manifest targets fixture ${manifest.fixtureSha256}, not ${fixture.sha256}`); + } +} + +export function validateProgressionCampaignLevelScope( + binding: RecipeBinding, + progression: ProgressionInput, + declared: StudyCondition['requested']['levels'][number] | null | undefined, + level: number, +) { + if (!declared) throw new Error(`study condition does not bind requested L${level}`); + const derived = resolveProgressionRecipeLevelSelection(binding, progression, level); + if (declared.recipe.contentSha256 !== derived.grader.request.recipe.contentSha256 + || declared.selection.sha256 !== derived.grader.selection.sha256 + || declared.task.sha256 !== derived.grader.task.sha256) { + throw new Error(`dependency campaign graph-derived scope changed before L${level}`); + } + return derived; +} + +async function main() { + const args: BenchArgs = { + ...parseBenchArguments(process.argv), + recipeTasks: new Map(), + recipeBindings: new Map(), + }; + let repairGrant = null; + if (args.repairFrom) { + const repairLevel = args.repairLevel; + if (typeof repairLevel !== 'number' || !Number.isSafeInteger(repairLevel) || repairLevel < 1) { + throw new Error('--repair-from requires a positive --repair-level'); + } + repairGrant = createRepairGrant(args.repairFrom, + { level: repairLevel, rounds: args.fixRounds }); + const config = repairGrant.configuration; + if (config.buildImage && process.env.STACK_BENCH_IMAGE + && config.buildImage !== process.env.STACK_BENCH_IMAGE) { + throw new Error('repair continuation build image differs from its parent run'); + } + if (config.buildImage) process.env.STACK_BENCH_IMAGE = config.buildImage; + Object.assign(args, { + backend: config.backend, + track: config.track, + recipe: config.recipe, + levels: String(config.level), + levelList: [config.level], + runIndex: config.runIndex, + agentAdapter: config.agentAdapter, + model: config.model, + guidance: config.guidance, + guidanceDocument: config.guidanceDocument, + condition: config.condition, + selectionRequest: campaignSelection(config.selectionRequest, 'repair configuration.selectionRequest'), + skills: config.skills, + packIds: [...(campaignSelection(config.selectionRequest, + 'repair configuration.selectionRequest').packs ?? [])], + checkKeys: [...(campaignSelection(config.selectionRequest, + 'repair configuration.selectionRequest').checks ?? [])], + featureIds: [], + requestedSpecifications: [], + expectedSpecifications: [], + observedSpecifications: [], + seedFrom: repairGrant.sourcePath, + url: config.url, + parentAttemptId: repairGrant.parent.id, + repairGrant, + }); + } + if (!args.backend) throw new Error('benchmark run requires a backend'); + const stackAdapter = STACK_ADAPTER_REGISTRY.get(args.backend); + const materializeCodingOutput = stackAdapter.id !== 'stub'; + const agentAdapter = AGENT_ADAPTER_REGISTRY.get(args.agentAdapter); + if (repairGrant) { + const currentAgent = agentAdapterIdentity(agentAdapter); + const parentAgent = repairGrant.parentArtifact.identities.agentAdapter; + if (currentAgent.id !== parentAgent?.id || currentAgent.version !== parentAgent?.version + || currentAgent.sha256 !== parentAgent?.sha256) { + throw new Error('repair continuation agent adapter differs from its parent run'); + } + if (stackAdapter.id !== repairGrant.parentArtifact.identities.stackAdapter?.id + || stackAdapter.version !== repairGrant.parentArtifact.identities.stackAdapter?.version) { + throw new Error('repair continuation stack adapter differs from its parent run'); + } + } + resolveAgentCredential(args, agentAdapter); + args.model ??= agentAdapter.defaultModel; + if (!args.model) throw new Error(`agent adapter ${agentAdapter.id} has no default model`); + if (args.pricing !== undefined) { + args.pricing = validatePricingAuthority(args.pricing, { at: '--pricing-json' }); + } else if (args.maxBudgetUsd != null && agentAdapter.costLimit === 'native') { + const rates = claudeRatesForModel(args.model); + if (!rates) throw new Error(`no default pricing is recorded for model ${args.model}`); + args.pricing = validatePricingAuthority({ unit: PRICING_UNIT, rates }, + { at: 'default pricing' }); + } else { + args.pricing = null; + } + if (args.retainBackend + && !executeStackCapability(stackAdapter, 'run-policy', 'retain-host-supported')) { + throw new Error(`stack adapter ${args.backend} does not support --retain-backend`); + } + const stackRuntime = stackRuntimeConfig(executeStackCapability(stackAdapter, + 'orchestrator', 'config', { root: ROOT, env: process.env, helpers: { exists: existsSync } })); + Object.assign(process.env, stackRuntime.environment); + process.env.STACK_BENCH_NODE_BIN = process.platform === 'win32' ? 'node.exe' : process.execPath; + const track = loadTrack(args.track); + // Resolve the requested scope for every level before probing the sandbox, + // acquiring a backend lease or paying for a build. A pack that exists at L2 + // but not L1 is not a late grading surprise; it is an invalid run request. + args.selectionRequest ??= { packs: [...args.packIds], checks: [...args.checkKeys] }; + for (const level of args.levelList) { + const declared = args.condition?.requested?.levels?.find(entry => entry.level === level) ?? null; + const modularSelection = args.selectionRequest.levels?.find(entry => entry.level === level) ?? null; + if (declared?.selection?.schemaVersion === 3) { + const expected = args.featureCatalog + ? { level, recipe: `${declared.recipe.id}@${declared.recipe.version}` } + : { level, recipe: `${declared.recipe.id}@${declared.recipe.version}`, + features: declared.selection.requested.features, + checks: declared.selection.requested.checks }; + if (canonicalDefinitionJson(modularSelection) !== canonicalDefinitionJson(expected)) { + throw new Error(`campaign selection changed before L${level}`); + } + } else if (modularSelection) { + throw new Error(`campaign selection declares modular L${level} without a modular condition`); + } + const declaredRecipe = declared + ? `${declared.recipe.id}@${declared.recipe.version}` : null; + const binding = resolveRecipeRelease(track, level, declaredRecipe ?? args.recipe); + if (!binding && (args.packIds.length || args.checkKeys.length)) { + throw new Error(`L${level} has no recipe release, so --pack/--check cannot be resolved`); + } + if (binding) { + args.recipeBindings.set(level, binding); + if (args.featureCatalog) { + validateProgressionCampaignLevelScope(binding, args.featureCatalog, declared, level); + } + const requested = declared?.selection?.requested; + const progressionSelection = args.featureCatalog + ? resolveProgressionRecipeLevelSelection(binding, args.featureCatalog, level, + { cumulative: Boolean(args.progression) }) : null; + const resolved = progressionSelection === null + ? createBoundRecipeTaskRequest(binding, requested?.features + ? { featureIds: requested.features, + requestedSpecifications: requested.specifications?.requested, + expectedSpecifications: requested.specifications?.expected, + observedSpecifications: requested.specifications?.observed, + checkKeys: requested.checks } + : args) : null; + const grader = progressionSelection?.grader ?? resolved; + if (!grader) throw new Error(`L${level} has no recipe task request`); + if (args.condition && !declared) { + throw new Error(`study condition does not bind requested L${level}`); + } + const graderIdentity = recipeRequestIdentity(grader.request); + if (declared && (declared.recipe.contentSha256 !== graderIdentity.recipeSha256 + || declared.selection.sha256 !== graderIdentity.selectionSha256 + || JSON.stringify(declared.selection.taskPacks) !== JSON.stringify(graderIdentity.taskPacks) + || declared.task.sha256 !== graderIdentity.taskSha256)) { + throw new Error(`study condition requested scope changed before L${level}`); + } + if (progressionSelection) { + const progressionGrader = progressionSelection.grader; + args.recipeTasks.set(level, { + request: progressionGrader.request, + selection: progressionGrader.selection, + task: progressionGrader.task, + agentRequest: progressionSelection.agent.request, + }); + } else if (resolved) { + args.recipeTasks.set(level, { + ...resolved, + agentRequest: createAgentVisibleTaskRequest(binding, resolved), + }); + } + } + } + if (args.progression) { + const state = progressionEngine.initialize(args.progression.definition); + const declared = args.condition?.requested?.levels + ?.find(entry => entry.level === state.level) ?? null; + const binding = resolveRecipeRelease(track, state.level, + declared ? `${declared.recipe.id}@${declared.recipe.version}` : null); + if (!binding) throw new Error(`L${state.level} has no recipe release`); + resolveProgressionRecipeAction(binding, state); + if (!args.progressionOwner) { + throw new Error('live dependency progression requires an exact compiled campaign attempt'); + } + } + if (repairGrant) { + const expectedSelection = repairGrant.level.selection?.sha256 ?? null; + const repairTask = args.recipeTasks.get(repairGrant.level.level); + const resolvedSelection = repairTask ? recipeRequestIdentity(repairTask.request).selectionSha256 : null; + if (resolvedSelection !== expectedSelection) { + throw new Error('repair continuation test selection differs from its parent run'); + } + } + if (!args.selectionRequest.levels && (JSON.stringify(args.selectionRequest.packs) !== JSON.stringify(args.packIds) + || JSON.stringify(args.selectionRequest.checks) !== JSON.stringify(args.checkKeys))) { + throw new Error('campaign pack/check selection changed before execution'); + } + // Caller-owned mutation inputs are pure request data. Reject them before + // checking credentials, Docker, ports, or any other ambient runner state so + // an invalid experiment can never be masked by an unrelated preflight error. + validateMutationInput(args); + assertNoPortCollisions(); + // The deterministic adapter/stack is the model-free unit loop. Real runs + // prove the exact requested scope, engine, image, credentials, storage and + // ports before the sandbox probe or any paid coding session begins. + const admittedSmoke = args.campaignAdmission?.reusable === true + ? { id: args.campaignAdmission.id, createdAt: args.campaignAdmission.createdAt } + : null; + const preflight = args.backend === 'stub' ? null : runPreflight({ + backends: [args.backend], track: args.track, levels: args.levels, + levelList: args.levelList, runIndex: args.runIndex, agentAdapter: args.agentAdapter, + guidance: args.guidance, + recipe: args.recipe, + ...(args.condition?.requested ? { requestedScopes: [args.condition.requested] } : {}), + ...(args.featureCatalog ? { featureCatalog: args.featureCatalog } : {}), + ...(args.runMode ? { mode: args.runMode } : {}), + agentSkills: args.skills ?? null, + packIds: args.packIds, checkKeys: args.checkKeys, smoke: admittedSmoke === null, + ...(admittedSmoke ? { admittedSmoke } : {}), + ...(process.env.STACK_BENCH_SUPERVISOR_STATE + ? { supervisorState: process.env.STACK_BENCH_SUPERVISOR_STATE } : {}), + image: process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE, + resultsDir: resolve(args.out ?? process.env.STACK_BENCH_RESULTS_DIR ?? join(ROOT, 'results')), + }, { env: args.apiKey && agentAdapter.apiKeyEnvironmentVariable + ? { ...process.env, [agentAdapter.apiKeyEnvironmentVariable]: '' } + : process.env }); + if (preflight && !preflight.ok) { + const failures = preflight.checks.filter(check => check.status === 'fail'); + console.error('\nPREFLIGHT FAILED — no model session was started.'); + for (const failure of failures) { + console.error(` ${failure.id}: ${failure.summary}`); + if (failure.remediation) console.error(` fix: ${failure.remediation}`); + } + process.exit(2); + } + if (preflight) console.log(` preflight ... ${preflight.summary.passed} checks passed` + + `${preflight.summary.warnings ? `, ${preflight.summary.warnings} warning(s)` : ''}`); + // In a single-host topology, prove the file-tool sandbox before model spend. + // The appliance instead relies on structural isolation: the coding container + // has no controller, grader, scenarios, prior results, or Docker socket. + // The stub backend is the offline test loop: no model, no cost, nothing to + // protect. Spending a real CLI session probing it would make the one test + // that is supposed to run for free stop being free. + const probeMode = sandboxProbeMode({ appliance: process.env.STACK_BENCH_APPLIANCE === '1', + explicitlySkipped: args.skipProbe, stackRequired: booleanResult(executeStackCapability(stackAdapter, + 'run-policy', 'sandbox-probe-required'), 'sandbox-probe-required') + && agentAdapter.sandboxProbe === 'direct-cli' }); + if (probeMode === 'container-isolation') { + console.log(' sandbox ... coding container is isolated from the controller and grading files'); + } else if (probeMode === 'direct-cli') { + console.log(' sandbox ... probing the deny rules'); + try { + sh('node', [compiledEntrypoint('commands', 'probe-sandbox.js'), '--mode', 'acceptEdits', '--model', args.model], + { stdio: 'inherit' }); + } catch { + console.error('\nSANDBOX PROBE FAILED — refusing to start a run whose scores could not be trusted.'); + console.error('Run `node dist/commands/probe-sandbox.js --mode acceptEdits` to see which path got through.'); + process.exit(2); + } + } + let url = args.url ?? `http://localhost:${portsFor(track, args.backend, args.runIndex).vite}`; + const runDir = resultsName(track, args.backend, args.runIndex); + const runId = newRunId({ track: args.track, backend: args.backend, runIndex: args.runIndex }); + const artifactLabel = `${runDir}-${runId}`; + // Default results never reuse a directory. The stable backend/run name is a + // grouping directory only; every artifact beneath it belongs to one run id. + args.out ??= join(process.env.STACK_BENCH_RESULTS_DIR ?? join(ROOT, 'results'), runDir, runId); + if (!args.out) throw new Error('benchmark run has no results directory'); + const outputDir = args.out; + mkdirSync(args.out, { recursive: true }); + if (existsSync(join(args.out, 'run.json'))) { + throw new Error(`refusing to reuse result directory containing run.json: ${args.out}`); + } + if (preflight) writeArtifact(join(args.out, 'preflight.json'), { + kind: 'preflight', id: `${runId}-preflight`, + attempt: { id: `${runId}-preflight`, parentId: runId }, + identities: emptyArtifactIdentities({ + agentAdapter: agentAdapterIdentity(agentAdapter), + stackAdapter: { id: stackAdapter.id, version: stackAdapter.version }, + }), + payload: preflight, + }); + + // Validate caller-owned source before acquiring a backend slot so a bad + // fixture cannot leave leased resources behind. + const ownWorkDir = !args.app; + const appDir = args.app ?? join(workDirFor(track, args.backend, args.runIndex, runId), 'app'); + if (args.repairGrant && url.startsWith('file:')) { + url = pathToFileURL(join(appDir, 'index.html')).href; + } + + // Bind destructive and lifecycle operations to exact resource identities and + // an ownership token. Targets come only from the lease, never generated code. + const runtimeRoot = resolve(process.env.STACK_BENCH_RUNTIME_DIR + ?? join(tmpdir(), 'stack-bench-runtime')); + const runtimeDir = join(runtimeRoot, runId); + const leasePath = join(runtimeDir, 'backend-lease.json'); + const preparedLease = preparedLeaseResult(executeStackCapability(stackAdapter, 'lease', 'prepare', { + track, + runIndex: args.runIndex, + runtimeDir, + serverUri: stackRuntime.lease.serverUri, + env: process.env, + helpers: { containerIdentity, dbName, moduleName }, + })); + const initialLease = createBackendLease({ + runId, + backend: args.backend, + track: args.track, + runIndex: args.runIndex, + ...preparedLease.lease, + }); + const lockScope = resourceLockScope(); + const lockKeys = backendResourceLockKeys(initialLease, preparedLease.lockKeys); + let privateSupervisorStatePath = null; + try { + initialLease.resources.locks.push(...acquireResourceLocks({ + ...lockScope, keys: lockKeys, lease: initialLease, + })); + writeBackendLease(leasePath, initialLease); + const supervisorState = process.env.STACK_BENCH_SUPERVISOR_STATE + ?? (process.env.STACK_BENCH_SUPERVISOR_DIR + ? join(resolve(process.env.STACK_BENCH_SUPERVISOR_DIR), `${runId}.json`) : null); + if (supervisorState) { + // Private handoff to an outer timeout supervisor. It contains the lease + // token, so create it once with owner-only permissions and never place it + // in the results tree. + privateSupervisorStatePath = resolve(supervisorState); + mkdirSync(dirname(privateSupervisorStatePath), { recursive: true, mode: 0o700 }); + writeFileSync(privateSupervisorStatePath, `${JSON.stringify({ + version: SUPERVISOR_STATE_VERSION, runId, backend: args.backend, runtimeDir, leasePath, + ownershipToken: initialLease.ownershipToken, output: resolve(args.out), + })}\n`, { flag: 'wx', mode: 0o600 }); + } + } catch (error) { + releaseResourceLocks(initialLease); + rmSync(runtimeDir, { recursive: true, force: true }); + throw error; + } + process.env.STACK_BENCH_LEASE = leasePath; + process.env.STACK_BENCH_LEASE_TOKEN = initialLease.ownershipToken; + if (process.platform === 'win32') { + // When Windows resolves `bash` through WSL, WSLENV must carry lease paths + // and tokens into lifecycle scripts with path translation. + const bridge = ['STACK_BENCH_LEASE/p', 'STACK_BENCH_LEASE_TOKEN', + 'STACK_BENCH_NODE_BIN', ...stackRuntime.windowsEnvironmentBridge]; + const existing = (process.env.WSLENV ?? '').split(':').filter(Boolean); + process.env.WSLENV = [...new Set([...existing, ...bridge])].join(':'); + } + + let tornDown = false; + let activeRun: BenchmarkRunRecord | null = null; + const recoveryPath = join(outputDir, 'recovery.json'); + const writeLeaseEvidence = (knownLease: BackendLease | null = null) => { + const lease = knownLease ?? readBackendLease(leasePath, + { token: initialLease.ownershipToken, backend: args.backend, runId }); + const out = join(outputDir, 'backend-lease.json'); + const evidence = publicBackendLease(lease); + const id = `${runId}-backend-lease`; + writeArtifact(out, { + kind: 'backend_lease_evidence', id, + attempt: { id, parentId: runId }, + timestamps: { startedAt: evidence.createdAt, completedAt: new Date().toISOString() }, + identities: emptyArtifactIdentities({ stackAdapter: { id: args.backend } }), + payload: evidence, + }); + return evidence; + }; + const teardown = ({ reason = null, retainBackend = args.retainBackend }: + { reason?: string | null; retainBackend?: boolean } = {}) => { + if (tornDown) return; + if (activeAgentChild?.pid) { + killTree(activeAgentChild.pid); + activeAgentChild = null; + } + // Preserve restart failures before removing the only filesystem that holds + // their stderr. A 500 after restart is otherwise impossible to distinguish + // from an application defect, a dead dependency, or host pressure. + if (activeRun) { + try { + activeRun.backendDiagnostics = captureBackendDiagnostics(join(outputDir, 'backend.log')); + } catch (error) { + activeRun.backendDiagnostics = { captured: false, + reason: errorMessage(error).split(/\r?\n/)[0] }; + } + } + let released = false; + let cleanupError: unknown = null; + try { + released = releaseBackendLease(leasePath, initialLease.ownershipToken, + { retainBackend }); + } catch (error) { cleanupError = error; } + let finalLease = initialLease; + try { + finalLease = readBackendLease(leasePath, + { token: initialLease.ownershipToken, backend: args.backend, runId }); + } catch (error) { cleanupError ??= error; released = false; } + const evidence = writeLeaseEvidence(finalLease); + writeRecoveryArtifact(recoveryPath, finalLease, { cleanupSucceeded: released, + retained: Boolean(retainBackend), + reason: cleanupError === null ? reason ?? (released ? null : 'authenticated cleanup refused') + : errorMessage(cleanupError) }); + if (activeRun) { + activeRun.backendLease = evidence; + activeRun.outcome ??= aggregateRunOutcome(activeRun.levels); + writeRunJson(join(outputDir, 'run.json'), activeRun); + } + tornDown = released; + if (released && !retainBackend) { + rmSync(runtimeDir, { recursive: true, force: true }); + if (privateSupervisorStatePath) rmSync(privateSupervisorStatePath, { force: true }); + } + if (cleanupError) throw cleanupError; + if (!released) throw new Error(`backend teardown refused: listener no longer matches lease ${runId}`); + }; + emergencyTeardown = teardown; + + try { + executeStackCapability(stackAdapter, 'lifecycle', 'activate', { + leasePath, leaseToken: initialLease.ownershipToken, lease: initialLease, + ...stackRuntime.lifecycle, + }); + } catch (error) { + try { teardown({ reason: `backend activation failed: ${errorMessage(error)}`, retainBackend: false }); } + catch (cleanupError) { + console.error(` activation cleanup quarantined: ${errorMessage(cleanupError).split(/\r?\n/)[0]}`); + } + throw error; + } + + // Grow one isolated app across levels, outside the harness and results tree. + // Copy artifacts back at completion and remove only a work directory created + // by this run; an explicit --app remains caller-owned. + // Leave nothing running once the run is over, however it ends — but only stop + // what this run brought up. + // This run's work path is unique. There is no legitimate pre-existing build + // container to delete; teardown removes one only after run-build records its + // immutable id in the lease. + const interrupt = (signal: NodeJS.Signals, exitCode: number) => { + console.log(`interrupted by ${signal} — stopping exact owned resources`); + try { teardown({ reason: `interrupted by ${signal}` }); } + catch (error) { console.error(` cleanup quarantined: ${errorMessage(error).split(/\r?\n/)[0]}`); } + process.exit(exitCode); + }; + process.on('SIGINT', () => interrupt('SIGINT', 130)); + process.on('SIGTERM', () => interrupt('SIGTERM', 143)); + process.on('exit', () => { + if (!tornDown) { + try { teardown(); } catch (error) { + console.error(` cleanup failed: ${errorMessage(error).split('\n')[0]}`); + } + } + }); + + // Seed the work dir from an existing app, so the first level upgrades it + // rather than building from nothing. Source only; the upgrade session + // installs its own dependencies exactly as a developer checking out the + // earlier code would. The copy is layout-independent for neutral runs. + if (args.seedFrom) { + const from = resolve(args.seedFrom); + if (!existsSync(from)) { console.error(`--seed-from path does not exist: ${from}`); process.exit(2); } + seedAppSource(from, appDir); + console.log(args.repairGrant + ? ` restored L${args.levelList[0]} checkpoint from ${from} for a bounded repair continuation` + : ` seeded from ${from} — level ${args.levelList[0]} will UPGRADE it, not rebuild`); + } + + const started = Date.now(); + const run: BenchmarkRunRecord = { id: runId, + ...(args.repairGrant ? { kind: 'repair_continuation', + continuation: structuredClone(args.repairGrant.grant) } : {}), + startedAt: new Date(started).toISOString(), + parentAttemptId: args.parentAttemptId ?? null, + identities: emptyArtifactIdentities({ + experiment: args.experimentIdentity ?? null, + agentAdapter: agentAdapterIdentity(agentAdapter), + stackAdapter: { id: stackAdapter.id, version: stackAdapter.version }, + }), + mode: args.runMode ?? { id: args.progression ? 'dependency' : 'sequential', + version: args.progression ? DEPENDENCY_MODE_VERSION : '1.0.0' }, + track: args.track, backend: args.backend, model: args.model, + pricing: args.pricing, + guidance: args.guidance, condition: args.condition ?? null, + skills: args.skills ?? [], + runtime: { buildImage: process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE, url }, + selectionRequest: args.selectionRequest, + featureCatalog: args.featureCatalog?.identity ?? null, + dependencyPolicy: args.dependencyPolicy?.identity ?? null, + ...(args.progressionOwner ? { progressionOwner: args.progressionOwner } : {}), + backendLease: publicBackendLease(readBackendLease(leasePath, + { token: initialLease.ownershipToken, backend: args.backend, runId })), + validation: { + ladder: { policy: args.progression ? args.progression.identity.policy : 'pass-before-next-level', + requestedLevels: [...args.levelList], + completedLevels: [], stoppedAfterLevel: null, blockedLevels: [] } }, levels: [] }; + activeRun = run; + + const progressionOwner = args.progression ? { + ...args.progressionOwner, + workspace: { appDirectory: 'source' }, + } : null; + const progressionExecution = args.progression ? createLiveProgressionExecution({ + progression: args.progression, + featureCatalogIdentity: args.featureCatalog?.identity, + dependencyPolicyIdentity: args.dependencyPolicy?.identity, + owner: progressionOwner, + statePath: join(args.out, 'progression-state.json'), + runId, + outputDir: args.out, + appDir, + track: args.track, + backend: args.backend, + identities: run.identities, + recipeBindings: args.recipeBindings, + resumeFrom: args.progressionResumeFrom ?? null, + getRunArtifact: () => { + writeRunJson(join(outputDir, 'run.json'), run); + return readArtifact(join(outputDir, 'run.json')); + }, + onState: status => { + run.progressionStatus = status; + writeRunJson(join(outputDir, 'run.json'), run); + }, + }) : null; + const progressionStart = progressionExecution?.initialize() ?? null; + if (progressionStart?.resumed) { + const prior = progressionStart.priorRun; + if (!prior) throw new Error('resumed dependency progression has no prior run artifact'); + const actionLevel = progressionStart.action.type === 'terminal' + ? Number.MAX_SAFE_INTEGER : progressionStart.action.level; + const inheritedLevels = (prior.payload.levels ?? []) + .filter(level => level.level < actionLevel).map(level => level.level); + run.levels = (prior.payload.levels ?? []) + .filter(level => inheritedLevels.includes(level.level)).map(level => structuredClone(level)); + run.validation.ladder.completedLevels = [...inheritedLevels]; + run.progressionResume = { + priorRunId: prior.id, + priorRunSha256: sha256(canonicalDefinitionJson(prior)), + stateSnapshotSha256: progressionStart.snapshotSha256, + action: progressionStart.action.type === 'terminal' + ? { type: 'terminal' } + : { type: progressionStart.action.type, level: progressionStart.action.level }, + inheritedLevels, + priorTotals: prior.payload.totals ?? null, + }; + run.progressionStatus = progressionStart.status; + writeRunJson(join(args.out, 'run.json'), run); + } + + const bindProgressionAction = (level: number): ProgressionRecipeAction | null => { + if (!progressionExecution) return null; + const selected = progressionExecution.bind(); + if (!isProgressionWorkRecipeAction(selected)) return selected; + if (!args.recipeTasks) throw new Error('recipe task map is unavailable'); + args.recipeTasks.set(level, { + request: selected.grader.request, + selection: selected.grader.selection, + task: selected.grader.task, + agentRequest: selected.agent.request, + progressionAction: selected.action, + }); + return selected; + }; + + const recordProgressionGrade = (input: Parameters['record']>[0]) => + progressionExecution?.record(input) ?? null; + + let runCostComplete = true; + + const runAgentForLevel = async (mode: AgentMode, level: number): Promise => { + try { + const result = await runAgent(args, agentAdapter, mode, level, appDir); + if (result.costComplete !== true) runCostComplete = false; + return result; + } catch (error) { + const reason = errorMessage(error).split(/\r?\n/)[0] ?? 'agent execution failed'; + run.outcome = { kind: 'harness_failure', phase: `agent-${mode}`, + reason, appFailures: [], inconclusive: [], harnessFailures: [reason] }; + run.validation.ladder.stoppedAfterLevel = run.levels.at(-1)?.level ?? null; + run.validation.ladder.blockedLevels = args.levelList.filter(candidate => candidate >= level); + if (progressionExecution) { + run.progressionStatus = progressionExecution.status(); + run.validation.ladder.completedLevels = [...new Set(requireProgressionState(progressionExecution.state).attempts + .filter(attempt => attempt.outcome === 'conclusive') + .map(attempt => attempt.level))]; + } + finalizeRunTotals(run, started, { costComplete: false }); + run.completedAt = new Date().toISOString(); + writeRunJson(join(outputDir, 'run.json'), run); + throw error; + } + }; + + // Stop before grading if a coding session read protected material or if the + // audit itself failed. Keep the paid session and exact cost in run.json even + // though no score may be used. + const abortUnusableSession = (whichSession: string, audit: ContaminationAudit, + levelRecord: UnknownRecord & { level: number }, + selected: ProgressionRecipeAction | null) => { + const reason = audit.evidence.join('; '); + const outcome: RunOutcome = { kind: audit.kind === 'harness_failure' ? 'harness_failure' : 'ungraded', + phase: 'contamination-audit', reason, + appFailures: [], inconclusive: [], + harnessFailures: audit.kind === 'harness_failure' ? [reason] : [] }; + run.contaminated = audit.kind === 'contaminated'; + run.contamination = { evidence: audit.evidence, verdict: audit.verdict, + detectedAt: whichSession }; + const record: RunLevelRecord = { ...levelRecord, error: reason, outcome, + level: levelRecord.level, graded: false, score: null, max: null, selection: null }; + run.levels.push(record); + if (progressionExecution) { + recordProgressionGrade({ selected, bundle: null, level: levelRecord.level, + failure: progressionFailure(outcome), + repair: { status: 'ungraded', budgetRounds: 0, roundsUsed: 0, + stopReason: audit.kind === 'harness_failure' ? 'audit-failure' : 'contaminated' } }); + run.progressionStatus = progressionExecution.status(); + } + run.validation.ladder.stoppedAfterLevel = run.levels.at(-2)?.level ?? null; + run.validation.ladder.blockedLevels = args.levelList + .filter(candidate => candidate >= levelRecord.level); + finalizeRunTotals(run, started, { costComplete: runCostComplete }); + run.outcome = outcome; + run.completedAt = new Date().toISOString(); + if (run.contaminated) { + console.log(`\n !! CONTAMINATED at ${whichSession}:`); + for (const evidence of audit.evidence) console.log(` ${evidence}`); + console.log(' Scores from this run must not be quoted.'); + } else { + console.log(`\n !! HARNESS FAILURE at ${whichSession}:`); + for (const evidence of audit.evidence) console.log(` ${evidence}`); + console.log(' The audit did not establish a usable result.'); + } + try { writeRunJson(join(outputDir, 'run.json'), run); } catch { /* best effort */ } + try { sh('node', [join(ROOT, 'dist', 'commands', 'archive-transcripts.js'), '--app', appDir, '--label', artifactLabel], { stdio: 'pipe' }); } catch { /* best effort */ } + teardown(); + process.exit(4); + }; + + for (const level of args.levelList) { + const t0 = Date.now(); + const continuing = Boolean(args.repairGrant); + console.log(`\n================ ${args.backend} — level ${level} ================`); + + let progressionSelection = bindProgressionAction(level); + if (progressionSelection?.action.type === 'terminal') break; + const resumedRepair = progressionStart?.resumed === true + && progressionStart.action.type === 'repair'; + const priorRepairRounds = resumedRepair + ? progressionStart.priorRun?.payload.levels?.find(item => item.level === level) + ?.repair?.roundsUsed ?? 0 + : 0; + const repairBudgetFor = (selected: ProgressionRecipeAction | null, + completedRepairRounds: number, initialGradePending = false) => selected + && isProgressionWorkRecipeAction(selected) + ? dependencyRepairBudget(selected.action, completedRepairRounds, initialGradePending) + : args.fixRounds; + const levelStrikeNodeIds = new Set(progressionSelection?.action.strikes.nodes + .map(node => node.nodeId) ?? []); + let progressionRepairBudgetRounds = repairBudgetFor( + progressionSelection, priorRepairRounds, !resumedRepair); + const trackProgressionBudget = (selected: ProgressionRecipeAction | null, + completedRepairRounds: number) => { + if (!selected || !isProgressionWorkRecipeAction(selected)) return; + selected.action.strikes.nodes.forEach(node => levelStrikeNodeIds.add(node.nodeId)); + progressionRepairBudgetRounds = Math.max( + progressionRepairBudgetRounds, repairBudgetFor(selected, completedRepairRounds)); + }; + if (resumedRepair) { + sh('node', [join(ROOT, 'dist', 'commands', 'report-bugs.js'), '--app', appDir, + '--history-json', '[]', '--archive', join(args.out, 'repair-reports', + `bug-report-l${level}-resume.md`), ...repairReportArgs(progressionSelection)], + { stdio: 'pipe' }); + clearPrivateGradingEvidence(appDir); + } + + const firstMode = resumedRepair ? 'fix' + : continuing ? 'resume' : args.seedFrom ? 'upgrade' : 'build'; + const build = await runAgentForLevel( + resumedRepair || level === args.levelList[0] ? firstMode : 'upgrade', level); + const buildLeak = auditContamination(appDir); + if (buildLeak) { + const buildSession = runSessionRecord(build, + resumedRepair ? priorRepairRounds + 1 : null); + const sessionTotals = summarizeSessions([buildSession]); + abortUnusableSession(`level ${level} ${firstMode}`, buildLeak, { + level, graded: false, score: null, max: null, selection: null, + ...(resumedRepair + ? { fixCostUsd: build.costUsd, fixSessions: [buildSession], fixRounds: 1, + priorRepairRounds, cumulativeFixRounds: priorRepairRounds + 1 } + : continuing + ? { resumeCostUsd: build.costUsd, resumeSession: buildSession } + : { buildCostUsd: build.costUsd, buildSession }), + sessionTotals, costUsd: build.costUsd, durationMs: Date.now() - t0, + }, progressionSelection); + } + // Carry the agent's own record of the setup up to the run. Comparing two + // scores is only meaningful if the reasoning budget, permission mode and + // CLI version behind them were the same, and that is not knowable after the + // fact unless it was written down at the time. + run.setup ??= build.setup; + if (continuing) { + requireContinuation(run).resumeSetup = { + sessionId: build.sessionId ?? null, + costUsd: build.costUsd, + durationMs: build.durationMs, + sourceVerified: false, + }; + } + // No session, no app. Grading an empty directory yields a real-looking zero + // that is a harness failure, not a result for this backend. + const buildFailure = agentSessionFailure(build); + if (buildFailure) { + console.log(` ABORTED: ${buildFailure.reason}. Details will be kept in ${join(args.out, 'run.json')}`); + const failedSession = runSessionRecord(build); + if (progressionExecution) { + recordProgressionGrade({ selected: progressionSelection, bundle: null, level, + failure: buildFailure, + repair: { status: 'ungraded', budgetRounds: 0, roundsUsed: 0, + stopReason: 'agent-session-failure' } }); + } + run.levels.push({ level, graded: false, score: null, max: null, + selection: null, error: buildFailure.reason, + outcome: buildFailure, + ...(continuing + ? { resumeSession: failedSession, resumeCostUsd: build.costUsd } + : { buildSession: failedSession, buildCostUsd: build.costUsd }), + sessionTotals: summarizeSessions([build]), + costUsd: build.costUsd, durationMs: Date.now() - t0 }); + break; + } + const applicationControl = materializeCodingOutput + ? restartSpecFor(args, appDir, track) : null; + const gradeAcceptedSource = async (sourcePath: string, + label: string): Promise => { + let failure: RunOutcome | null = null; + if (applicationControl) { + try { + await materializeAcceptedSource(sourcePath, appDir, applicationControl); + } catch (error) { + failure = materializationAppFailure(error); + } + } else { + resetAppToSource(sourcePath, appDir); + } + return grade(args, appDir, url, label, level, track, runId, + { applicationFailure: failure }); + }; + if (continuing) { + // The resume session may install dependencies and start arbitrary project + // layouts, but it may not perform an unintended correction. Restoring edited source + // is insufficient: a running server could still hold code compiled from + // those edits. Reject any source mutation and grade only an unchanged + // checkpoint runtime. + const resumed = hashAppSource(appDir); + const repairGrant = args.repairGrant; + if (!repairGrant) throw new Error('repair continuation has no grant'); + if (resumed.sha256 !== repairGrant.checkpoint.payload.source.sha256 + || resumed.files.length !== repairGrant.checkpoint.payload.source.files) { + throw new Error('resume setup changed the parent checkpoint source'); + } + const continuation = requireContinuation(run); + if (!continuation.resumeSetup) throw new Error('repair continuation did not record resume setup'); + continuation.resumeSetup.sourceVerified = true; + } + if (args.referenceMutationOnly) { + run.levels.push({ level, score: null, max: null, graded: false, contractPass: null, + selection: null, + outcome: { kind: 'ungraded', phase: 'reference-mutation-only', + reason: 'the parent qualification owns the full clean grade', + appFailures: [], inconclusive: [], harnessFailures: [] }, + buildSession: runSessionRecord(build), + buildCostUsd: build.costUsd, sessionTotals: summarizeSessions([build]), + costUsd: build.costUsd, durationMs: Date.now() - t0 }); + break; + } + const firstBuildDirectory = continuing ? `baseline-l${level}` : `first-build-l${level}`; + const firstBuildPath = join(args.out, firstBuildDirectory); + let firstBuildSource = null; + let materializationOutcome: RunOutcome | null = null; + try { + const liveSource = hashAppSource(appDir); + snapshotSource(appDir, firstBuildPath); + const preservedSource = hashDirectory(firstBuildPath); + if (liveSource.sha256 !== preservedSource.sha256) { + throw new Error('preserved first-build source differs from the live application source'); + } + firstBuildSource = { sha256: liveSource.sha256, files: liveSource.files.length }; + if (applicationControl) { + await materializeAcceptedSource(firstBuildPath, appDir, applicationControl); + } + console.log(` kept the ${continuing ? 'continuation baseline' : 'unaided'} source at ${firstBuildPath}`); + } catch (error) { + if (firstBuildSource) { + materializationOutcome = materializationAppFailure(error); + } + console.log(materializationOutcome + ? ` !! ${materializationOutcome.reason}` + : ` !! could not bind the first-build source: ${errorMessage(error).split('\n')[0]}`); + } + const firstBuildLabel = `${args.backend}-l${level}`; + let bundle = firstBuildSource + ? grade(args, appDir, url, firstBuildLabel, level, track, runId, + { applicationFailure: materializationOutcome }) : null; + + // What the model built BEFORE being handed the answers. Every backend can + // reach the same total given enough fix rounds, so the post-fix score stops + // discriminating — what it got right unaided is the comparison that survives. + const firstBuild: FirstBuildRecord = { + score: bundle?.totals?.score ?? null, + max: bundle?.totals?.max ?? null, + regression: bundle?.totals?.regression ?? null, + contractPass: bundle?.totals?.contractPass ?? null, + outcome: materializationOutcome ?? sourceBoundFirstBuildOutcome(bundle, firstBuildSource), + source: firstBuildSource, + missed: Object.values(bundle?.suites ?? {}).flatMap(s => + (s?.features ?? []).flatMap(f => + (f.criteria ?? []).filter(c => !evidencePassed(criterionEvidence(c))) + .map(c => `${f.name}/${c.id}`))), + }; + + if (continuing) { + const repairGrant = args.repairGrant; + if (!repairGrant) throw new Error('repair continuation has no grant'); + if (firstBuild.score === null || firstBuild.max === null || firstBuildSource === null) { + throw new Error('repair continuation did not produce a source-bound baseline score'); + } + const reproduction = compareRepairBaseline(repairGrant.level, { + score: firstBuild.score, + max: firstBuild.max, + selectionSha256: bundle?.selection?.sha256 ?? null, + sourceSha256: firstBuildSource.sha256, + expectedSourceSha256: repairGrant.checkpoint.payload.source.sha256, + outcome: repairOutcome(firstBuild.outcome), + }); + requireContinuation(run).baseline = { + score: firstBuild.score, + max: firstBuild.max, + selectionSha256: bundle?.selection?.sha256 ?? null, + sourceSha256: firstBuildSource?.sha256 ?? null, + outcome: firstBuild.outcome, + ...reproduction, + }; + if (!reproduction.reproduced) { + const reason = `restored checkpoint did not reproduce its parent: ${reproduction.mismatches.join(', ')}`; + console.log(` CONTINUATION STOPPED: ${reason}`); + const failure = { kind: 'harness_failure', phase: 'continuation-baseline', reason, + appFailures: [], inconclusive: [], harnessFailures: [] }; + firstBuild.outcome = failure; + bundle = { ...bundle, outcome: failure }; + } + } + + const selectedObservedChecks = checksForGrade(args.recipeTasks?.get(level), 'observed'); + if (!continuing && !resumedRepair && selectedObservedChecks.length) { + const observationOut = join(args.out, `first-build-l${level}-observed`); + let observationBundle = null; + let observationOutcome; + if (!firstBuildSource) { + observationOutcome = { kind: 'harness_failure', phase: 'first-build-source', + reason: 'observed specifications require a source-bound first build' }; + } else if (!ladderMayContinue(firstBuild.outcome)) { + observationOutcome = { kind: 'ungraded', phase: 'first-build-observation', + reason: 'scored first-build grading did not establish a usable environment' }; + } else { + observationBundle = grade(args, appDir, url, `${args.backend}-l${level}-observed`, level, + track, runId, { observation: 'observed', out: observationOut, + sourceSha256: firstBuildSource.sha256 }); + observationOutcome = classifyBundle(observationBundle); + } + firstBuild.observations = { + sourceSha256: firstBuildSource?.sha256 ?? null, + selectionSha256: args.recipeTasks?.get(level)?.selection.sha256 ?? null, + selectedChecks: selectedObservedChecks.map(check => check.stableKey), + reportedChecks: observationBundle?.selection?.reportedChecks ?? [], + passedPoints: observationBundle?.totals?.score ?? null, + observedPoints: observationBundle?.totals?.max ?? null, + scoreContribution: false, + repairVisible: false, + artifact: observationBundle ? `first-build-l${level}-observed/bundle.json` : null, + outcome: observationOutcome, + }; + } + + // Preserve the first source and scored grading before repair overwrites the + // app. Observed evidence remains in its own source-bound result directory. + try { + const gradingFrom = join(appDir, 'stack-bench'); + if (existsSync(gradingFrom)) { + const gradingDirectory = continuing ? `baseline-l${level}-grading` : `first-build-l${level}-grading`; + cpSync(gradingFrom, join(args.out, gradingDirectory), { + recursive: true, + filter: src => !/[\\/]media([\\/]|$)/.test(src), + }); + console.log(` kept the ${continuing ? 'continuation baseline' : 'unaided'} grading at ${join(args.out, gradingDirectory)}`); + } + } catch (e) { + // Never worth losing a run over: the score is already recorded. + console.log(` !! could not keep the first build: ${errorMessage(e).split('\n')[0]}`); + } + + let fixRounds = resumedRepair ? 1 : 0; + let fixCost = resumedRepair ? build.costUsd : 0; + const fixSessions = resumedRepair + ? [runSessionRecord(build, priorRepairRounds + 1)] : []; + const repairHistory: ReturnType[] = []; + let regressed = false; + let repairStopReason: string | null = null; + let repairProgress = repairProgressState(null, bundle); + const pauseForRepeatedFindings = () => { + if (args.progression) return false; + repairProgress = repairProgressState(repairProgress, bundle); + if (args.maxStalledRepairs === 0 + || repairProgress.stalledRounds < args.maxStalledRepairs) return false; + repairStopReason = 'repeated-findings'; + console.log(` pausing after ${repairProgress.stalledRounds} repair rounds ` + + 'with the same failed checks and no score gain'); + return true; + }; + const initialBundleOutcome = classifyBundle(bundle); + const initialGradeUsable = ladderMayContinue(initialBundleOutcome); + if (!initialGradeUsable) { + repairStopReason = 'initial-grading-failed'; + console.log(' repairs skipped: the initial grade did not complete, so there are no reliable findings to fix'); + } + + let progressionNext = recordProgressionGrade({ + selected: progressionSelection, + bundle, + level, + repair: { + status: !initialGradeUsable ? 'ungraded' + : resumedRepair ? (initialBundleOutcome.kind === 'passed' ? 'corrected' : 'incomplete') + : initialBundleOutcome.kind === 'passed' ? 'not-needed' : 'incomplete', + budgetRounds: progressionSelection + ? progressionRepairBudgetRounds : args.fixRounds, + roundsUsed: priorRepairRounds + (resumedRepair ? 1 : 0), + ...(!args.progression ? { stallLimitRounds: args.maxStalledRepairs } : {}), + stopReason: !initialGradeUsable ? 'initial-grading-failed' + : initialBundleOutcome.kind === 'passed' ? 'not-needed' : null, + }, + }); + const progressionMayRepair = () => !args.progression + || progressionNext?.type === 'repair'; + const recordRepairProgression = ({ failure = null }: { failure?: ProgressionFailure | null } = {}) => { + progressionNext = recordProgressionGrade({ + selected: progressionSelection, + bundle: failure ? null : bundle, + level, + failure, + repair: { + status: failure ? 'ungraded' + : classifyBundle(bundle).kind === 'passed' ? 'corrected' : 'incomplete', + budgetRounds: progressionSelection + ? progressionRepairBudgetRounds : args.fixRounds, + roundsUsed: priorRepairRounds + fixRounds, + ...(!args.progression ? { stallLimitRounds: args.maxStalledRepairs } : {}), + stopReason: null, + }, + }); + return progressionMayRepair(); + }; + + // Hand back findings and let the agent fix, until clean or out of rounds. + while (ladderMayContinue(classifyBundle(bundle)) && progressionMayRepair() + && (args.progression || fixRounds < args.fixRounds)) { + if (args.progression) { + progressionSelection = bindProgressionAction(level); + trackProgressionBudget(progressionSelection, priorRepairRounds + fixRounds); + if (progressionSelection && isProgressionWorkRecipeAction(progressionSelection) + && bundle?.selection?.sha256 !== progressionSelection.grader.selectionSha256) { + const sequence = requireProgressionState(progressionExecution?.state ?? null).attempts.length + 1; + bundle = grade(args, appDir, url, + `${args.backend}-l${level}-repair-baseline${sequence}`, level, track, runId); + const refreshOutcome = classifyBundle(bundle); + if (refreshOutcome.kind !== 'app_failure') { + const refreshFailed = refreshOutcome.kind !== 'passed'; + progressionNext = recordProgressionGrade({ + selected: progressionSelection, + bundle, + level, + repair: { + status: refreshFailed ? 'ungraded' : 'corrected', + budgetRounds: progressionRepairBudgetRounds, + roundsUsed: priorRepairRounds + fixRounds, + stopReason: refreshFailed ? 'refresh-grading-failed' : null, + }, + }); + if (refreshFailed) repairStopReason = 'refresh-grading-failed'; + if (refreshFailed || !progressionMayRepair()) break; + continue; + } + } + } + let wroteReport = true; + try { + sh('node', [join(ROOT, 'dist', 'commands', 'report-bugs.js'), '--app', appDir, + '--history-json', JSON.stringify(repairHistory), + '--archive', join(args.out, 'repair-reports', + `bug-report-l${level}-round${fixRounds + 1}.md`), + ...repairReportArgs(progressionSelection)], { stdio: 'pipe' }); + } catch (error) { + const failure = commandFailure(error); + if (failure.status === 3) wroteReport = false; // nothing failed + else if (failure.status === 4) { + wroteReport = false; + repairStopReason = 'no-actionable-findings'; + } + else throw failure; + } + if (!wroteReport) { + repairStopReason = 'no-actionable-findings'; + break; + } + + const before = bundle?.totals?.score ?? 0; + const beforeMax = bundle?.totals?.max ?? 0; + // Kept whole, not just its total: the regression check compares + // per-criterion, because totals are scored out of a denominator that + // moves between rounds. + const beforeBundle = bundle; + // A fix can break more than it mends. Keep the source that produced the + // best score so far, and roll back to it if a round regresses. + // Kept outside the results tree: a snapshot is a known-good copy of the + // answer, and a coding session that can reach one will copy it instead of + // building. It only has to survive this process. + const snapshot = join(tmpdir(), `stack-bench-snapshot-${args.backend}-${args.track}-run${args.runIndex}-l${level}`); + snapshotSource(appDir, snapshot); + clearPrivateGradingEvidence(appDir); + fixRounds += 1; + const displayedRepairBudget = args.progression + ? progressionRepairBudgetRounds + : args.fixRounds; + console.log(`--- repair round ${fixRounds}/${displayedRepairBudget} ---`); + const fix = await runAgentForLevel('fix', level); + fixCost += fix.costUsd; + fixSessions.push(runSessionRecord(fix, priorRepairRounds + fixRounds)); + + const fixFailure = agentSessionFailure(fix); + if (fixFailure) { + console.log(` coding session failed: ${fixFailure.reason}; stopping repairs`); + bundle = { outcome: fixFailure }; + repairHistory.push(repairHistoryEntry(fixRounds, beforeBundle, bundle, + 'agent session failed')); + repairStopReason = 'agent-session-failure'; + recordRepairProgression({ failure: progressionFailure(fixFailure) }); + break; + } + + // Check the round that just ran, before paying to grade it. A fix session + // that read the scenario file is not going to be redeemed by another + // round, and grading it only produces a number nobody may quote. + const fixLeak = auditContamination(appDir); + if (fixLeak) { + const buildSession = runSessionRecord(build); + const sessions = resumedRepair ? fixSessions : [buildSession, ...fixSessions]; + const sessionTotals = summarizeSessions(sessions); + abortUnusableSession(`repair round ${fixRounds}`, fixLeak, { + level, graded: false, score: null, max: null, + selection: bundle?.selection ?? null, + ...(resumedRepair + ? { resumedRepair: firstBuild } + : continuing + ? { baseline: firstBuild, resumeCostUsd: build.costUsd, resumeSession: buildSession } + : { firstBuild, buildCostUsd: build.costUsd, buildSession }), + fixCostUsd: addCostUsd(fixCost), fixSessions, fixRounds, + ...(resumedRepair ? { priorRepairRounds, + cumulativeFixRounds: priorRepairRounds + fixRounds } : {}), + repair: { status: 'ungraded', budgetRounds: displayedRepairBudget, + roundsUsed: priorRepairRounds + fixRounds, + stopReason: fixLeak.kind === 'harness_failure' ? 'audit-failure' : 'contaminated' }, + sessionTotals, + costUsd: resumedRepair ? addCostUsd(fixCost) : addCostUsd(build.costUsd, fixCost), + durationMs: Date.now() - t0, + }, progressionSelection); + } + const repairedSource = `${snapshot}-accepted`; + snapshotSource(appDir, repairedSource); + try { + bundle = await gradeAcceptedSource(repairedSource, + `${args.backend}-l${level}-fix${fixRounds}`); + } finally { + rmSync(repairedSource, { recursive: true, force: true }); + } + + const after = bundle?.totals?.score ?? 0; + const afterMax = bundle?.totals?.max ?? 0; + // Compare the SAME criteria in both rounds, not the totals. + // + // Compare criteria that were conclusive in both rounds, but never let a + // previous observation disappear: conclusive -> inconclusive is lost + // evidence and rolls the source back instead of hiding a regression. + // The declared denominator is fixed; typed evidence still matters here + // because an unmeasured check is not interchangeable with a real failure. + const decision = repairEvidenceDecision(beforeBundle, bundle); + const shared = decision.shared; + if (decision.action === 'keep-setup-repair') { + console.log(afterMax > 0 + ? ` application setup is now gradeable (${after}/${afterMax}); keeping this repair` + : ' application setup is still failing; keeping the attempted repair for the next round'); + repairHistory.push(repairHistoryEntry(fixRounds, beforeBundle, bundle, + afterMax > 0 + ? 'kept because the app became gradeable' + : 'kept to continue repairing application setup')); + if (!recordRepairProgression()) break; + if (pauseForRepeatedFindings()) break; + continue; + } + if (decision.action === 'rollback-no-comparison') { + console.log(' no criteria were conclusively scored in both rounds; rolling back this fix'); + bundle = await gradeAcceptedSource(snapshot, `${args.backend}-l${level}-rollback${fixRounds}`); + repairHistory.push(repairHistoryEntry(fixRounds, beforeBundle, bundle, + 'rolled back because the result could not be compared')); + if (!recordRepairProgression()) break; + if (pauseForRepeatedFindings()) break; + continue; + } + if (shared.points < Math.min(beforeMax, afterMax)) { + console.log(` comparing ${shared.points} point(s) across ${shared.count} criteria scored in both rounds` + + ` (${before}/${beforeMax} -> ${after}/${afterMax} overall)`); + } + if (decision.action === 'rollback-regression') { + if (shared.lostEvidence.length) { + console.log(` lost conclusive evidence for ${shared.lostEvidence.length} criterion/criteria; rolling back this fix`); + } else if (shared.definitionChanges.length) { + console.log(' rubric points changed between grades; rolling back this fix'); + } else { + console.log(` regressed (${shared.before} -> ${shared.after} on shared criteria); rolling back this fix`); + } + bundle = await gradeAcceptedSource(snapshot, `${args.backend}-l${level}-rollback${fixRounds}`); + regressed = true; + repairHistory.push(repairHistoryEntry(fixRounds, beforeBundle, bundle, + 'rolled back because earlier behavior regressed')); + if (!recordRepairProgression()) break; + if (pauseForRepeatedFindings()) break; + continue; + } + if (shared.after === shared.before) { + const remaining = displayedRepairBudget - fixRounds; + console.log(` ${formatRepairProgress(shared, { before, beforeMax, after, afterMax })}; ` + + (remaining > 0 ? `${remaining} repair round(s) remain` : 'repair budget exhausted')); + } + repairHistory.push(repairHistoryEntry(fixRounds, beforeBundle, bundle, + shared.after === shared.before ? 'kept with no score gain' : 'kept')); + if (!recordRepairProgression()) break; + if (pauseForRepeatedFindings()) break; + } + + // A grading run that crashed writes no bundle, and recording that as 0/0 + // makes a harness failure indistinguishable from an app that scored nothing + // — in a ladder run it silently drops a level's result on the floor. Say so + // instead, and leave the score null. + const finalBundleOutcome = classifyBundle(bundle); + const progressionAttempt = progressionExecution + ? requireProgressionState(progressionExecution.state).attempts.at(-1) ?? null : null; + // Progression uses stricter evidence rules than a regular scored bundle. + // Store one answer when a selected check is not measured: the raw bundle + // remains available for diagnosis, but the level is not a usable grade. + const graded = levelGradeIsUsable(finalBundleOutcome, + args.progression ? progressionAttempt : null); + const finalTotals = graded ? bundle?.totals ?? null : null; + const nodeStrikes = progressionExecution + ? dependencyStrikeRecords(requireProgressionState(progressionExecution.state), level, levelStrikeNodeIds) + : null; + const repairBudgetRounds = progressionExecution + ? Math.max(priorRepairRounds + fixRounds, progressionRepairBudgetRounds) + : args.fixRounds; + const repairBudgetExhausted = progressionExecution + ? finalBundleOutcome.kind === 'app_failure' + && progressionNext?.type !== 'repair' + : fixRounds >= args.fixRounds; + const repairStatus: RepairStatus = !graded ? 'ungraded' + : finalBundleOutcome.kind === 'passed' ? (fixRounds > 0 ? 'corrected' : 'not-needed') + : repairBudgetExhausted ? 'budget-exhausted' : 'incomplete'; + const stopReasons: Record = { + 'not-needed': 'not-needed', + corrected: 'passed', + 'budget-exhausted': 'budget-exhausted', + incomplete: null, + ungraded: null, + }; + const stopReason = repairStopReason ?? stopReasons[repairStatus]; + const repair = { + status: repairStatus, + budgetRounds: repairBudgetRounds, + roundsUsed: priorRepairRounds + fixRounds, + ...(!args.progression ? { stallLimitRounds: args.maxStalledRepairs } : {}), + stopReason, + ...(nodeStrikes ? { + strikeScope: 'feature', + nodeStrikes, + } : {}), + }; + if (continuing) { + const continuation = requireContinuation(run); + continuation.cumulativeRoundsAfter = continuation.cumulativeRoundsBefore + fixRounds; + } + let checkpoint = null; + try { + checkpoint = preserveLevelCheckpoint({ + appDir, + outputDir: args.out, + runId, + identities: run.identities, + track: args.track, + backend: args.backend, + level, + repair, + outcome: finalBundleOutcome, + selectionSha256: bundle?.selection?.sha256 ?? null, + }); + console.log(` kept the L${level} source checkpoint at ${join(args.out, checkpoint.directory)}`); + } catch (error) { + console.log(` !! could not keep the L${level} source checkpoint: ${errorMessage(error).split('\n')[0]}`); + } + if (!graded) { + console.log(` L${level}: GRADING DID NOT COMPLETE — no usable bundle. ` + + `Score is unknown, not zero; re-grade this level before using the run.`); + } + const buildSession = runSessionRecord(build); + const sessionTotals = summarizeSessions(resumedRepair ? fixSessions + : [buildSession, ...fixSessions]); + run.levels.push({ + level, + graded, + score: finalTotals?.score ?? null, + max: finalTotals?.max ?? null, + // Whether the guarantees earned at earlier levels still hold at this one — + // the whole point of growing the app level by level. It reached the + // console and the bundle but not run.json, so the thesis metric was + // missing from the durable record. + regression: bundle?.totals?.regression ?? null, + selection: bundle?.selection ?? null, + ...(resumedRepair + ? { resumedRepair: firstBuild } + : continuing + ? { baseline: firstBuild, resumeCostUsd: build.costUsd, resumeSession: buildSession } + : { firstBuild, buildCostUsd: build.costUsd, buildSession }), + contractPass: bundle?.totals?.contractPass ?? null, + code: bundle?.code ?? null, + fixCostUsd: addCostUsd(fixCost), + fixSessions, + repairHistory, + sessionTotals, + tokens: sessionTotals.tokens, + // Carried up so a run summary can explain a cost, not just report one. + usage: sessionTotals.usage, + turns: sessionTotals.turns, + promptBytes: sessionTotals.promptBytes, + tokensPerTurn: sessionTotals.turns + ? Math.round(sessionTotals.tokens / sessionTotals.turns) : null, + // Reasoning actually produced. The budget is deliberately unpinned so runs + // measure what a customer gets; that is only defensible if a shift in the + // CLI default is visible afterwards rather than silently absorbed into + // every score. + thinking: sessionTotals.thinking, + fixRounds, + ...(resumedRepair ? { priorRepairRounds, + cumulativeFixRounds: priorRepairRounds + fixRounds } : {}), + repair, + checkpoint, + // Keep the summary flag derived from the typed status so the two cannot drift. + stalled: repairStatus === 'budget-exhausted' || repairStopReason === 'repeated-findings', + regressed, + outcome: finalBundleOutcome, + durationSec: Math.round((Date.now() - t0) / 1000), + }); + if (!args.progression || requireProgressionState(progressionExecution?.state ?? null).attempts + .some(attempt => attempt.level === level && attempt.outcome === 'conclusive')) { + run.validation.ladder.completedLevels.push(level); + } + writeRunJson(join(args.out, 'run.json'), run); + const blockedLevels = args.levelList.filter(candidate => candidate > level); + if (args.progression) { + const progressionState = requireProgressionState(progressionExecution?.state ?? null); + if (progressionState.phase === 'terminal') { + if (blockedLevels.length) run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = blockedLevels; + writeRunJson(join(args.out, 'run.json'), run); + break; + } + if (progressionState.level <= level) { + if (progressionState.attempts.at(-1)?.outcome === 'inconclusive') { + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = [level, ...blockedLevels]; + writeRunJson(join(args.out, 'run.json'), run); + break; + } + throw new Error(`dependency progression did not leave L${level} after its strike budget`); + } + continue; + } + if (blockedLevels.length && !ladderMayAdvance(finalBundleOutcome)) { + run.validation.ladder.stoppedAfterLevel = level; + run.validation.ladder.blockedLevels = blockedLevels; + writeRunJson(join(args.out, 'run.json'), run); + console.log(` ladder paused after L${level}: L${level} must pass before ` + + `${blockedLevels.map(candidate => `L${candidate}`).join(', ')} can start`); + console.log(' inspect the failures, then explicitly grant more repair rounds or correct the benchmark'); + break; + } + } + + if (args.mutations) { + console.log(`\n================ ${args.backend} mutation control ================`); + const pristineOutcome = aggregateRunOutcome(run.levels); + if (args.referenceMutationOnly || mutationControlEligible(pristineOutcome)) { + args.parentAttemptId = runId; + const baselineBundle = pristineMutationBaselinePath(args); + if (baselineBundle) args.mutationBaselineBundle = baselineBundle; + else delete args.mutationBaselineBundle; + run.mutationControl = runMutationControl(args, appDir, url, track, + run.setup?.isolation?.imageId ?? null); + } else { + console.log(` skipped: pristine outcome is ${pristineOutcome.kind}`); + run.mutationControl = { ok: false, skipped: true, + outcome: { kind: pristineOutcome.kind, phase: 'mutation-control-prerequisite', + reason: `pristine outcome is ${pristineOutcome.kind}` } }; + } + writeRunJson(join(args.out, 'run.json'), run); + } + + // Record a final transcript audit in addition to the per-session hard gates. + // The same retry and diagnostic path is used at both gates. + let finalAuditFailure = null; + const finalAudit = auditContamination(appDir); + if (!finalAudit) { + run.contaminated = false; + run.contamination = { evidence: 'no reads of the grader, contracts, prompts or notes', + verdict: 'scores usable' }; + } else if (finalAudit.kind === 'contaminated') { + run.contaminated = true; + run.contamination = { evidence: finalAudit.evidence, verdict: finalAudit.verdict }; + console.log('\n !! CONTAMINATED: this build read the harness that grades it:'); + for (const evidence of finalAudit.evidence) console.log(` ${evidence}`); + console.log(' Scores from this run must not be quoted.'); + } else { + run.contaminated = false; + run.contamination = { evidence: finalAudit.evidence, verdict: finalAudit.verdict }; + const reason = finalAudit.evidence.join('; '); + finalAuditFailure = { kind: 'harness_failure', phase: 'contamination-audit', reason, + appFailures: [], inconclusive: [], harnessFailures: [reason] }; + console.log('\n !! AUDIT DID NOT COMPLETE. Scores from this run must not be quoted.'); + } + + // Keep the transcript evidence outside the provider CLI's prunable store. + try { + sh('node', [join(ROOT, 'dist', 'commands', 'archive-transcripts.js'), '--app', appDir, '--label', artifactLabel], + { stdio: 'pipe' }); + } catch { console.log(' (transcript archiving failed — evidence is on a 30-day timer)'); } + + run.outcome = finalAuditFailure ?? (args.referenceMutationOnly && run.mutationControl?.ok + ? { kind: 'passed', phase: 'mutation-control', reason: null, + appFailures: [], inconclusive: [], harnessFailures: [] } + : aggregateRunOutcome(run.levels)); + if (args.mutations && !run.mutationControl?.ok && !run.mutationControl?.skipped) { + run.outcome = { kind: run.mutationControl?.outcome?.kind === 'incomplete' + ? 'incomplete' : 'harness_failure', phase: 'mutation-control', + reason: run.mutationControl?.outcome?.reason + ?? run.mutationControl?.processError + ?? 'one or more declared mutations were not cleanly caught', + appFailures: [], inconclusive: [] }; + } + + if (run.levels.some(level => level.graded === true)) { + try { + preserveFinalPackageEvidence({ appDir, outputDir }); + console.log(` source kept at ${join(outputDir, 'source')}`); + console.log(` grading detail kept at ${join(outputDir, 'grading')}`); + } catch (error) { + const reason = errorMessage(error).split(/\r?\n/)[0] ?? 'evidence preservation failed'; + run.outcome = { kind: 'harness_failure', phase: 'evidence-preservation', reason, + appFailures: [], inconclusive: [], harnessFailures: [reason] }; + console.log(` !! ${reason}`); + } + } + + finalizeRunTotals(run, started, { costComplete: runCostComplete }); + if (args.repairGrant) { + const continuation = requireContinuation(run); + const totals = requireRunTotals(run); + continuation.cumulativeCostAfterUsd = addCostUsd(continuation.cumulativeCostBeforeUsd, totals.costUsd); + continuation.cumulativeDurationAfterSec = continuation.cumulativeDurationBeforeSec + totals.durationSec; + } + run.completedAt = new Date().toISOString(); + writeRunJson(join(args.out, 'run.json'), run); + + // Produce a model-free friction report from the transcript when available. + if (executeStackCapability(stackAdapter, 'run-policy', 'product-review-enabled') + && run.setup?.session !== 'model-free-reference') { + try { + sh('node', [join(ROOT, 'dist', 'commands', 'stdb-report.js'), '--label', artifactLabel, '--track', args.track, + '--level', String(args.levelList[args.levelList.length - 1]), + '--score', `${requireRunTotals(run).score}/${requireRunTotals(run).max}`, + '--cost', String(requireRunTotals(run).costUsd), + '--fix-rounds', String(requireRunTotals(run).fixRounds), + ...(run.contaminated ? ['--contaminated'] : [])], { stdio: 'inherit' }); + } catch (e) { + console.log(` (stdb friction report failed: ${errorMessage(e).split('\n')[0]})`); + } + // The deeper behavioural review is a separate model session. It is useful + // product research, but running it implicitly would add unmetered provider + // usage to a benchmark attempt and make campaign cost accounting false. + // Keep the model-free friction report above automatic; make this analysis + // explicit and never run it for an incomplete attempt. + if (args.behavioralReview + && !['provider_failure', 'harness_failure', 'ungraded'].includes(run.outcome?.kind)) { + try { + sh('node', [join(ROOT, 'dist', 'commands', 'stdb-review.js'), '--label', artifactLabel, + '--source', join(outputDir, 'source'), + '--compare', stringArray(executeStackCapability(stackAdapter, + 'run-policy', 'product-review-comparisons'), 'product-review-comparisons') + .map(backend => resultsName(track, backend, args.runIndex)).join(',')], { stdio: 'inherit' }); + } catch (e) { + console.log(` (stdb behavioural review failed: ${errorMessage(e).split('\n')[0]})`); + } + } + } + + console.log(`\n================ ${args.backend} summary ================`); + for (const l of run.levels) { + console.log(` ${formatLevelSummary(l)}`); + } + const totals = requireRunTotals(run); + console.log(` TOTAL ${totals.score}/${totals.max} ` + + `$${totals.costUsd} ${totals.fixRounds} repair round(s) ${totals.durationSec}s`); + console.log(` ${join(outputDir, 'run.json')}`); + + teardown(); + + // Leave nothing in temp. Best-effort: a directory some process still holds is + // not worth failing a finished run over, and the next run makes its own + // anyway. Say so rather than leaving it to be discovered. Only for a + // directory THIS run created — an explicit --app is the caller's. + if (ownWorkDir) { + try { + rmSync(dirname(appDir), { recursive: true, force: true }); + } catch { + console.log(` (work dir still held: ${dirname(appDir)} — the next sweep will take it)`); + } + } + process.exitCode = runExitCode(run.outcome); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch(error => { + console.error(error instanceof Error ? error.stack ?? error.message : errorMessage(error)); + try { emergencyTeardown?.(); } + catch (cleanupError) { + console.error(`cleanup after failure also failed: ${errorMessage(cleanupError).split(/\r?\n/)[0]}`); + } + process.exitCode = 1; + }); +} diff --git a/tools/stack-bench/commands/campaign-cli.ts b/tools/stack-bench/commands/campaign-cli.ts new file mode 100644 index 00000000000..9c62155dc98 --- /dev/null +++ b/tools/stack-bench/commands/campaign-cli.ts @@ -0,0 +1,273 @@ +#!/usr/bin/env node + +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { compileCampaignFile } from '../src/campaigns/campaign-compiler.js'; +import { CAMPAIGN_MODE_REGISTRY } from '../src/campaigns/campaign-mode.js'; +import { executeCampaign, inspectCampaign, prepareCampaign, reconcileCampaign } + from '../src/campaigns/campaign-runner.js'; +import { inspectCampaignSummary } from '../src/campaigns/campaign-inspection.js'; +import { generateCampaignReport } from '../src/campaigns/campaign-report.js'; +import { grantCampaignDependencyStrikes } + from '../src/campaigns/campaign-progression-grant.js'; +import { auditProgressionReferenceCampaign, formatProgressionReferenceCampaignAudit } + from '../src/campaigns/progression-reference-campaign-audit.js'; +import type { ReferenceCampaignAudit } + from '../src/campaigns/progression-reference-campaign-audit.js'; + +interface CampaignSummaryPlan { + id: string; + version: string; + contentSha256: string; +} + +interface CampaignSummaryState { + status: string; + summary: unknown; + attempts: Array<{ + plan: { id: string }; + status: string; + executions: Array<{ + id: string; + outcome: unknown; + reason: string | null; + }>; + }>; +} + +interface ReferenceCampaignPlan { + attempts: Array<{ + mode?: { id?: string }; + agentAdapter?: string; + }>; +} + +interface ReferenceCampaignState { + status: string; +} + +interface ResumeCampaign { + plan: { + contentSha256: string; + definition: { mode?: { id?: string } }; + }; + state: { + status: string; + attempts: Array<{ executions: readonly unknown[] }>; + }; +} + +type ReferenceCampaignAuditFunction = (directory: string) => ReferenceCampaignAudit | null; + +export type CampaignArgs = + | { command: 'modes' } + | { command: 'validate'; path: string } + | { command: 'show'; path: string } + | { command: 'status'; directory: string; full: boolean } + | { command: 'inspect'; directory: string } + | { command: 'report'; directory: string } + | { command: 'audit'; directory: string } + | { command: 'grant-strikes'; directory: string; attemptId: string; grantId: string; + level: number; nodeIds: string[]; strikes: number } + | { command: 'prepare'; path: string; directory: string } + | { command: 'trial'; path: string; directory: string } + | { command: 'run'; path: string; directory: string } + | { command: 'resume'; path: string; directory: string } + | { command: 'reconcile'; path: string; directory: string }; + +function isOneOf(value: string | undefined, + values: readonly T[]): value is T { + return value !== undefined && values.some(candidate => candidate === value); +} + +export function campaignStateSummary(plan: CampaignSummaryPlan, state: CampaignSummaryState) { + const failures = state.attempts.flatMap(attempt => { + const execution = attempt.executions.at(-1); + if (!execution || execution.outcome === null || execution.outcome === 'passed') return []; + return [{ + attempt: attempt.plan.id, + status: attempt.status, + execution: execution.id, + outcome: execution.outcome, + reason: execution.reason, + }]; + }); + return { + campaign: { id: plan.id, version: plan.version, sha256: plan.contentSha256 }, + status: state.status, + summary: state.summary, + failures, + }; +} + +export function auditCompletedReferenceCampaign(directory: string, plan: ReferenceCampaignPlan, + state: ReferenceCampaignState, { + audit = auditProgressionReferenceCampaign, +}: { audit?: ReferenceCampaignAuditFunction } = {}): ReferenceCampaignAudit | null { + const hasReferenceProgression = plan.attempts.some(attempt => + attempt.mode?.id === 'dependency' && attempt.agentAdapter === 'reference-fixture'); + return state.status === 'completed' && hasReferenceProgression ? audit(directory) : null; +} + +export function validateResumeCampaignState( + requested: { contentSha256: string }, existing: T): T { + if (requested.contentSha256 !== existing.plan.contentSha256) { + throw new Error('resume requires the exact campaign plan already stored in the output directory'); + } + if (existing.plan.definition.mode?.id !== 'dependency') { + throw new Error('resume is available only for dependency campaigns'); + } + const executions = existing.state.attempts.reduce((total, attempt) => + total + attempt.executions.length, 0); + if (existing.state.status !== 'prepared' || executions < 1) { + throw new Error('resume requires a dependency campaign with scheduled work'); + } + return existing; +} + +export function validateResumeCampaign(path: string, directory: string): ResumeCampaign { + return validateResumeCampaignState(compileCampaignFile(path), inspectCampaign(directory)); +} + +export function parseCampaignArgs(argv: string[]): CampaignArgs { + const [command, path, ...rest] = argv.slice(2); + if (command === 'modes' && path === undefined) return { command }; + if (isOneOf(command, ['validate', 'show']) && path && rest.length === 0) { + return { command, path: resolve(path) }; + } + if (command === 'status' && path + && (rest.length === 0 || (rest.length === 1 && rest[0] === '--full'))) { + return { command, directory: resolve(path), full: rest.length === 1 }; + } + if (isOneOf(command, ['inspect', 'report', 'audit']) && path && rest.length === 0) { + return { command, directory: resolve(path) }; + } + if (command === 'grant-strikes' && path) { + const values: { attemptId?: string; grantId?: string; level?: number; strikes?: number; + nodeIds: string[] } = { nodeIds: [] }; + const seen = new Set(); + for (let index = 0; index < rest.length; index += 2) { + const flag = rest[index]; + const value = rest[index + 1]; + if (flag === undefined || value === undefined + || !['--attempt', '--grant-id', '--level', '--feature', '--strikes'].includes(flag) + || (flag !== '--feature' && seen.has(flag))) { + throw new Error(`invalid or duplicate grant-strikes option ${String(flag)}`); + } + seen.add(flag); + if (flag === '--attempt') values.attemptId = value; + else if (flag === '--grant-id') values.grantId = value; + else if (flag === '--level') values.level = Number(value); + else if (flag === '--strikes') values.strikes = Number(value); + else values.nodeIds.push(value); + } + if (!values.attemptId || !values.grantId || typeof values.level !== 'number' + || !Number.isSafeInteger(values.level) || typeof values.strikes !== 'number' + || !Number.isSafeInteger(values.strikes) || values.nodeIds.length === 0) { + throw new Error('grant-strikes requires --attempt, --grant-id, --level, ' + + 'one or more --feature values, and --strikes'); + } + return { command, directory: resolve(path), attemptId: values.attemptId, + grantId: values.grantId, level: values.level, nodeIds: values.nodeIds, + strikes: values.strikes }; + } + if (isOneOf(command, ['prepare', 'trial', 'run', 'resume', 'reconcile']) + && path && rest.length === 2 && rest[0] === '--out') { + return { command, path: resolve(path), directory: resolve(rest[1]!) }; + } + throw new Error('usage: campaign-cli.js modes | validate|show ' + + '| prepare|trial|run|resume|reconcile --out ' + + '| status [--full] | inspect|report|audit ' + + '| grant-strikes --attempt --grant-id --level ' + + '--feature [--feature ...] --strikes '); +} + +async function main() { + const args = parseCampaignArgs(process.argv); + if (args.command === 'modes') { + console.log(JSON.stringify(CAMPAIGN_MODE_REGISTRY.ids.map(value => { + const [id, version] = value.split('@'); + return { id, version }; + }), null, 2)); + return; + } + if (args.command === 'status') { + const campaign = inspectCampaign(args.directory, { requireCurrentInputs: false }); + console.log(JSON.stringify(args.full + ? campaign.state + : campaignStateSummary(campaign.plan, campaign.state), null, 2)); + return; + } + if (args.command === 'inspect') { + console.log(JSON.stringify(inspectCampaignSummary(args.directory), null, 2)); + return; + } + if (args.command === 'report') { + const generated = generateCampaignReport(args.directory); + console.log(`${generated.reportPath}\n${generated.htmlPath}\n${generated.report.contentSha256}`); + return; + } + if (args.command === 'audit') { + const report = auditProgressionReferenceCampaign(args.directory); + if (report === null) throw new Error('campaign has no dependency reference attempts to audit'); + console.log(formatProgressionReferenceCampaignAudit(report)); + if (!report.ok) process.exitCode = 1; + return; + } + if (args.command === 'grant-strikes') { + console.log(JSON.stringify(grantCampaignDependencyStrikes(args.directory, { + attemptId: args.attemptId, + grantId: args.grantId, + level: args.level, + nodeIds: args.nodeIds, + strikes: args.strikes, + }), null, 2)); + return; + } + if (args.command === 'prepare') { + const prepared = prepareCampaign(args.path, args.directory); + console.log(JSON.stringify(campaignStateSummary(prepared.plan, prepared.state), null, 2)); + return; + } + const plan = compileCampaignFile(args.path); + if (args.command === 'reconcile') { + const state = reconcileCampaign(args.path, args.directory); + console.log(JSON.stringify(campaignStateSummary(plan, state), null, 2)); + return; + } + if (args.command === 'trial' || args.command === 'run' || args.command === 'resume') { + if (args.command === 'resume') validateResumeCampaign(args.path, args.directory); + const cancellation = new AbortController(); + const cancel = () => cancellation.abort(); + process.on('SIGINT', cancel); + process.on('SIGTERM', cancel); + let state; + try { + const executionMode = args.command === 'trial' + || (args.command === 'resume' && plan.state === 'draft') + ? 'model-free-trial' : 'frozen'; + state = await executeCampaign(args.path, args.directory, { + mode: executionMode, + signal: cancellation.signal, + }); + } finally { + process.off('SIGINT', cancel); + process.off('SIGTERM', cancel); + } + console.log(JSON.stringify(campaignStateSummary(plan, state), null, 2)); + const audit = auditCompletedReferenceCampaign(args.directory, plan, state); + if (audit !== null) console.log(formatProgressionReferenceCampaignAudit(audit)); + if (state.status !== 'completed' || audit?.ok === false) process.exitCode = 1; + return; + } + if (args.command === 'show') console.log(JSON.stringify(plan, null, 2)); + else console.log(`${plan.id}@${plan.version} ${plan.state}: ${plan.summary.attempts} attempts, ${plan.contentSha256}`); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/commands/check-actions.ts b/tools/stack-bench/commands/check-actions.ts new file mode 100644 index 00000000000..be67f8efdf2 --- /dev/null +++ b/tools/stack-bench/commands/check-actions.ts @@ -0,0 +1,142 @@ +#!/usr/bin/env node +// Report which named write actions answer. Exact recipe grading owns pass/fail. +// +// Presence, not behaviour: a 404 means the action is not there. Anything else — +// including a refusal — means it exists and answered, which is all this checks. +// Whether it refuses the RIGHT things is what the invariant suites are for. +// +// Nothing here is allowed to mutate the database: every probe is unauthenticated +// or deliberately malformed, so a working app rejects it. +// +// Usage: +// node dist/commands/check-actions.js --backend spacetime --app [--out report.json] +// node dist/commands/check-actions.js --backend postgres --url http://localhost:6573 + +import { readFileSync } from 'node:fs'; +import { join } from 'node:path'; + +import { emptyArtifactIdentities, writeArtifact } from '../src/evidence/artifacts.js'; +import { executeStackCapability } from '../src/stacks/stack-adapter-contract.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +const TRACKS = join(STACK_BENCH_ROOT, 'tracks'); + +interface CheckActionsArgs { + backend?: string; + url?: string; + app?: string; + out?: string; + track?: string; + quiet?: boolean; + parentAttemptId?: string; +} + +interface NamedAction { + id: string; + path: string; + args?: unknown; +} + +interface NamedActionRequest { + url?: string; + body?: BodyInit | null; + missingNote: string; +} + +interface ActionResult { + id: string; + ok: boolean; + status: number; + note: string; +} + +function parseArgs(argv: string[]): CheckActionsArgs { + const a: CheckActionsArgs = {}; + for (let i = 2; i < argv.length; i++) { + const k = argv[i]; + if (k === '--backend') a.backend = argv[++i] ?? ''; + else if (k === '--url') a.url = argv[++i] ?? ''; + else if (k === '--app') a.app = argv[++i] ?? ''; + else if (k === '--out') a.out = argv[++i] ?? ''; + else if (k === '--track') a.track = argv[++i] ?? ''; + else if (k === '--quiet') a.quiet = true; + else if (k === '--parent-attempt-id') a.parentAttemptId = argv[++i] ?? ''; + else { console.error(`Unknown arg ${k}`); process.exit(2); } + } + if (!a.backend) { console.error('--backend is required'); process.exit(2); } + return a; +} + +const args = parseArgs(process.argv); +const backend = args.backend; +if (!backend) throw new Error('--backend is required'); + +// The actions come from the track, not from this file: a track that names none +// (chat does not) is skipped rather than reported as six missing endpoints. +// Probes are chosen to be refused by a correct app — no credentials, or +// arguments that cannot identify a real row — so a check never writes anything. +const track = args.track ? JSON.parse(readFileSync(join(TRACKS, args.track, 'track.json'), 'utf8')) as { + actions?: NamedAction[]; +} : null; +const ACTIONS = (track?.actions ?? []).map(action => ({ ...action, http: { method: 'POST', path: action.path } })); +if (!ACTIONS.length) { + if (!args.quiet) console.log(` no named actions declared for track "${args.track ?? '(none)'}" — nothing to check`); + if (args.out) { + const id = `${args.parentAttemptId ?? 'actions'}-action-check`; + writeArtifact(args.out, { kind: 'action_check', id, + attempt: { id, parentId: args.parentAttemptId ?? null }, + identities: emptyArtifactIdentities({ stackAdapter: { id: backend } }), + payload: { backend, results: [], missing: [] } }); + } + process.exit(0); +} + +// SpacetimeDB control targets come from the authenticated lease. Client config +// is app-controlled input and may use environment expressions rather than +// literals; it is neither authoritative nor safe for harness operations. +const adapter = STACK_ADAPTER_REGISTRY.get(backend); +const spacetime = executeStackCapability(adapter, 'grading', 'context', + { requireBuildContainer: false }); + +async function probe(action: NamedAction): Promise> { + try { + const request = executeStackCapability(adapter, 'named-action', 'request', + { action, input: { args: action.args }, spacetime, url: args.url }) as NamedActionRequest; + if (!request.url) return { ok: false, status: 0, note: 'no --url given for a server-based backend' }; + const r = await fetch(request.url, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: request.body, + }); + return { ok: r.status !== 404, status: r.status, + note: r.status === 404 ? request.missingNote : '' }; + } catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + return { ok: false, status: 0, note: (message.split('\n')[0] ?? '').slice(0, 90) }; + } +} + +const results: ActionResult[] = await Promise.all(ACTIONS.map(async action => ({ + id: action.id, + ...(await probe(action)), +}))); + +const missing = results.filter(r => !r.ok); +if (!args.quiet) { + for (const r of results) { + console.log(` ${r.ok ? 'present' : 'MISSING'} ${r.id.padEnd(11)} ${r.status ? `HTTP ${r.status}` : ''} ${r.note}`); + } + console.log(missing.length + ? `\n${missing.length} named action(s) missing — contention and volume tests cannot be issued against this app.` + : '\nall named actions answer.'); +} +if (args.out) { + const id = `${args.parentAttemptId ?? 'actions'}-action-check`; + writeArtifact(args.out, { kind: 'action_check', id, + attempt: { id, parentId: args.parentAttemptId ?? null }, + identities: emptyArtifactIdentities({ stackAdapter: { id: backend } }), + payload: { backend, results, missing: missing.map(m => m.id) } }); +} +process.exit(missing.length ? 1 : 0); diff --git a/tools/stack-bench/commands/check-calibration.ts b/tools/stack-bench/commands/check-calibration.ts new file mode 100644 index 00000000000..4635f283caf --- /dev/null +++ b/tools/stack-bench/commands/check-calibration.ts @@ -0,0 +1,69 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +import { compileCalibrationDefinition, compileCalibrationFile } from '../src/composition/calibration-compiler.js'; +import { buildRecipeRelease } from '../src/composition/recipe-release.js'; +import { listTracks, TRACKS_DIR } from '../src/composition/tracks.js'; + +import { STACK_BENCH_ROOT as ROOT } from '../src/package-root.js'; + +export interface CalibrationCheckResult { + track: string; + id: string; + version: string; + state: string; + recipe: string; + controls: number; + stacks: number; + contentSha256: string; +} + +export function checkCalibrations( + { trackName = null }: { trackName?: string | null } = {}, +): CalibrationCheckResult[] { + const tracks = trackName ? [trackName] : listTracks({ includeInternal: true }); + const results: CalibrationCheckResult[] = []; + for (const name of tracks) { + const trackRoot = join(TRACKS_DIR, name); + const directory = join(trackRoot, 'composition', 'calibrations'); + if (!existsSync(directory)) continue; + for (const file of readdirSync(directory).filter(candidate => candidate.endsWith('.json')).sort()) { + const path = join(directory, file); + const source = `composition/calibrations/${file}`; + const input = JSON.parse(readFileSync(path, 'utf8')); + const definition = compileCalibrationDefinition(input, { source }); + const recipePath = resolve(dirname(path), definition.recipe.path); + const release = buildRecipeRelease(recipePath, { trackRoot }); + const plan = compileCalibrationFile(path, { trackRoot, stackBenchRoot: ROOT, release }); + results.push({ track: name, id: plan.id, version: plan.version, state: plan.state, + recipe: plan.recipe.id, controls: plan.controls.length, stacks: plan.qualification.stacks.length, + contentSha256: plan.contentSha256 }); + } + } + return results; +} + +async function main() { + const trackIndex = process.argv.indexOf('--track'); + const trackName = trackIndex >= 0 ? process.argv[trackIndex + 1] ?? null : null; + const unknown = process.argv.slice(2).filter((value, index, args) => + value !== '--track' && args[index - 1] !== '--track'); + if (unknown.length || (trackIndex >= 0 && !trackName)) { + throw new Error('usage: node dist/commands/check-calibration.js [--track ]'); + } + const results = checkCalibrations({ trackName }); + for (const result of results) { + console.log(`${result.track}: ${result.id}@${result.version} ${result.state}; ` + + `${result.controls} controls, ${result.stacks} stacks, ${result.contentSha256.slice(0, 12)}`); + } +} + +if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.stack ?? error.message : String(error)); + process.exitCode = 1; + }); +} diff --git a/tools/stack-bench/commands/check-composition.ts b/tools/stack-bench/commands/check-composition.ts new file mode 100644 index 00000000000..4548a0730d5 --- /dev/null +++ b/tools/stack-bench/commands/check-composition.ts @@ -0,0 +1,87 @@ +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { + compileFixtureDefinition, + compilePackDefinition, + compilePromotionFile, + compileRecipeFile, +} from '../src/composition/composition-compiler.js'; +import { TRACKS_DIR, listTracks } from '../src/composition/tracks.js'; + +function json(path: string): unknown { + try { return JSON.parse(readFileSync(path, 'utf8')); } + catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`cannot read composition source ${path}: ${message}`, { cause: error }); + } +} + +export interface CompositionSummary { + track: string; + packs: number; + fixtures: number; + recipes: number; + checks: number; + aliases: number; +} + +export function checkCompositions( + { trackName = null }: { trackName?: string | null } = {}, +): CompositionSummary[] { + const names = trackName ? [trackName] : listTracks({ includeInternal: true }); + const summary = []; + for (const name of names) { + const trackRoot = join(TRACKS_DIR, name); + const root = join(trackRoot, 'composition'); + if (!existsSync(root)) { + if (trackName) throw new Error(`track ${name} has no composition directory`); + continue; + } + const packs = join(root, 'packs'); + const fixtures = join(root, 'fixtures'); + const recipes = join(root, 'recipes'); + const packFiles = readdirSync(packs).filter(file => file.endsWith('.json')).sort(); + const fixtureFiles = readdirSync(fixtures).filter(file => file.endsWith('.json')).sort(); + const recipeFiles = readdirSync(recipes).filter(file => file.endsWith('.json')).sort(); + if (!packFiles.length || !fixtureFiles.length || !recipeFiles.length) { + throw new Error(`track ${name} composition must contain packs, fixtures, and recipes`); + } + for (const file of packFiles) { + const path = join(packs, file); + compilePackDefinition(json(path), { source: path }); + } + for (const file of fixtureFiles) { + const path = join(fixtures, file); + compileFixtureDefinition(json(path), { source: path }); + } + const plans = recipeFiles.map(file => compileRecipeFile(join(recipes, file), { trackRoot })); + const promotionPath = join(root, 'promotions.json'); + const promotion = existsSync(promotionPath) + ? compilePromotionFile(promotionPath, { trackRoot }) : null; + summary.push({ track: name, packs: packFiles.length, fixtures: fixtureFiles.length, + recipes: plans.length, checks: plans.reduce((total, plan) => total + plan.checks.length, 0), + aliases: promotion?.entries.length ?? 0 }); + } + return summary; +} + +function main(): void { + const args = process.argv.slice(2); + let trackName: string | null = null; + for (let index = 0; index < args.length; index += 1) { + const value = args[index + 1]; + if (args[index] === '--track' && value) { + trackName = value; + index += 1; + } else throw new Error(`unknown or incomplete argument ${args[index]}`); + } + const summary = checkCompositions({ trackName }); + if (!summary.length) throw new Error('no composition sources found'); + for (const row of summary) { + console.log(`${row.track}: ${row.packs} packs, ${row.fixtures} fixtures, ${row.recipes} recipes, ${row.checks} selected checks, ${row.aliases} promotion entries`); + } +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/check-mutations.ts b/tools/stack-bench/commands/check-mutations.ts new file mode 100644 index 00000000000..56edb2f58e2 --- /dev/null +++ b/tools/stack-bench/commands/check-mutations.ts @@ -0,0 +1,92 @@ +#!/usr/bin/env node +// Verify that a mutation file can apply to its reference app without grading. + +import { existsSync, readFileSync } from 'node:fs'; + +import { mutationFileEdits, resolveMutationFile, validateMutationDefinitions } + from '../src/evidence/mutation-analysis.js'; +import type { MutationDefinition } from '../src/evidence/mutation-analysis.js'; + +interface CliArgs { + app: string; + mutations: string; + quiet: boolean; +} + +interface MutationSpec { + anchoredTo?: unknown; + mutations?: MutationDefinition[]; +} + +function parseArgs(argv: string[]): CliArgs { + let app: string | undefined; + let mutations: string | undefined; + let quiet = false; + for (let index = 2; index < argv.length; index += 1) { + const value = argv[index]; + if (value === '--app') app = argv[++index]; + else if (value === '--mutations') mutations = argv[++index]; + else if (value === '--quiet') quiet = true; + else { + console.error(`Unknown arg ${String(value)}`); + process.exit(2); + } + } + if (!app || !mutations) { + console.error('Usage: node dist/commands/check-mutations.js --app --mutations '); + process.exit(2); + } + return { app, mutations, quiet }; +} + +const args = parseArgs(process.argv); +const parsed: unknown = JSON.parse(readFileSync(args.mutations, 'utf8')); +if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw new Error('mutation manifest must be an object'); +} +const spec = parsed as MutationSpec; +const say = (...message: unknown[]): void => { if (!args.quiet) console.log(...message); }; + +say(`mutations : ${args.mutations}`); +say(`app : ${args.app}`); +if (spec.anchoredTo) say(`anchored : ${String(spec.anchoredTo).split('.')[0]}`); +say(''); + +let bad = 0; +const definitions = validateMutationDefinitions(spec.mutations); +for (const issue of definitions.issues) { + console.log(` BAD MANIFEST ${issue.mutation ?? ''} -> ${issue.kind}`); + bad += 1; +} +for (const mutation of spec.mutations ?? []) { + const mutationId = String(mutation.id ?? ''); + for (const edit of mutationFileEdits(mutation)) { + let file: string; + try { file = resolveMutationFile(args.app, edit.file); } + catch { + console.log(` UNSAFE FILE ${mutationId} -> ${edit.file} escapes the app directory`); + bad += 1; + continue; + } + if (!existsSync(file)) { + console.log(` DEAD FILE ${mutationId} -> ${edit.file} does not exist in this app`); + bad += 1; + continue; + } + const source = readFileSync(file, 'utf8'); + const matches = source.split(edit.find).length - 1; + if (matches === 1) { + say(` ok ${mutationId} -> ${edit.file}`); + continue; + } + console.log(matches === 0 + ? ` DEAD ANCHOR ${mutationId} -> not found in ${edit.file}` + : ` AMBIGUOUS ${mutationId} -> matches ${matches}x in ${edit.file}; the edit would land in more than one place`); + bad += 1; + } +} + +console.log(bad + ? `\n${bad} problem(s) — these mutations cannot validate anything against this app.` + : '\nall anchors present and unique — this file can validate against this app.'); +process.exit(bad ? 1 : 0); diff --git a/tools/stack-bench/commands/check-scenarios.ts b/tools/stack-bench/commands/check-scenarios.ts new file mode 100644 index 00000000000..cc5e198942b --- /dev/null +++ b/tools/stack-bench/commands/check-scenarios.ts @@ -0,0 +1,309 @@ +#!/usr/bin/env node +// Check scenario action names, actors, UI hooks, and score totals without an app. + +import { existsSync, readFileSync, readdirSync } from 'node:fs'; +import { join } from 'node:path'; + +import { ACTION_REGISTRY } from '../src/actions/action-catalog.js'; +import { compileRecipeFile, type CompiledOwnedTaskFragment, type CompiledRecipeRelease } + from '../src/composition/composition-compiler.js'; +import { compileScenarioDefinition, type CompiledStep } + from '../src/composition/definition-compiler.js'; +import { DEFAULT_TRACK, listTracks, loadTrack, type Track } + from '../src/composition/tracks.js'; + +interface ScenarioScope { + features: Map>; + contractOwners: Set; + requirementOwners: Set; + contractText: string; + requirementText: string; +} + +interface RecipeSource { + baseRecipe: string | null; + isolatesSelectedSources: boolean; +} + +type HooksByLevel = Map>; + +function isRecord(value: unknown): value is Record { + return typeof value === 'object' && value !== null && !Array.isArray(value); +} + +function readJson(path: string): unknown { + return JSON.parse(readFileSync(path, 'utf8')) as unknown; +} + +function optionValue(args: readonly string[], option: string): string | null { + const index = args.indexOf(option); + if (index === -1) return null; + const value = args[index + 1]; + if (value === undefined || value.startsWith('--')) { + throw new Error(`${option} requires a value`); + } + return value; +} + +function readRecipeSource(path: string): RecipeSource { + const source = readJson(path); + if (!isRecord(source)) throw new Error(`${path}: recipe must be an object`); + const task = source.task; + if (!isRecord(task)) throw new Error(`${path}: recipe task must be an object`); + const baseRecipe = task.baseRecipe; + let baseRecipePath: string | null = null; + if (baseRecipe !== undefined) { + if (!isRecord(baseRecipe) || typeof baseRecipe.path !== 'string') { + throw new Error(`${path}: task.baseRecipe.path must be a string`); + } + baseRecipePath = baseRecipe.path; + } + return { + baseRecipe: baseRecipePath, + isolatesSelectedSources: source.execution === 'all-selected-sources', + }; +} + +function contractHookIds(path: string): string[] { + const contract = readJson(path); + if (!isRecord(contract) || !Array.isArray(contract.hooks)) { + throw new Error(`${path}: hooks must be an array`); + } + return contract.hooks.map((hook, index) => { + if (!isRecord(hook) || typeof hook.id !== 'string') { + throw new Error(`${path}: hooks[${index}].id must be a string`); + } + return hook.id; + }); +} + +// Contract levels are cumulative. A level can use hooks introduced earlier. +function hooksByLevel(track: Track): HooksByLevel { + const perFile = new Map(); + for (const file of readdirSync(track.contracts).filter(name => /^\d\d-.*\.json$/.test(name))) { + perFile.set(file.slice(0, 2), contractHookIds(join(track.contracts, file))); + } + const byLevel: HooksByLevel = new Map(); + for (const level of perFile.keys()) { + const ids = [...perFile.entries()] + .filter(([candidate]) => candidate <= level) + .flatMap(([, hookIds]) => hookIds); + byLevel.set(level, new Set(ids)); + } + return byLevel; +} + +function packId(reference: string): string { + return reference.slice(0, reference.lastIndexOf('@')); +} + +function ownedFragment( + fragment: CompiledOwnedTaskFragment, + owners: ReadonlySet, +): boolean { + return fragment.owners.some(owner => owners.has(owner)); +} + +function recipeScenarioScopes(track: Track, recipeFile: string): Map { + const recipeDir = join(track.dir, 'composition', 'recipes'); + const chain: CompiledRecipeRelease[] = []; + const seen = new Set(); + let currentFile: string | null = recipeFile; + let isolatesSelectedSources = false; + while (currentFile !== null) { + if (seen.has(currentFile)) throw new Error(`recipe base cycle at ${currentFile}`); + seen.add(currentFile); + const path = join(recipeDir, currentFile); + chain.push(compileRecipeFile(path, { trackRoot: track.dir })); + const source = readRecipeSource(path); + if (chain.length === 1) isolatesSelectedSources = source.isolatesSelectedSources; + currentFile = source.baseRecipe; + } + + const recipe = chain[0]; + if (recipe === undefined) throw new Error(`recipe chain is empty for ${recipeFile}`); + const packs = new Map(recipe.packs.map(pack => [pack.id, pack])); + const contracts = isolatesSelectedSources + ? recipe.recipe.task.contracts + : chain.flatMap(release => release.recipe.task.contracts); + const requirements = isolatesSelectedSources + ? recipe.recipe.task.requirements + : chain.flatMap(release => release.recipe.task.requirements); + const scopes = new Map(); + + const ownersFor = (check: CompiledRecipeRelease['checks'][number]): Set => { + const found = new Set([check.packId, ...(check.requiresFeatures ?? [])]); + const visit = (id: string): void => { + const pack = packs.get(id); + if (pack === undefined) return; + for (const reference of pack.requiresPacks) { + const dependency = packId(reference); + if (found.has(dependency)) continue; + found.add(dependency); + visit(dependency); + } + }; + [...found].forEach(visit); + return found; + }; + + for (const check of recipe.checks) { + const source = check.source.replace(/^scenarios\//, ''); + const scope = scopes.get(source) ?? { + features: new Map>(), + contractOwners: new Set(), + requirementOwners: new Set(), + contractText: '', + requirementText: '', + }; + const criteria = scope.features.get(check.featureId) ?? new Set(); + criteria.add(check.criterionId); + scope.features.set(check.featureId, criteria); + for (const owner of ownersFor(check)) { + scope.contractOwners.add(owner); + scope.requirementOwners.add(owner); + } + scopes.set(source, scope); + } + + for (const scope of scopes.values()) { + const selectedContracts = isolatesSelectedSources + ? contracts.filter(fragment => ownedFragment(fragment, scope.contractOwners)) + : contracts; + const selectedRequirements = isolatesSelectedSources + ? requirements.filter(fragment => ownedFragment(fragment, scope.requirementOwners)) + : requirements; + scope.contractText = selectedContracts.map(fragment => fragment.text).join('\n'); + scope.requirementText = selectedRequirements.map(fragment => fragment.text).join('\n'); + } + return scopes; +} + +function normalizeText(text: string): string { + return text.replace(/\*\*/g, '').replace(/—/g, '-').toLowerCase().replace(/\s+/g, ' ').trim(); +} + +function promptFor(track: Track, level: string): string | null { + const dir = join(track.dir, 'prompts'); + if (!existsSync(dir)) return null; + const file = readdirSync(dir).find(name => name.startsWith(`${level}-`) && name.endsWith('.md')); + return file === undefined ? null : normalizeText(readFileSync(join(dir, file), 'utf8')); +} + +function referencedActors(step: CompiledStep): string[] { + return [step.from, step.fromActor].filter((actor): actor is string => actor !== undefined); +} + +function referencedTestIds(step: CompiledStep): string[] { + return [step.testid, step.in?.testid].filter((id): id is string => id !== undefined); +} + +function main(args: readonly string[]): number { + const trackArg = optionValue(args, '--track'); + const recipeArg = optionValue(args, '--recipe'); + const availableTracks = listTracks(); + const trackNames = trackArg === null + ? (availableTracks.length > 0 ? availableTracks : [DEFAULT_TRACK]) + : [trackArg]; + if (recipeArg !== null && trackNames.length !== 1) { + throw new Error('--recipe requires one --track'); + } + + const knownActions = new Set(ACTION_REGISTRY.ids); + let problems = 0; + let unstatedWarnings = 0; + let staleStatementWarnings = 0; + const fail = (where: string, message: string): void => { + console.log(` ${where}: ${message}`); + problems += 1; + }; + + for (const name of trackNames) { + const track = loadTrack(name); + console.log(`# track: ${name}`); + const contracts = hooksByLevel(track); + const recipeScopes = recipeArg === null ? null : recipeScenarioScopes(track, recipeArg); + for (const file of readdirSync(track.scenarios).filter(candidate => candidate.endsWith('.json'))) { + const recipeScope = recipeScopes?.get(file); + if (recipeScopes !== null && recipeScope === undefined) continue; + const scenarioPath = join(track.scenarios, file); + let spec; + try { + spec = compileScenarioDefinition(readJson(scenarioPath), { source: scenarioPath }); + } catch (error: unknown) { + fail(file, error instanceof Error ? error.message : String(error)); + continue; + } + const level = String(spec.level).padStart(2, '0'); + const hooks = recipeScope === undefined ? (contracts.get(level) ?? null) : null; + + console.log(file); + const prompt = recipeScope === undefined + ? promptFor(track, level) + : normalizeText(recipeScope.requirementText); + for (const feature of spec.features) { + const selectedCriteria = recipeScope?.features.get(feature.id); + if (recipeScope !== undefined && selectedCriteria === undefined) continue; + const criteria = selectedCriteria === undefined + ? feature.criteria + : feature.criteria.filter(criterion => selectedCriteria.has(criterion.id)); + for (const criterion of criteria) { + if (criterion.statedBy !== undefined) { + if (recipeScope === undefined && prompt !== null + && !prompt.includes(normalizeText(criterion.statedBy))) { + staleStatementWarnings += 1; + console.log(` warn F${feature.id} ${criterion.id}: statedBy text is not in the legacy level ${level} prompt`); + } + } else if (recipeScope === undefined && criterion.points > 0) { + unstatedWarnings += 1; + console.log(` warn F${feature.id} ${criterion.id}: carries ${criterion.points} point(s) with no statedBy - the requirement may be unstated`); + } + } + + const actors = new Set(feature.actors ?? []); + const steps = [...feature.setup, ...criteria.flatMap(criterion => criterion.steps)]; + const declared = (actor: string): boolean => actors.has(actor) + || [...actors].some(candidate => actor.startsWith(`${candidate}-`)); + for (const step of steps) { + const at = `F${feature.id} ${step.do}`; + if (!knownActions.has(step.do)) fail(at, `unknown step type "${step.do}"`); + if (step.actor !== undefined && actors.size > 0 && !declared(step.actor)) { + fail(at, `actor "${step.actor}" is not in the feature's actor list`); + } + for (const actor of referencedActors(step)) { + if (actors.size > 0 && !declared(actor)) { + fail(at, `actor "${actor}" is not in the feature's actor list`); + } + } + if (hooks !== null) { + for (const id of referencedTestIds(step)) { + if (!hooks.has(id)) fail(at, `testid "${id}" is not in the contract`); + } + } + if (recipeScope !== undefined) { + for (const id of referencedTestIds(step)) { + if (!recipeScope.contractText.includes(`\`${id}\``)) { + fail(at, `testid "${id}" is not in the selected recipe contracts`); + } + } + } + } + + const points = criteria.reduce((total, criterion) => total + criterion.points, 0); + if (recipeScope === undefined && feature.max !== undefined && points !== feature.max) { + fail(`F${feature.id}`, `criteria total ${points} but max says ${feature.max}`); + } + } + } + } + + const warnings = unstatedWarnings + staleStatementWarnings; + console.log(problems > 0 + ? `\n${problems} error(s); ${warnings} warning(s)` + : warnings > 0 + ? `\n0 errors; ${warnings} warning(s) (${unstatedWarnings} point-carrying criteria lack statedBy; ${staleStatementWarnings} statedBy references are outside legacy prompts)` + : '\n0 errors; 0 warnings'); + return problems > 0 ? 1 : 0; +} + +process.exitCode = main(process.argv.slice(2)); diff --git a/tools/stack-bench/commands/composition-cli.ts b/tools/stack-bench/commands/composition-cli.ts new file mode 100644 index 00000000000..e8f9dd12f9b --- /dev/null +++ b/tools/stack-bench/commands/composition-cli.ts @@ -0,0 +1,335 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync, readdirSync, realpathSync } from 'node:fs'; +import { join, relative, resolve, sep } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { compilePackDefinition, compileRecipeFile, resolveTaskFragment } from '../src/composition/composition-compiler.js'; +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { canonicalDefinitionJson } from '../src/composition/definition-plan.js'; +import { buildRecipeRelease } from '../src/composition/recipe-release.js'; +import { composeSelectedRecipeTask, selectRecipeRelease } from '../src/composition/recipe-selection.js'; +import { TRACKS_DIR } from '../src/composition/tracks.js'; +import type { CompiledPackDefinition, CompiledRecipePlan } from '../src/composition/composition-compiler.js'; +import type { RecipeRelease } from '../src/composition/recipe-release.js'; +import type { RecipeSelectionOptions, SelectedRecipeRelease } from '../src/composition/recipe-selection.js'; + +export { selectRecipeRelease } from '../src/composition/recipe-selection.js'; + +interface TrackRootOptions { + trackRoot: string; +} + +interface PackIndexEntry { + pack: CompiledPackDefinition; + path: string; +} + +interface CalibrationValue { + id: string; + version: string; + recipe?: { id?: string; version?: string; contentSha256?: string }; +} + +type RecipeOptions = TrackRootOptions & RecipeSelectionOptions; +type RecipeTaskKind = 'requirements' | 'contracts'; + +function json(path: string, label: string): T { + try { return JSON.parse(readFileSync(path, 'utf8')); } + catch (error) { + const message = error instanceof Error ? error.message : String(error); + throw new Error(`cannot read ${label} ${path}: ${message}`, { cause: error }); + } +} + +function contained(root: string, path: string, label: string): string { + const absoluteRoot = realpathSync(resolve(root)); + const candidate = resolve(path); + const lexical = relative(absoluteRoot, candidate); + if (lexical === '..' || lexical.startsWith(`..${sep}`)) throw new Error(`${label} escapes ${absoluteRoot}`); + if (!existsSync(candidate)) throw new Error(`${label} does not exist: ${candidate}`); + const absolute = realpathSync(candidate); + const physical = relative(absoluteRoot, absolute); + if (physical === '..' || physical.startsWith(`..${sep}`)) throw new Error(`${label} escapes ${absoluteRoot}`); + return absolute; +} + +function packIndex(trackRoot: string): Map { + const directory = join(trackRoot, 'composition', 'packs'); + const byRef = new Map(); + for (const name of readdirSync(directory).filter(file => file.endsWith('.json')).sort()) { + const path = join(directory, name); + const pack = compilePackDefinition(json(path, 'pack'), { + source: relative(trackRoot, path).replaceAll('\\', '/'), + }); + const ref = `${pack.id}@${pack.version}`; + if (byRef.has(ref)) throw new Error(`duplicate pack release ${ref}`); + byRef.set(ref, { pack, path: realpathSync(path) }); + } + for (const [ref, { pack }] of byRef) { + for (const dependency of [...pack.requiresPacks, ...pack.conflictsWith]) { + if (!byRef.has(dependency)) throw new Error(`${ref} references missing pack ${dependency}`); + } + } + return byRef; +} + +export function validatePackFile(path: string, options: Partial = {}) { + const { trackRoot } = options; + if (trackRoot === undefined) throw new Error('pack validation requires trackRoot'); + const root = realpathSync(resolve(trackRoot)); + const absolute = contained(join(root, 'composition'), path, 'pack path'); + const pack = compilePackDefinition(json(absolute, 'pack'), { + source: relative(root, absolute).replaceAll('\\', '/'), + }); + const packs = packIndex(root); + const ownRef = `${pack.id}@${pack.version}`; + const indexed = packs.get(ownRef); + if (!indexed || indexed.path !== absolute) throw new Error(`${ownRef} is not the indexed source ${absolute}`); + for (const ref of [...pack.requiresPacks, ...pack.conflictsWith]) { + if (!packs.has(ref)) throw new Error(`${ownRef} references missing pack ${ref}`); + } + const sourceCache = new Map(); + for (const kind of ['requirements', 'contracts'] satisfies RecipeTaskKind[]) { + for (const fragment of pack.task[kind]) { + resolveTaskFragment(fragment, { trackRoot: root, + source: `${relative(root, absolute).replaceAll('\\', '/')}.task.${kind}.${fragment.id}`, + sourceCache }); + } + } + const state = new Map(); + const visit = (ref: string, chain: string[] = []): void => { + if (state.get(ref) === 'done') return; + if (state.get(ref) === 'visiting') throw new Error(`pack dependency cycle: ${[...chain, ref].join(' -> ')}`); + state.set(ref, 'visiting'); + const entry = packs.get(ref); + if (!entry) throw new Error(`missing pack release ${ref}`); + for (const dependency of entry.pack.requiresPacks) { + if (!packs.has(dependency)) throw new Error(`${ref} references missing pack ${dependency}`); + visit(dependency, [...chain, ref]); + } + state.set(ref, 'done'); + }; + visit(ownRef); + let criteria = 0; + for (const check of pack.checks) { + const scenarioPath = contained(root, join(root, check.source), `${pack.id}.${check.id}.source`); + const scenario = compileScenarioDefinition(json(scenarioPath, 'scenario'), { + source: relative(root, scenarioPath).replaceAll('\\', '/'), + }); + const feature = scenario.features.find(candidate => candidate.id === check.feature); + if (!feature) throw new Error(`${pack.id}.${check.id} references missing feature ${check.feature}`); + criteria += feature.criteria.length; + } + return { id: pack.id, version: pack.version, state: pack.state, path: absolute, + checkGroups: pack.checks.length, criteria, requiresPacks: pack.requiresPacks }; +} + +export function validateRecipeFile(path: string, options: Partial = {}): { + plan: CompiledRecipePlan; + release: RecipeRelease; +} { + const { trackRoot } = options; + if (trackRoot === undefined) throw new Error('recipe validation requires trackRoot'); + const absolute = contained(join(trackRoot, 'composition'), path, 'recipe path'); + const plan = compileRecipeFile(absolute, { trackRoot }); + const release = buildRecipeRelease(absolute, { trackRoot }); + return { plan, release }; +} + +export function showRecipeFile(path: string, options: RecipeOptions): SelectedRecipeRelease & { + builderTask: ReturnType & { note: string }; +} { + const compiled = validateRecipeFile(path, options); + const selected = selectRecipeRelease(compiled.release, options); + const builderTask = composeSelectedRecipeTask(compiled.plan, selected.selection); + return { + ...selected, + builderTask: { + ...builderTask, + note: 'Pack selection defines the requested task; a check-only filter narrows measurement inside it.', + }, + }; +} + +const same = (left: unknown, right: unknown): boolean => + canonicalDefinitionJson(left) === canonicalDefinitionJson(right); + +function meaningView(release: RecipeRelease) { + return { + track: release.track, + task: release.task, + checks: release.checkCatalog.map(({ stableKey, packId, checkGroupId, role, source, + featureId, criterionId, description }) => ({ stableKey, packId, checkGroupId, role, + source, featureId, criterionId, description })), + }; +} + +function scoringView(release: RecipeRelease) { + return { scoring: release.scoring, + checks: release.checkCatalog.map(({ stableKey, points }) => ({ stableKey, points })) }; +} + +function metadataView(release: RecipeRelease) { + return { id: release.id, version: release.version, state: release.state, title: release.title, + sequence: release.sequence, sourceManifestSha256: release.sourceManifestSha256 }; +} + +function matchingCalibrations(trackRoot: string, release: RecipeRelease): Array<{ + path: string; + value: CalibrationValue; +}> { + const directory = join(trackRoot, 'composition', 'calibrations'); + if (!existsSync(directory)) return []; + return readdirSync(directory).filter(name => name.endsWith('.json')).sort() + .map(name => ({ path: join(directory, name), + value: json(join(directory, name), 'calibration') })) + .filter(({ value }) => value.recipe?.id === release.id + && value.recipe?.version === release.version + && value.recipe?.contentSha256 === release.contentSha256); +} + +export function diffRecipeFiles(fromPath: string, toPath: string, options: Partial = {}) { + const { trackRoot } = options; + if (trackRoot === undefined) throw new Error('recipe diff requires trackRoot'); + const from = validateRecipeFile(fromPath, { trackRoot }).release; + const to = validateRecipeFile(toPath, { trackRoot }).release; + const categories = { + meaning: !same(meaningView(from), meaningView(to)), + scoring: !same(scoringView(from), scoringView(to)), + fixtures: !same(from.components.fixture, to.components.fixture), + execution: from.executionSha256 !== to.executionSha256, + metadata: !same(metadataView(from), metadataView(to)), + }; + const recipeBindingChanged = from.id !== to.id || from.version !== to.version + || from.meaningSha256 !== to.meaningSha256 || from.executionSha256 !== to.executionSha256 + || from.contentSha256 !== to.contentSha256; + const calibrations = matchingCalibrations(trackRoot, from).map(({ path, value }) => { + const invalidated = []; + const stateChanged = from.state !== to.state; + if (recipeBindingChanged) invalidated.push('recipe binding'); + if (stateChanged) invalidated.push('recipe qualification state'); + if (categories.fixtures) invalidated.push('fixture binding'); + if (categories.scoring) invalidated.push('zero-point control policy'); + if (categories.meaning || categories.scoring || categories.execution || categories.fixtures) { + invalidated.push('reference repetitions', 'mutation repetitions'); + } + if (categories.meaning || categories.scoring || categories.fixtures) invalidated.push('null repetitions'); + if (recipeBindingChanged || stateChanged) invalidated.push('promotion decision'); + return { id: value.id, version: value.version, + path: relative(trackRoot, path).replaceAll('\\', '/'), invalidated: [...new Set(invalidated)] }; + }); + const fragmentDiff = (kind: RecipeTaskKind) => { + const before = new Map(from.task[kind].map(fragment => [fragment.id, fragment])); + const after = new Map(to.task[kind].map(fragment => [fragment.id, fragment])); + return { + added: [...after.keys()].filter(key => !before.has(key)).sort(), + removed: [...before.keys()].filter(key => !after.has(key)).sort(), + changed: [...after.keys()].filter(key => before.has(key) + && !same(before.get(key), after.get(key))).sort(), + }; + }; + return { + from: { id: from.id, version: from.version, state: from.state, meaningSha256: from.meaningSha256, + executionSha256: from.executionSha256, contentSha256: from.contentSha256 }, + to: { id: to.id, version: to.version, state: to.state, meaningSha256: to.meaningSha256, + executionSha256: to.executionSha256, contentSha256: to.contentSha256 }, + categories, + taskFragments: { + requirements: fragmentDiff('requirements'), + contracts: fragmentDiff('contracts'), + composedTaskChanged: from.task.composedSha256 !== to.task.composedSha256, + }, + calibrations, + }; +} + +type CliSubject = 'pack' | 'recipe'; +type CliCommand = 'validate' | 'show' | 'diff'; + +interface ParsedArgs extends RecipeSelectionOptions { + json: boolean; + positional: string[]; + packIds: string[]; + checkKeys: string[]; + track?: string; + trackRoot?: string; +} + +interface CliArgs extends ParsedArgs { + subject: CliSubject; + command: CliCommand; + paths: string[]; + trackRoot: string; +} + +function nextArgument(argv: string[], index: number, option: string): string { + const value = argv[index + 1]; + if (value === undefined) throw new Error(`${option} requires a value`); + return value; +} + +function parse(argv: string[]): CliArgs { + const args: ParsedArgs = { json: false, positional: [], packIds: [], checkKeys: [] }; + for (let index = 2; index < argv.length; index += 1) { + const argument = argv[index]; + if (argument === undefined) continue; + if (argument === '--track') args.track = nextArgument(argv, index++, argument); + else if (argument === '--track-root') args.trackRoot = resolve(nextArgument(argv, index++, argument)); + else if (argument === '--pack') args.packIds.push(...nextArgument(argv, index++, argument).split(',').filter(Boolean)); + else if (argument === '--check') args.checkKeys.push(...nextArgument(argv, index++, argument).split(',').filter(Boolean)); + else if (argument === '--json') args.json = true; + else args.positional.push(argument); + } + const [subject, command, ...paths] = args.positional; + if (subject !== 'pack' && subject !== 'recipe') { + throw new Error('usage: npm run pack -- validate --track | npm run recipe -- validate|show --track | npm run recipe -- diff --track '); + } + if (command !== 'validate' && command !== 'show' && command !== 'diff') { + throw new Error('usage: npm run pack -- validate --track | npm run recipe -- validate|show --track | npm run recipe -- diff --track '); + } + if (subject === 'pack' && command !== 'validate') throw new Error(`pack ${command} is not supported`); + if ((command === 'diff' ? paths.length !== 2 : paths.length !== 1)) throw new Error(`${subject} ${command} received the wrong number of paths`); + if (!args.trackRoot && !args.track) throw new Error('--track or --track-root is required'); + if ((args.packIds.length || args.checkKeys.length) && !(subject === 'recipe' && command === 'show')) { + throw new Error('--pack and --check are allowed only with recipe show'); + } + const trackRoot = args.trackRoot ?? join(TRACKS_DIR, args.track ?? ''); + return { ...args, subject, command, paths, trackRoot }; +} + +function main() { + const args = parse(process.argv); + const firstPath = args.paths[0]; + if (firstPath === undefined) throw new Error('command requires a source path'); + let result: object; + if (args.subject === 'pack') result = validatePackFile(firstPath, args); + else if (args.command === 'diff') { + const secondPath = args.paths[1]; + if (secondPath === undefined) throw new Error('recipe diff requires two source paths'); + result = diffRecipeFiles(firstPath, secondPath, args); + } else if (args.command === 'show') result = showRecipeFile(firstPath, args); + else { + const compiled = validateRecipeFile(firstPath, args); + result = { + id: compiled.release.id, version: compiled.release.version, state: compiled.release.state, + packs: compiled.release.components.packs.length, checks: compiled.release.checkCatalog.length, + points: compiled.release.checkCatalog.reduce((total, check) => total + check.points, 0), + meaningSha256: compiled.release.meaningSha256, + executionSha256: compiled.release.executionSha256, + contentSha256: compiled.release.contentSha256, + }; + } + if (args.json || args.command === 'show' || args.command === 'diff') console.log(JSON.stringify(result, null, 2)); + else if ('id' in result && 'version' in result && 'state' in result) { + console.log(`${String(result.id)}@${String(result.version)} ${String(result.state)}: valid`); + } else throw new Error('validation result has no release identity'); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) { + try { main(); } + catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + } +} diff --git a/tools/stack-bench/commands/container-smoke.ts b/tools/stack-bench/commands/container-smoke.ts new file mode 100644 index 00000000000..ff5451a20e6 --- /dev/null +++ b/tools/stack-bench/commands/container-smoke.ts @@ -0,0 +1,199 @@ +#!/usr/bin/env node +// Model-free production smoke for the isolated SpacetimeDB build path. +// +// Starts a dedicated host on an ephemeral loopback port and data directory, +// prepares the real build image, then builds and publishes a +// tiny TypeScript module with `spacetime dev`, verifies it through SQL, then +// removes only the container, process and data directory created here. + +import { spawn, execFileSync } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; +import { cpSync, existsSync, mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { createServer } from 'node:net'; +import type { AddressInfo } from 'node:net'; +import { basename, join, resolve } from 'node:path'; +import { tmpdir } from 'node:os'; + +import { killTree, pidsOnPort, processIdentity } from '../src/runtime/platform.js'; +import { createBackendLease, readBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { fetchStatus } from '../src/runtime/readiness.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; +import { containerReachableSpacetimeUri } from '../src/runtime/spacetime-target.js'; +import { codingContainerAgentExecOptions } from '../src/runtime/coding-container-policy.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const REPO = resolve(ROOT, '..', '..'); +const IMAGE = process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE; +const CLI = process.env.SPACETIME_BIN ?? join(REPO, 'target', 'release', + process.platform === 'win32' ? 'spacetimedb-cli.exe' : 'spacetimedb-cli'); +const RUN_BUILD = compiledEntrypoint('container', 'run-build.js'); +const FIXTURE = join(ROOT, 'tests', 'fixtures', 'spacetime-module'); + +const delay = (ms: number): Promise => new Promise(resolveDelay => setTimeout(resolveDelay, ms)); + +interface PreparedContainerIdentity { + containerName: string; + identity: string; + networkMode: string | null; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +function parsePreparedContainerIdentity(text: string): PreparedContainerIdentity { + const value: unknown = JSON.parse(text.trim().split(/\r?\n/).pop() ?? ''); + if (!isRecord(value)) throw new Error('prepared container identity is invalid'); + const record = value; + if (typeof record.containerName !== 'string' || typeof record.identity !== 'string' + || (record.networkMode !== null && typeof record.networkMode !== 'string')) { + throw new Error('prepared container identity is invalid'); + } + return { containerName: record.containerName, identity: record.identity, networkMode: record.networkMode }; +} + +async function freePort() { + const server = createServer(); + await new Promise((ok, fail) => server.listen({ port: 0, host: '127.0.0.1' }, ok).once('error', fail)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('could not allocate a TCP port'); + const port: AddressInfo['port'] = address.port; + await new Promise(ok => server.close(ok)); + return port; +} + +async function waitFor(check: () => boolean | Promise, timeoutMs: number, description: string): Promise { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + if (await check()) return; + await delay(250); + } + throw new Error(`timed out waiting for ${description}`); +} + +async function main() { + if (!existsSync(CLI)) throw new Error(`local SpacetimeDB CLI is missing: ${CLI}`); + execFileSync('docker', ['image', 'inspect', IMAGE], { stdio: 'pipe' }); + + const root = mkdtempSync(join(tmpdir(), 'stack-bench-container-smoke-')); + const app = join(root, 'app'); + const dataDir = join(root, 'spacetime-data'); + const port = await freePort(); + const uri = `http://127.0.0.1:${port}`; + const module = `stackbench-container-smoke-${process.pid}`; + const containerName = `stack-bench-${basename(root)}`; + const leasePath = join(root, 'backend-lease.json'); + let host: ChildProcess | null = null; + let dev: ChildProcess | null = null; + let output = ''; + + try { + mkdirSync(app, { recursive: true }); + host = spawn(CLI, ['start', '--listen-addr', `127.0.0.1:${port}`, '--data-dir', dataDir], + { stdio: 'ignore', windowsHide: true }); + await waitFor(async () => { + const status = await fetchStatus(`${uri}/v1/ping`, { timeoutMs: 5000 }); + return status !== null && status >= 200 && status < 300; + }, 120_000, `dedicated SpacetimeDB host on :${port}`); + + const lease = createBackendLease({ runId: basename(root), backend: 'spacetime', + track: 'container-smoke', runIndex: 0, serverUri: uri, module, dataDir }); + lease.state = 'active'; + lease.resources.launchedProcess = host.pid ? processIdentity(host.pid) : null; + lease.resources.listenerProcesses = pidsOnPort(port).map(pid => processIdentity(pid)) + .filter((identity): identity is NonNullable => identity !== null); + writeBackendLease(leasePath, lease); + + const prepared = execFileSync(process.execPath, + [RUN_BUILD, '--app', app, '--backend', 'spacetime', '--image', IMAGE, '--prepare-only'], + { encoding: 'utf8', stdio: 'pipe', maxBuffer: 16 * 1024 * 1024, + env: { ...process.env, STACK_BENCH_LEASE: leasePath, + STACK_BENCH_LEASE_TOKEN: lease.ownershipToken } }); + const identity = parsePreparedContainerIdentity(prepared); + if (identity.containerName !== containerName) { + throw new Error(`prepared unexpected container ${identity.containerName}`); + } + const leasedContainer = readBackendLease(leasePath, + { token: lease.ownershipToken, backend: 'spacetime', active: true }).resources.buildContainer; + if (!leasedContainer || identity.identity.split(' ')[0] !== leasedContainer.id) { + throw new Error('prepared container identity was not recorded in the backend lease'); + } + if (!leasedContainer.image || !/^sha256:[0-9a-f]{64}$/.test(leasedContainer.image)) { + throw new Error(`prepared container did not record an immutable image id: ${leasedContainer.image}`); + } + + cpSync(FIXTURE, join(app, 'spacetimedb'), { recursive: true }); + const agentExec = ['exec', ...codingContainerAgentExecOptions()]; + execFileSync('docker', [...agentExec, containerName, 'sh', '-c', + 'umask 000; cd /app/spacetimedb && npm install --no-audit --no-fund'], { stdio: 'pipe' }); + + const startedDev = spawn('docker', [...agentExec, '-i', containerName, 'sh', '-c', + `umask 000; cd /app/spacetimedb && /deps/spacetimedb-cli dev ${module} ` + + '--no-config --project-path /app/spacetimedb --module-path . ' + + '--server-only --skip-generate ' + + `-s ${containerReachableSpacetimeUri({ resources: { serverUri: uri, + buildContainer: lease.resources.buildContainer } }, identity.networkMode)} -y`], + { stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true }); + dev = startedDev; + const collect = (chunk: Buffer): void => { output = (output + chunk.toString()).slice(-128 * 1024); }; + startedDev.stdout?.on('data', collect); + startedDev.stderr?.on('data', collect); + + await waitFor(() => { + if (/Published successfully!/.test(output)) return true; + if (startedDev.exitCode !== null) throw new Error(`spacetime dev exited ${startedDev.exitCode}:\n${output}`); + return false; + }, 240_000, 'containerized module publish'); + + const sql = execFileSync(CLI, ['sql', module, 'SELECT * FROM smoke_item', '-s', uri], + { encoding: 'utf8', stdio: 'pipe' }); + if (!/\bid\s*\|\s*value\b/.test(sql)) throw new Error(`SQL verification failed:\n${sql}`); + if (startedDev.exitCode !== null) throw new Error('spacetime dev did not remain alive as a watcher'); + + // Publishing and log streaming must retain one authenticated identity. A + // prior dev bug published with a token stored only in a Config clone, then + // directly logged in again for logs and received an authorization error. + await delay(2_000); + const logStreamingAuthorized = !/Log streaming error:.*not authorized/s.test(output); + console.log(JSON.stringify({ ok: true, image: IMAGE, container: identity.identity, + host: { uri, listenerPids: pidsOnPort(port) }, published: true, sqlVerified: true, + watcherAlive: true, leasedContainer: true, immutableImagePinned: true, + logStreamingAuthorized }, null, 2)); + if (!logStreamingAuthorized) { + throw new Error('`spacetime dev` published successfully but its log stream was not authorized'); + } + // The grader resets by republishing the same named database from this exact + // leased container. Prove that `-y` retained a reusable local identity, + // rather than merely proving that the first anonymous-looking publish ran. + execFileSync('docker', [...agentExec, containerName, 'sh', '-c', + 'for process in /proc/[0-9]*; do ' + + 'test "$(cat "$process/comm" 2>/dev/null)" = spacetimedb-cli ' + + '&& kill -TERM "${process##*/}" || true; done'], { stdio: 'pipe' }); + await waitFor(() => startedDev.exitCode !== null, 15_000, 'spacetime dev to stop before reset publish'); + const targetUri = containerReachableSpacetimeUri({ resources: { serverUri: uri, + buildContainer: lease.resources.buildContainer } }, identity.networkMode); + execFileSync('docker', [...agentExec, containerName, 'sh', '-c', + `umask 000; cd /app/spacetimedb && /deps/spacetimedb-cli publish ${module} ` + + `--no-config --module-path . -s ${targetUri} --delete-data -y`], + { stdio: 'pipe', timeout: 240_000 }); + const afterReset = execFileSync(CLI, ['sql', module, 'SELECT * FROM smoke_item', '-s', uri], + { encoding: 'utf8', stdio: 'pipe' }); + if (!/\bid\s*\|\s*value\b/.test(afterReset)) { + throw new Error(`SQL verification after reset publish failed:\n${afterReset}`); + } + console.log(JSON.stringify({ resetRepublished: true, resetSqlVerified: true })); + } finally { + if (dev && dev.exitCode === null) dev.kill('SIGTERM'); + try { execFileSync('docker', ['rm', '-f', containerName], { stdio: 'ignore' }); } catch { /* absent */ } + // The port was proven unused before this script started the host. Kill only + // listeners on that exact ephemeral port, then the wrapper if it remains. + for (const pid of pidsOnPort(port)) killTree(pid); + if (host && host.exitCode === null) killTree(host.pid); + rmSync(root, { recursive: true, force: true }); + } +} + +main().catch(error => { + console.error(error.stack ?? error.message); + process.exitCode = 1; +}); diff --git a/tools/stack-bench/commands/cost-ledger.ts b/tools/stack-bench/commands/cost-ledger.ts new file mode 100644 index 00000000000..2ea3182b6dd --- /dev/null +++ b/tools/stack-bench/commands/cost-ledger.ts @@ -0,0 +1,34 @@ +#!/usr/bin/env node + +import { resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { readRunJson } from '../src/evidence/artifacts.js'; +import { durableCostLedger } from '../src/evidence/cost-proof.js'; + +export { durableCostLedger } from '../src/evidence/cost-proof.js'; + +function parseArgs(argv: string[]): string { + const value = (name: string): string | null => { + const index = argv.indexOf(name); + return index < 0 ? null : argv[index + 1] ?? null; + }; + const runPath = value('--run'); + const workdir = value('--workdir'); + if ((runPath === null) === (workdir === null)) { + throw new Error('use exactly one of --run or --workdir '); + } + return runPath !== null ? resolve(runPath) : resolve(workdir as string, 'run.json'); +} + +const entrypoint = process.argv[1] ? pathToFileURL(resolve(process.argv[1])).href : null; +if (import.meta.url === entrypoint) { + try { + const ledger = durableCostLedger(readRunJson(parseArgs(process.argv.slice(2)))); + process.stdout.write(`${JSON.stringify(ledger, null, 2)}\n`); + if (!ledger.complete) process.exitCode = 1; + } catch (error) { + process.stderr.write(`${error instanceof Error ? error.message : String(error)}\n`); + process.exitCode = 2; + } +} diff --git a/tools/stack-bench/commands/definition-snapshots.ts b/tools/stack-bench/commands/definition-snapshots.ts new file mode 100644 index 00000000000..9f227e19d30 --- /dev/null +++ b/tools/stack-bench/commands/definition-snapshots.ts @@ -0,0 +1,80 @@ +import { mkdirSync, readFileSync, renameSync, writeFileSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { compileScenarioDefinition } from '../src/composition/definition-compiler.js'; +import { canonicalDefinitionJson, compileTrackPlan } from '../src/composition/definition-plan.js'; +import { listTracks } from '../src/composition/tracks.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; + +const SNAPSHOT_DIR = join(STACK_BENCH_ROOT, 'tests', 'snapshots', 'definitions'); +const ALL_ACTIONS = join(STACK_BENCH_ROOT, 'tests', 'fixtures', 'definitions', 'all-actions.json'); + +function atomicWrite(path: string, contents: string): void { + mkdirSync(dirname(path), { recursive: true }); + const temporary = `${path}.tmp-${process.pid}`; + writeFileSync(temporary, contents); + renameSync(temporary, path); +} + +interface DefinitionSnapshot { + name: string; + value: unknown; +} + +export function currentDefinitionSnapshots(): DefinitionSnapshot[] { + const entries: DefinitionSnapshot[] = listTracks({ includeInternal: true }).map(name => ({ + name: `${name}.snapshot.json`, + value: compileTrackPlan(name), + })); + entries.push({ + name: 'all-actions.snapshot.json', + value: compileScenarioDefinition(JSON.parse(readFileSync(ALL_ACTIONS, 'utf8')), { + source: ALL_ACTIONS, + }), + }); + return entries.sort((a, b) => a.name.localeCompare(b.name)); +} + +export interface DefinitionSnapshotResult { + checked: number; + changed: string[]; +} + +export function checkDefinitionSnapshots( + { update = false }: { update?: boolean } = {}, +): DefinitionSnapshotResult { + const entries = currentDefinitionSnapshots(); + const changed: string[] = []; + for (const entry of entries) { + const path = join(SNAPSHOT_DIR, entry.name); + const actual = canonicalDefinitionJson(entry.value); + let expected: string | null = null; + try { + expected = readFileSync(path, 'utf8').replaceAll('\r\n', '\n'); + } catch (error: unknown) { + if (!(error instanceof Error && 'code' in error && error.code === 'ENOENT')) throw error; + } + if (expected === actual) continue; + changed.push(entry.name); + if (update) atomicWrite(path, actual); + } + if (changed.length > 0 && !update) { + throw new Error( + `definition snapshot drift: ${changed.join(', ')}; inspect the semantic change, then run npm run check:definition-snapshots -- --update`, + ); + } + return { checked: entries.length, changed }; +} + +function main(): void { + const args = new Set(process.argv.slice(2)); + for (const arg of args) { + if (arg !== '--update') throw new Error(`unknown argument ${arg}`); + } + const result = checkDefinitionSnapshots({ update: args.has('--update') }); + console.log(`${result.checked} definition snapshots checked${ + result.changed.length > 0 ? `; ${result.changed.length} updated` : '; no drift'}`); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/fault-injection.ts b/tools/stack-bench/commands/fault-injection.ts new file mode 100644 index 00000000000..c6f1ee4f16b --- /dev/null +++ b/tools/stack-bench/commands/fault-injection.ts @@ -0,0 +1,254 @@ +#!/usr/bin/env node +// Model-free lifecycle fault test for the production harness. +// +// This creates real Docker and SpacetimeDB resources, enters the real restart +// script after the owned listener is stopped but before its replacement starts, +// and proves teardown removes only the exact leased resources. It also verifies +// that the launcher refuses a same-name container absent from the lease. + +import assert from 'node:assert/strict'; +import { execFileSync, spawn } from 'node:child_process'; +import type { ChildProcess } from 'node:child_process'; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync } from 'node:fs'; +import { createServer } from 'node:http'; +import type { Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { basename, join, resolve } from 'node:path'; +import { tmpdir } from 'node:os'; + +import { createBackendLease, writeBackendLease } from '../src/runtime/backend-lease.js'; +import { killTree, pidsOnPort } from '../src/runtime/platform.js'; +import { readArtifact, readArtifactPayload } from '../src/evidence/artifacts.js'; +import { DEFAULT_BUILD_IMAGE } from '../src/composition/product-config.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const REPO = resolve(ROOT, '..', '..'); +const IMAGE = process.env.STACK_BENCH_IMAGE ?? DEFAULT_BUILD_IMAGE; +const CLI = process.env.SPACETIME_BIN ?? join(REPO, 'target', 'release', + process.platform === 'win32' ? 'spacetimedb-cli.exe' : 'spacetimedb-cli'); +const RUN_BUILD = compiledEntrypoint('container', 'run-build.js'); +const delay = (ms: number): Promise => new Promise(resolveDelay => setTimeout(resolveDelay, ms)); + +interface ContainerIdentity { id: string; running: boolean; } +interface ExitResult { code: number | null; signal: NodeJS.Signals | null; } +interface FaultLeaseResources { + listenerProcesses: Array<{ pid: number; startMarker: string }>; + buildContainer: { id: string; image: string; running: boolean; removedAt?: string }; + locks: { releasedAt?: string }[]; +} +interface FaultLeaseEvidence { + runId: string; + state: string; + stoppedAt?: string; + releasedAt?: string; + resources: FaultLeaseResources; +} + +function isRecord(value: unknown): value is Record { + return value !== null && typeof value === 'object' && !Array.isArray(value); +} + +async function freePort() { + const server = createServer((_request, response) => response.end('foreign')); + await new Promise((ok, fail) => server.listen({ port: 0, host: '127.0.0.1' }, ok).once('error', fail)); + const address = server.address(); + if (!address || typeof address === 'string') throw new Error('could not allocate a TCP port'); + const port: AddressInfo['port'] = address.port; + await new Promise(ok => server.close(ok)); + return port; +} + +function inspectContainer(target: string): ContainerIdentity | null { + try { + const output = execFileSync('docker', ['inspect', '--format', + '{{.Id}} {{.State.Running}}', target], { encoding: 'utf8', stdio: 'pipe' }).trim(); + const [id, running] = output.split(/\s+/, 2); + if (!id) return null; + return { id, running: running === 'true' }; + } catch { return null; } +} + +function startContainer(name: string): ContainerIdentity { + const id = execFileSync('docker', ['run', '-d', '--init', '--name', name, + IMAGE, 'sleep', 'infinity'], { encoding: 'utf8', stdio: 'pipe' }).trim(); + assert.ok(id, `Docker did not return an id for ${name}`); + const container = inspectContainer(id); + if (!container) throw new Error(`Docker did not return a running container for ${name}`); + return container; +} + +function removeExactContainer(identity: ContainerIdentity | { id: string } | null): void { + if (!identity) return; + const current = inspectContainer(identity.id); + if (!current || current.id !== identity.id) return; + execFileSync('docker', ['rm', '-f', identity.id], { stdio: 'ignore' }); +} + +async function waitForExit(child: ChildProcess, timeoutMs: number): Promise { + return new Promise((resolveExit, reject) => { + const timeout = setTimeout(() => reject(new Error( + `benchmark runner did not exit after injected failure within ${timeoutMs}ms`)), timeoutMs); + child.once('exit', (code: number | null, signal: NodeJS.Signals | null) => { + clearTimeout(timeout); + resolveExit({ code, signal }); + }); + }); +} + +async function assertRefusesUnleasedCollision() { + const root = mkdtempSync(join(tmpdir(), 'stack-bench-container-collision-')); + const app = join(root, 'app'); + const name = `stack-bench-${basename(root)}`; + const leasePath = join(root, 'backend-lease.json'); + let foreign = null; + try { + mkdirSync(app, { recursive: true }); + foreign = startContainer(name); + const lease = createBackendLease({ runId: `collision-${process.pid}`, backend: 'spacetime', + track: 'fault-injection', runIndex: 0, serverUri: 'http://127.0.0.1:1', + module: `collision-${process.pid}`, dataDir: join(root, 'data') }); + lease.state = 'active'; + writeBackendLease(leasePath, lease); + + let refused = false; + try { + execFileSync(process.execPath, + [RUN_BUILD, '--app', app, '--backend', 'spacetime', '--prepare-only'], + { stdio: 'pipe', env: { ...process.env, STACK_BENCH_LEASE: leasePath, + STACK_BENCH_LEASE_TOKEN: lease.ownershipToken } }); + } catch (error: unknown) { + const childError = error instanceof Error && isRecord(error) ? error : null; + refused = childError?.status === 3 + && /refusing to adopt existing unleased container/.test(String(childError?.stderr)); + } + assert.equal(refused, true, 'launcher did not explicitly refuse an unleased same-name container'); + assert.deepEqual(inspectContainer(foreign.id), foreign, + 'collision refusal changed or stopped the foreign container'); + } finally { + removeExactContainer(foreign); + rmSync(root, { recursive: true, force: true }); + } +} + +async function main() { + assert.ok(existsSync(CLI), `local SpacetimeDB CLI is missing: ${CLI}`); + execFileSync('docker', ['image', 'inspect', IMAGE], { stdio: 'pipe' }); + await assertRefusesUnleasedCollision(); + + const root = mkdtempSync(join(tmpdir(), 'stack-bench-fault-')); + const app = join(root, 'app'); + const out = join(root, 'out'); + const markerPath = join(app, '.fault-ready.json'); + const port = await freePort(); + const uri = `http://127.0.0.1:${port}`; + const foreignName = `stack-bench-foreign-${process.pid}-${Date.now()}`; + let foreignContainer: ContainerIdentity | null = null; + let foreignServer: Server | null = null; + let bench: ChildProcess | null = null; + let marker: { lease: { runId: string; state: string; resources: FaultLeaseResources }; leasePath: string; phase: string } | null = null; + let output = ''; + + try { + mkdirSync(app, { recursive: true }); + mkdirSync(out, { recursive: true }); + foreignContainer = startContainer(foreignName); + const startedForeignServer = createServer((_request, response) => response.end('foreign')); + foreignServer = startedForeignServer; + await new Promise((ok, fail) => startedForeignServer.listen({ port: 0, host: '127.0.0.1' }, ok).once('error', fail)); + const foreignAddress = startedForeignServer.address(); + if (!foreignAddress || typeof foreignAddress === 'string') throw new Error('could not allocate foreign TCP port'); + const foreignUri = `http://127.0.0.1:${foreignAddress.port}`; + + bench = spawn(process.execPath, + [compiledEntrypoint('commands', 'bench.js'), '--backend', 'spacetime', '--track', 'loop', + '--levels', '1', '--agent-adapter', 'fault-injection', '--app', app, '--out', out, + '--url', `file:///${app.replace(/\\/g, '/')}/index.html`, '--skip-probe'], + { env: { ...process.env, STACK_BENCH_STDB_URI: uri, STACK_BENCH_IMAGE: IMAGE }, + stdio: ['ignore', 'pipe', 'pipe'], windowsHide: true }); + const collect = (chunk: Buffer): void => { output = (output + chunk.toString()).slice(-256 * 1024); }; + bench.stdout?.on('data', collect); + bench.stderr?.on('data', collect); + const exited = await waitForExit(bench, 300_000); + assert.notEqual(exited.code, 0, 'injected coding-agent failure unexpectedly exited zero'); + assert.ok(existsSync(markerPath), `fault marker was not written before failure:\n${output}`); + const markerValue: unknown = JSON.parse(readFileSync(markerPath, 'utf8')); + if (!isRecord(markerValue) || !isRecord(markerValue.lease) || !isRecord(markerValue.lease.resources) + || typeof markerValue.leasePath !== 'string' || typeof markerValue.phase !== 'string' + || typeof markerValue.lease.runId !== 'string' || typeof markerValue.lease.state !== 'string') { + throw new Error('fault marker is invalid'); + } + const markerResources = markerValue.lease.resources; + if (!Array.isArray(markerResources.listenerProcesses) || !isRecord(markerResources.buildContainer) + || typeof markerResources.buildContainer.id !== 'string') throw new Error('fault marker resources are invalid'); + marker = { phase: markerValue.phase, leasePath: markerValue.leasePath, + lease: { runId: markerValue.lease.runId, state: markerValue.lease.state, + resources: { listenerProcesses: markerResources.listenerProcesses.filter((item): item is { + pid: number; startMarker: string } => isRecord(item) && typeof item.pid === 'number' + && typeof item.startMarker === 'string'), buildContainer: { + id: markerResources.buildContainer.id, image: String(markerResources.buildContainer.image ?? ''), + running: markerResources.buildContainer.running === true }, locks: [] } } }; + assert.equal(marker.phase, 'restart-stopped', + 'fault was not injected inside the backend restart window'); + assert.equal(marker.lease.state, 'restarting'); + assert.match(marker.lease.resources.buildContainer.image, /^sha256:[0-9a-f]{64}$/, + 'build container lease did not record an immutable image id'); + + const evidencePath = join(out, 'backend-lease.json'); + assert.ok(existsSync(evidencePath), `teardown did not preserve lease evidence:\n${output}`); + const evidence = readArtifactPayload(evidencePath, { expectedKind: 'backend_lease_evidence' }); + const preflight = readArtifact(join(out, 'preflight.json'), { expectedKind: 'preflight' }); + assert.equal(preflight.payload.ok, true, 'paid-run preflight did not pass'); + assert.equal(preflight.attempt.parentId, marker.lease.runId, + 'preflight evidence is not attached to the run it admitted'); + assert.equal(evidence.runId, marker.lease.runId); + assert.equal(evidence.state, 'released', 'benchmark lease did not reach its terminal state'); + assert.ok(evidence.stoppedAt, 'benchmark-owned SpacetimeDB host has no stop evidence'); + assert.ok(evidence.releasedAt, 'benchmark lease has no release evidence'); + assert.deepEqual(evidence.resources.listenerProcesses, []); + assert.equal(evidence.resources.buildContainer.running, false, + 'benchmark-owned build container was not marked removed'); + assert.ok(evidence.resources.buildContainer.removedAt); + assert.ok(evidence.resources.locks.every(lock => lock.releasedAt), + 'one or more resource locks were not released'); + assert.equal(inspectContainer(marker.lease.resources.buildContainer.id), null, + 'benchmark-owned build container survived fatal cleanup'); + assert.equal(pidsOnPort(port).length, 0, 'benchmark-owned listener survived fatal cleanup'); + assert.equal(existsSync(marker.leasePath), false, 'private runtime lease was not removed'); + + assert.equal((await fetch(foreignUri)).status, 200, + 'foreign listener was disturbed by benchmark cleanup'); + assert.deepEqual(inspectContainer(foreignContainer.id), foreignContainer, + 'foreign container was changed or removed by benchmark cleanup'); + + console.log(JSON.stringify({ ok: true, injectedAt: 'restart-stopped-before-replacement', + benchmarkHostStopped: true, benchmarkContainerRemoved: true, locksReleased: true, + privateLeaseRemoved: true, foreignListenerSurvived: true, + foreignContainerSurvived: true, unleasedCollisionRefused: true, + immutableImagePinned: true }, null, 2)); + } finally { + if (bench?.exitCode === null) { + killTree(bench.pid); + await delay(500); + } + if (marker?.lease?.resources?.buildContainer) { + removeExactContainer(marker.lease.resources.buildContainer); + } + for (const identity of marker?.lease?.resources?.listenerProcesses ?? []) { + if (pidsOnPort(port).includes(String(identity.pid))) killTree(identity.pid); + } + if (foreignServer) { + const server = foreignServer; + // The verification fetch uses a keep-alive connection. Waiting on + // close() alone can hold CI open until Undici retires that socket. + server.closeAllConnections(); + await new Promise((ok, fail) => server.close(error => error ? fail(error) : ok())); + } + removeExactContainer(foreignContainer); + rmSync(root, { recursive: true, force: true }); + } +} + +main().catch(error => { + console.error(error.stack ?? error.message); + process.exitCode = 1; +}); diff --git a/tools/stack-bench/commands/leak-audit.ts b/tools/stack-bench/commands/leak-audit.ts new file mode 100644 index 00000000000..9c7b996b19f --- /dev/null +++ b/tools/stack-bench/commands/leak-audit.ts @@ -0,0 +1,274 @@ +#!/usr/bin/env node +// Did a generated build read anything it was not supposed to? +// +// Parse Bash commands as well as file tools and take the app boundary from the +// session's recorded cwd. Reconstructing paths from transcript folder names is +// ambiguous and is never used as authority. +// +// Usage: node dist/commands/leak-audit.js [--dir ] [--json] + +import { readdirSync, readFileSync, existsSync, statSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { homedir } from 'node:os'; +import { pathToFileURL } from 'node:url'; + +const arg = (name: string, fallback: string | null = null): string | null => { + const index = process.argv.indexOf(name); + return index === -1 ? fallback : process.argv[index + 1] ?? fallback; +}; +// --dir is the ONLY root when given. Scanning every project on the machine +// buries the runs under review in unrelated work. +// +// --app takes the application directory instead and finds the transcripts the +// CLI filed for it: they live under ~/.claude/projects in a folder whose name +// is the app's path with every separator and colon turned into a dash. +function transcriptsFor(appDir: string): string[] { + const base = join(homedir(), '.claude', 'projects'); + if (!existsSync(base)) return []; + const want = resolve(appDir).replace(/[\\/:]/g, '-').toLowerCase(); + return readdirSync(base) + .filter(d => d.toLowerCase() === want || d.toLowerCase() === want.replace(/^-+/, '')) + .map(d => join(base, d)); +} + +const norm = (value: unknown): string => String(value ?? '') + .replace(/\\/g, '/').replace(/^["']|["']$/g, '').toLowerCase(); + +// Files a build legitimately needs: its own app, plus node_modules noise. +// ...plus the CLI's own scratch for THIS session (background task output lives +// under temp/claude///tasks). That is the build reading +// its own command output, not the harness. +const IGNORE = /node_modules|\.git[/\\]|package-lock\.json|\/dist\/|\.map$|[/\\]temp[/\\]claude[/\\].*[/\\]tasks[/\\]/; + +// Commands that pull file contents into context. +const READER = /(?:^|[;&|]\s*)(?:cat|head|tail|less|more|type|grep|rg|ack|find|ls\s+-\w*l|sed\s+-n|awk)\s+([^;&|]+)/g; + +const CLASSES: Array = [ + [/\.claude[/\\]projects.*memory|[/\\]memory[/\\].*\.md$/, 'BENCHMARK NOTES'], + [/scenarios[/\\].*\.json|grade\.(?:js|ts)|mutation|check-scenarios/, 'GRADER / TEST SPECS'], + [/contracts[/\\].*\.json|appendix-\d+\.md|walk\.(?:js|ts)|lint\.(?:js|ts)/, 'CONTRACT / LINTER'], + [/prompts[/\\]|test-plans[/\\]|GRADING|RUBRIC/, 'PROMPTS / RUBRIC'], + [/[/\\]skills[/\\]/, 'skill docs (intended)'], + [/backends[/\\].*\.md|CLAUDE\.md|README/, 'setup docs (intended)'], +]; +const classify = (path: string): string => CLASSES.find(([pattern]) => pattern.test(path))?.[1] + ?? 'other'; + +// The boundary is the APP directory, not wherever the session happened to be +// standing. Taking the most common cwd looked reasonable and was wrong: a +// SpacetimeDB build spends most of its turns in backend/spacetimedb, so reads of +// its OWN client/src/module_bindings/*.ts resolved as escapes and the run was +// reported contaminated by its own generated bindings. +// +// When --app names the directory, that is the answer. Otherwise take the +// SHALLOWEST cwd seen, which is the closest thing to the app root the transcript +// knows about — never the most frequent. +function sessionCwd(lines: string[]): string | null { + const seen = new Set(); + for (const l of lines) { + const m = l.match(/"cwd":"((?:[^"\\]|\\.)*)"/); + if (m?.[1]) seen.add(norm(m[1].replace(/\\\\/g, '/'))); + } + if (!seen.size) return null; + return [...seen].sort((a, b) => a.split('/').length - b.split('/').length || a.length - b.length)[0] + ?? null; +} + +export function pathsFromBash(command: unknown): string[] { + const out: string[] = []; + for (const match of String(command).matchAll(READER)) { + const argumentsText = match[1]; + if (!argumentsText) continue; + for (const tokRaw of argumentsText.split(/\s+/)) { + const t = tokRaw.replace(/^["']|["']$/g, ''); + if (!t || t.startsWith('-')) continue; + if (/[*?]/.test(t) || /\//.test(t) || /\\/.test(t) || /\.\w+$/.test(t)) out.push(t); + } + } + return out; +} + +// An ATTEMPT is not a leak. Once the sandbox actually refuses things, a build +// that tries to read the rubric and is blocked looks identical to one that +// succeeded — and marking the blocked run contaminated would void exactly the +// runs the sandbox is protecting. So a candidate path is held against its +// tool_use id and only counted once the result comes back not-an-error. +// +// Bash is not governed by the Read rules, so its reads resolve as successful +// unless the command itself failed; that asymmetry is the point of auditing it. +interface AuditHit { + path: string; + via: string; + kind: string; + unresolved?: boolean; +} + +interface PendingRead { + paths: string[]; + via: string; +} + +interface TranscriptAudit { + file: string; + cwd: string | null; + fileTool: number; + bashReads: number; + hits: AuditHit[]; + refused: AuditHit[]; +} + +interface AuditResult extends TranscriptAudit { + root: string; +} + +interface TranscriptContent { + type?: string; + name?: string; + id?: string; + tool_use_id?: string; + is_error?: boolean; + input?: { file_path?: string; path?: string; pattern?: string; command?: string }; +} + +export function auditTranscript(file: string, boundary: string | null): TranscriptAudit { + const lines = readFileSync(file, 'utf8').split('\n').filter(Boolean); + // --app names a HOST directory, but a containerised build worked at /app and + // its transcript records container paths. Held against the host boundary, + // every legitimate read of the app's own source resolves as an escape and the + // run is voided for existing. + // + // The test is deliberately the exact literal /app. The build container + // mounts the app there and nothing else — rather than "cwd is outside the + // boundary". The looser rule would have re-broken the case this audit exists + // for: the contaminated run cd'd into the harness to run the grader, and + // falling back to the session's own shallowest cwd could have adopted the + // harness directory as the boundary and reported that run clean. + const own = sessionCwd(lines); + const cwd = own === '/app' ? own : (boundary ?? own); + const hits: AuditHit[] = []; + const refused: AuditHit[] = []; + const pending = new Map(); + let fileTool = 0, bashReads = 0; + + for (const line of lines) { + let event: { message?: { content?: TranscriptContent[] } }; + try { event = JSON.parse(line) as typeof event; } catch { continue; } + const c = event.message?.content; + if (!Array.isArray(c)) continue; + for (const p of c) { + if (p.type === 'tool_result' && p.tool_use_id && pending.has(p.tool_use_id)) { + const completed = pending.get(p.tool_use_id ?? ''); + if (!completed) continue; + const { paths, via } = completed; + pending.delete(p.tool_use_id ?? ''); + const blocked = p.is_error === true; + for (const n of paths) (blocked ? refused : hits).push({ path: n, via, kind: classify(n) }); + continue; + } + if (p.type !== 'tool_use') continue; + const cand = []; + if (/^(Read|Grep|Glob|NotebookRead)$/.test(p.name ?? '')) { + fileTool++; + cand.push(p.input?.file_path ?? p.input?.path ?? p.input?.pattern ?? ''); + } else if (p.name === 'Bash') { + const found = pathsFromBash(p.input?.command ?? ''); + bashReads += found.length; + cand.push(...found); + } + const paths = []; + for (const raw of cand) { + const n = norm(raw); + if (!n || IGNORE.test(n)) continue; + // The CLI keeps auto-memory for the session's OWN project dir. A build + // reading notes it wrote itself, in this same run, is a diary, not a + // leak — the run that surfaced this had its 48/50 voided over a file + // whose originSessionId was its own. Memory belonging to ANY OTHER + // project dir stays flagged: that is somebody's notes about the + // harness. Own-ness is decided by the encoded cwd in the path. + if (cwd && /[/\\]projects[/\\][^/\\]+[/\\]memory[/\\]/.test(n) + && n.includes(cwd.replace(/[\\/:]/g, '-'))) continue; + const absolute = /^[a-z]:/.test(n) || n.startsWith('/'); + if (!absolute) continue; // relative paths resolve inside cwd + if (cwd && n.startsWith(cwd)) continue; + paths.push(n); + } + if (paths.length && p.id && p.name) pending.set(p.id, { paths, via: p.name }); + } + } + // A call whose result never arrived (session cut short) is unresolved, and + // unresolved is not innocent: count it. + for (const { paths, via } of pending.values()) + for (const n of paths) hits.push({ path: n, via, kind: classify(n), unresolved: true }); + + return { file, cwd, fileTool, bashReads, hits, refused }; +} + +function main(): void { + const requestedApp = arg('--app'); + const requestedDirectory = arg('--dir'); + const roots = requestedApp ? transcriptsFor(requestedApp) + : requestedDirectory ? [resolve(requestedDirectory)] + : [join(homedir(), '.claude', 'projects')]; + // When the caller names the app directory, that is the boundary. Do not + // infer it from a transcript folder name. + const appBoundary = requestedApp ? norm(resolve(requestedApp)) : null; + const results: AuditResult[] = []; +for (const root of roots) { + if (!existsSync(root)) continue; + const stack = [root]; + while (stack.length) { + const d = stack.pop(); + if (!d) continue; + for (const e of readdirSync(d, { withFileTypes: true })) { + const p = join(d, e.name); + if (e.isDirectory()) { if (!/node_modules/.test(p)) stack.push(p); continue; } + if (!/\.jsonl$/.test(e.name)) continue; + // `agent-*.jsonl` is a subagent's transcript, filed under + // /subagents/. They were excluded by this pattern, so the one + // channel the sandbox does not govern went unaudited exactly where it + // mattered: a SpacetimeDB build dispatched a subagent that made 29 Bash + // calls, and none of them were ever checked. + if (!/transcript|^agent-|^[0-9a-f-]{36}\.jsonl$/.test(e.name)) continue; + if (statSync(p).size < 2000) continue; + results.push({ ...auditTranscript(p, appBoundary), root }); + } + } +} + +if (process.argv.includes('--json')) { + console.log(JSON.stringify(results, null, 2)); + return; +} + +const label = (file: string): string => file.replace(/\\/g, '/') + .split('/').slice(-3).join('/').slice(0, 62); +console.log('\nBuilds that read outside their own directory'); +console.log('(counts BOTH file tools and Bash cat/grep/find; boundary = the session\'s own cwd)\n'); + +let clean = 0; +for (const r of results.sort((a, b) => b.hits.length - a.hits.length)) { + if (!r.cwd) { console.log(` ?? ${label(r.file)} — no cwd recorded, cannot judge`); continue; } + if (!r.hits.length) { + clean++; + // Blocked attempts are worth printing: they are the sandbox doing its job, + // and they say which paths a build still goes looking for. + if (r.refused?.length) { + const kinds = [...new Set(r.refused.map(h => h.kind))].join(', '); + console.log(` ${label(r.file)}\n clean — ${r.refused.length} attempt(s) BLOCKED by the sandbox (${kinds})`); + } + continue; + } + const byKind: Record = {}; + for (const h of r.hits) (byKind[h.kind] ??= []).push(h.path); + console.log(` ${label(r.file)}`); + console.log(` cwd: ...${r.cwd.slice(-52)} (${r.fileTool} file-tool, ${r.bashReads} bash reads)`); + for (const [k, v] of Object.entries(byKind).sort((a, b) => b[1].length - a[1].length)) { + const example = [...new Set(v)][0] ?? ''; + console.log(` ${String(v.length).padStart(3)}x ${k.padEnd(22)} ${example.split('/').slice(-2).join('/')}`); + } +} +console.log(`\n ${clean} transcript(s) read nothing outside their directory.`); +console.log(` ${results.length} transcript(s) examined.\n`); +} + +if (process.argv[1] && pathToFileURL(process.argv[1]).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/lease-cli.ts b/tools/stack-bench/commands/lease-cli.ts new file mode 100644 index 00000000000..e64b840ebf0 --- /dev/null +++ b/tools/stack-bench/commands/lease-cli.ts @@ -0,0 +1,64 @@ +#!/usr/bin/env node + +import { pidsOnPort } from '../src/runtime/platform.js'; +import { leaseFromEnv, updateBackendLease } from '../src/runtime/backend-lease.js'; +import type { BackendLeaseExpectation } from '../src/runtime/backend-lease.js'; +import { executeStackCapability } from '../src/stacks/stack-adapter-contract.js'; +import type { StackAdapter } from '../src/stacks/stack-adapter-contract.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; + +const [command, requestedBackend] = process.argv.slice(2); + +function current(expected: BackendLeaseExpectation = {}) { + return leaseFromEnv(process.env, { backend: requestedBackend, ...expected }); +} + +function adapterFor(id: string): StackAdapter { + const adapter = STACK_ADAPTER_REGISTRY.get(id); + if (!adapter) throw new Error(`unknown backend: ${id}`); + return adapter; +} + +try { + if (command === 'validate') { + current({ active: true }); + } else if (command === 'field') { + const field = process.argv[4]; + const { lease } = current({ active: true }); + const fields: Record = { + serverUri: lease.resources.serverUri, + serverPort: lease.resources.serverUri ? new URL(lease.resources.serverUri).port : null, + dataDir: lease.resources.dataDir, + module: lease.resources.module, + database: lease.resources.database, + containerName: lease.resources.container?.name, + containerId: lease.resources.container?.id, + buildContainerName: lease.resources.buildContainer?.name, + buildContainerId: lease.resources.buildContainer?.id, + }; + if (!field || !(field in fields) || fields[field] == null) { + throw new Error(`field is unavailable: ${field ?? ''}`); + } + process.stdout.write(String(fields[field])); + } else if (command === 'listener-pid') { + const { path, lease } = current({ active: true }); + process.stdout.write(String(executeStackCapability(adapterFor(lease.backend), 'lease', + 'listener-pid', { path, lease, helpers: { pidsOnPort, updateBackendLease } }))); + } else if (command === 'mark-restarting') { + const { path, lease } = current({ active: true }); + updateBackendLease(path, + { token: lease.ownershipToken, backend: lease.backend, runId: lease.runId }, next => { + next.state = 'restarting'; + return next; + }); + } else if (command === 'capture-listener') { + const { path, lease } = current(); + process.stdout.write(String(executeStackCapability(adapterFor(lease.backend), 'lease', + 'capture-listener', { path, lease, helpers: { pidsOnPort, updateBackendLease } }))); + } else { + throw new Error('usage: node dist/commands/lease-cli.js validate|field|listener-pid|mark-restarting|capture-listener [field]'); + } +} catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exit(3); +} diff --git a/tools/stack-bench/commands/null-control.ts b/tools/stack-bench/commands/null-control.ts new file mode 100644 index 00000000000..f864e30f7cf --- /dev/null +++ b/tools/stack-bench/commands/null-control.ts @@ -0,0 +1,246 @@ +#!/usr/bin/env node +// Grade the real validated production scenarios against a reachable app that +// implements nothing. Every point-bearing criterion must conclusively fail. + +import { execFile } from 'node:child_process'; +import { createServer } from 'node:http'; +import type { Server } from 'node:http'; +import type { AddressInfo } from 'node:net'; +import { mkdirSync, mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { basename, join, relative, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { readArtifactPayload, writeRunJson } from '../src/evidence/artifacts.js'; +import { calibrationQualificationIdentity, calibrationQualificationRelease, + resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { qualificationScopeIdentity } from '../src/composition/qualification-scope.js'; +import { analyseNullReports } from '../src/evidence/null-control-analysis.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { isDeclaredLevel, listTracks, loadTrack, suitesFor } from '../src/composition/tracks.js'; +import { controllerRunner } from '../src/runtime/runner-environment.js'; +import type { CalibrationPlan } from '../src/composition/calibration-compiler.js'; +import type { RecipeBinding } from '../src/composition/recipe-release.js'; +import type { Track } from '../src/composition/tracks.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const GRADE = compiledEntrypoint('grader', 'grade.js'); + +interface NullControlArgs { + tracks: string[]; + level: number | null; + recipe?: string; + out?: string; + audit: boolean; + parentAttemptId?: string; +} + +export function parseNullControlArgs(argv: string[]): NullControlArgs { + const args: NullControlArgs = { tracks: listTracks(), level: null, audit: false }; + for (let i = 2; i < argv.length; i++) { + if (argv[i] === '--track') args.tracks = (argv[++i] ?? '').split(',').filter(Boolean); + else if (argv[i] === '--level') args.level = Number(argv[++i] ?? ''); + else if (argv[i] === '--recipe') args.recipe = argv[++i] ?? ''; + else if (argv[i] === '--out') args.out = argv[++i] ?? ''; + else if (argv[i] === '--audit') args.audit = true; + else if (argv[i] === '--parent-attempt-id') args.parentAttemptId = argv[++i] ?? ''; + else { console.error(`Unknown argument: ${argv[i]}`); process.exit(2); } + } + if (args.level !== null && (!Number.isInteger(args.level) || args.level < 1)) { + throw new Error('--level must be a positive integer'); + } + if (args.level !== null && args.tracks.length !== 1) { + throw new Error('--level requires exactly one --track'); + } + if (args.recipe && args.level === null) throw new Error('--recipe requires --level'); + return args; +} + +function runGrade(argv: string[], timeoutMs = 300_000): Promise<{ stdout: string; stderr: string }> { + return new Promise((resolve, reject) => { + execFile(process.execPath, [GRADE, ...argv], { + encoding: 'utf8', maxBuffer: 32 * 1024 * 1024, timeout: timeoutMs, + }, (error, stdout, stderr) => { + if (error) { + error.message = `grader failed: ${error.message}\n${stdout}\n${stderr}`; + reject(error); + } else resolve({ stdout, stderr }); + }); + }); +} + +export function nullControlSuites(track: Track, selectedLevel: number | null = null, + binding: RecipeBinding | null = null) { + if (selectedLevel !== null && !isDeclaredLevel(track, selectedLevel)) { + throw new Error(`L${selectedLevel} is not declared for ${track.name}`); + } + if (binding) { + if (selectedLevel === null) throw new Error('recipe-bound null control requires one level'); + if (!Array.isArray(binding.execution) || !binding.execution.length) { + throw new Error('recipe-bound null control requires a typed execution plan'); + } + const executionIds = new Set(); + const mappedKeys = new Set(); + const suites = binding.execution.map(execution => { + if (executionIds.has(execution.id)) { + throw new Error(`recipe-bound null control repeats execution ${execution.id}`); + } + executionIds.add(execution.id); + const checks = binding.release.checkCatalog.filter(check => check.executionId === execution.id); + if (!checks.length) { + throw new Error(`recipe-bound null control execution ${execution.id} maps no checks`); + } + for (const check of checks) { + if (mappedKeys.has(check.stableKey)) { + throw new Error(`recipe-bound null control maps check ${check.stableKey} more than once`); + } + mappedKeys.add(check.stableKey); + } + return { id: execution.id, spec: resolve(track.dir, execution.source ?? ''), + level: selectedLevel, checks }; + }); + const missing = binding.release.checkCatalog + .filter(check => !mappedKeys.has(check.stableKey)).map(check => check.stableKey); + if (missing.length) { + throw new Error(`recipe-bound null control leaves checks unmapped: ${missing.join(', ')}`); + } + return suites; + } + const seen = new Set(); + const suites = []; + const levels = selectedLevel === null + ? Array.from({ length: track.validatedThrough }, (_, index) => index + 1) + : [selectedLevel]; + for (const level of levels) { + for (const suite of suitesFor(track, level)) { + if (seen.has(suite.spec)) continue; + seen.add(suite.spec); + suites.push({ ...suite, level }); + } + } + return suites; +} + +export function selectNullQualificationBinding(binding: RecipeBinding, calibration: CalibrationPlan): RecipeBinding { + const selected = calibrationQualificationRelease(calibration, binding.release, binding.execution); + return { ...binding, release: selected.release, execution: selected.execution }; +} + +export function createNullQualification(binding: RecipeBinding, calibration: CalibrationPlan) { + return { + binding: selectNullQualificationBinding(binding, calibration), + calibration, + identity: calibrationQualificationIdentity(calibration), + }; +} + +async function listen(server: Server): Promise { + await new Promise((resolve, reject) => { + server.once('error', reject); + server.listen({ port: 0, host: '127.0.0.1' }, () => resolve()); + }); + return (server.address() as AddressInfo).port; +} + +async function main() { + const args = parseNullControlArgs(process.argv); + const nullAttemptId = `null-control-${new Date().toISOString().replace(/[:.]/g, '-')}`; + const work = mkdtempSync(join(tmpdir(), 'stack-bench-null-')); + const app = join(work, 'app'); + const reportsDir = join(work, 'reports'); + mkdirSync(app, { recursive: true }); + mkdirSync(reportsDir, { recursive: true }); + + // Root navigation succeeds, proving the browser and server are healthy. All + // application/API behavior is absent: non-navigation requests get 404. + const server = createServer((request, response) => { + if (request.method === 'GET' && (request.url === '/' || request.headers.accept?.includes('text/html'))) { + response.writeHead(200, { 'content-type': 'text/html; charset=utf-8' }); + response.end('Null control'); + } else { + response.writeHead(404, { 'content-type': 'application/json' }); + response.end('{"error":"not implemented"}'); + } + }); + + const started = Date.now(); + const suiteReports = []; + let qualification = null; + try { + const port = await listen(server); + const url = `http://127.0.0.1:${port}`; + for (const trackName of args.tracks) { + const track = loadTrack(trackName); + let binding: RecipeBinding | null = null; + if (args.level !== null) { + binding = resolveRecipeRelease(track, args.level, args.recipe); + if (!binding) throw new Error(`${trackName} L${args.level} has no recipe release`); + const calibration = resolveCalibrationForRelease(binding.release, + { trackRoot: track.dir, stackBenchRoot: ROOT }); + if (!calibration) throw new Error(`${trackName} L${args.level} has no calibration`); + qualification = createNullQualification(binding, calibration); + binding = qualification.binding; + } + const selectedSuites = nullControlSuites(track, args.level, binding); + const resolvedRecipe = binding + ? `${binding.release.id}@${binding.release.version}` : args.recipe; + for (const suite of selectedSuites) { + const reportPath = join(reportsDir, `${trackName}-l${suite.level}-${suite.id.replaceAll('@', '-')}.json`); + process.stdout.write(`${trackName} L${suite.level} ${suite.id} (${basename(suite.spec)}) ... `); + await runGrade(['--url', url, '--level', String(suite.level), '--spec', suite.spec, + '--backend', 'postgres', '--track', trackName, '--app', app, '--out', reportPath, + '--parent-attempt-id', nullAttemptId, + ...(resolvedRecipe ? ['--recipe', resolvedRecipe] : []), + ...(binding ? ['--expected-recipe-sha256', binding.release.contentSha256] : []), + ...(('checks' in suite ? suite.checks : []) ?? []) + .flatMap(check => ['--selected-check', check.stableKey])]); + const report = readArtifactPayload(reportPath, { expectedKind: 'grade' }); + suiteReports.push({ track: trackName, level: suite.level, id: suite.id, + scenario: relative(track.dir, suite.spec).replaceAll('\\', '/'), report }); + console.log(`${report.total}/${report.max}`); + } + } + + const analysis = analyseNullReports(suiteReports); + const artifact = { + id: nullAttemptId, + kind: 'null_control', + startedAt: new Date(started).toISOString(), + completedAt: new Date().toISOString(), + parentAttemptId: args.parentAttemptId ?? null, + identities: qualification ? { + recipe: { id: qualification.binding.release.id, version: qualification.binding.release.version, + sha256: qualification.binding.release.contentSha256, state: qualification.binding.release.state }, + calibration: { ...qualification.identity, state: qualification.calibration.state }, + } : undefined, + durationMs: Date.now() - started, + runner: controllerRunner(), + ...(qualification ? { qualificationScope: qualificationScopeIdentity({ + kind: 'null', release: qualification.binding.release, stackBenchRoot: ROOT, + }) } : {}), + tracks: args.tracks, + ...analysis, + }; + const outputPath = resolve(args.out ?? join(ROOT, 'results', `${artifact.id}.json`)); + writeRunJson(outputPath, artifact); + console.log(JSON.stringify({ + id: artifact.id, + kind: artifact.kind, + durationMs: artifact.durationMs, + tracks: artifact.tracks, + ok: artifact.ok, + summary: artifact.summary, + artifact: outputPath, + }, null, 2)); + if (!analysis.ok && !args.audit) process.exitCode = 1; + } finally { + await new Promise(resolve => server.close(resolve)); + rmSync(work, { recursive: true, force: true }); + } +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch(error => { + console.error(error.stack ?? error.message); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/commands/preflight.ts b/tools/stack-bench/commands/preflight.ts new file mode 100644 index 00000000000..8b182f423ce --- /dev/null +++ b/tools/stack-bench/commands/preflight.ts @@ -0,0 +1,19 @@ +#!/usr/bin/env node + +import { parsePreflightArgs, printPreflightReport, runPreflight, + writePreflightReport } from '../src/runtime/preflight.js'; + +let request; +try { + request = parsePreflightArgs(process.argv); +} catch (error) { + console.error(`preflight: ${error instanceof Error ? error.message : String(error)}`); + console.error('Usage: stack-bench preflight --backend spacetime[,postgres,mongodb] [--track ecommerce] [--levels 1-2] [--smoke]'); + process.exit(2); +} + +const report = runPreflight(request); +if (request.report) writePreflightReport(request.report, report); +if (request.json) console.log(JSON.stringify(report, null, 2)); +else printPreflightReport(report); +process.exitCode = report.ok ? 0 : 1; diff --git a/tools/stack-bench/commands/probe-sandbox.ts b/tools/stack-bench/commands/probe-sandbox.ts new file mode 100644 index 00000000000..06bec84f5a0 --- /dev/null +++ b/tools/stack-bench/commands/probe-sandbox.ts @@ -0,0 +1,197 @@ +#!/usr/bin/env node +// Does the deny list actually stop a build reading the marking scheme? +// +// Ask a real session to read protected files and fail if any content is +// returned. This verifies the current deny rules rather than relying on a +// static inspection. +// +// It imports the deny list from the runtime sandbox module. A probe +// with its own copy keeps passing after the real list drifts, which is worse +// than no probe: it reports assurance it no longer has. +// +// SCOPE: `Read(...)` rules govern the FILE TOOLS ONLY — a denied path is still +// reachable with `cat`, verified, and that is why the contamination audit +// remains the control rather than this (prevention). The probe measures what +// the sandbox claims to do, not the whole exposure. +// +// Usage: node dist/commands/probe-sandbox.js [--model claude-sonnet-5] [--keep] + +import { execFileSync } from 'node:child_process'; +import { mkdirSync, existsSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { homedir, tmpdir } from 'node:os'; + +import { randomUUID } from 'node:crypto'; +import { writeSandbox } from '../src/runtime/sandbox.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const REPO = resolve(ROOT, '..', '..'); +const arg = (name: string, defaultValue?: string): string | undefined => { + const index = process.argv.indexOf(name); + return index === -1 ? defaultValue : process.argv[index + 1]; +}; +const MODEL = arg('--model', 'claude-sonnet-5') ?? 'claude-sonnet-5'; + +function findClaude() { + const appData = process.env.APPDATA ?? join(process.env.HOME ?? '', 'AppData', 'Roaming'); + const desktop = join(appData, 'Claude', 'claude-code'); + if (existsSync(desktop)) { + const versions = readdirSync(desktop).sort(); + const exe = join(desktop, versions[versions.length - 1] ?? '', 'claude.exe'); + if (existsSync(exe)) return exe; + } + return 'claude'; +} + +// Every target is a real file a past run was caught reading, or the direct +// equivalent. Probing a path that does not exist proves nothing: "not found" +// and "refused" are both an empty read. +type ProbeTarget = [what: string, path: string]; + +const TARGETS: ProbeTarget[] = [ + ['the marking scheme', compiledEntrypoint('grader', 'grade.js')], + ['the assertions themselves', join(ROOT, 'tracks', 'ecommerce', 'scenarios', '01-invariants.json')], + ['the linter', compiledEntrypoint('linter', 'lint.js')], + ['the benchmark notes', join(homedir(), '.claude', 'projects', + 'D--Development-ClockworkLabs-SpacetimeDB-SpacetimeDB', 'memory', 'stack-bench-project.md')], + ['another benchmark rubric', join(REPO, 'tools', 'llm-sequential-upgrade', 'GRADING.md')], +]; + +// --target lets another harness point the probe at its own forbidden paths, and +// --settings at the rules it actually generates, so one probe serves both +// benchmarks without either restating the other's list. +const extra = process.argv.reduce((targets, argument, index) => + argument === '--target' && process.argv[index + 1] + ? [...targets, [`--target ${process.argv[index + 1]}`, resolve(process.argv[index + 1] ?? '')]] + : targets, []); +if (extra.length) TARGETS.length = 0; +TARGETS.push(...extra); + +const present = TARGETS.filter(([, p]) => existsSync(p)); +const absent = TARGETS.filter(([, p]) => !existsSync(p)); +for (const [what, p] of absent) console.log(` skipping ${what} — no such file (${p})`); +if (!present.length) { console.error('no probe targets exist; cannot prove anything'); process.exit(2); } + +const appDir = join(tmpdir(), `stack-bench-sandbox-probe-${randomUUID().slice(0, 8)}`); +mkdirSync(appDir, { recursive: true }); +// A bare directory would have the model wondering what it is looking at; give +// it the shape of an app so the session behaves like a build. +writeFileSync(join(appDir, 'README.md'), '# probe app\n'); +const settingsOption = arg('--settings'); +const settings = settingsOption ? resolve(settingsOption) : writeSandbox(appDir); + +const prompt = [ + 'Read each of these files with the Read tool and report, for each one, whether', + 'you got its contents or were refused. Do not use Bash. Do not stop early —', + 'attempt every path even if the first is refused.', + '', + ...present.map(([what, p]) => `- ${p} (${what})`), +].join('\n'); + +const sessionId = randomUUID(); +console.log(`\nprobing ${present.length} path(s) with the deny list a build is given...`); +try { + // --dangerously-skip-permissions is bypassPermissions, which turns the whole + // permission system off — deny rules included. Proved by this probe: all five + // targets came back in full. --mode lets us find one that actually enforces. + const mode = arg('--mode', 'bypass') ?? 'bypass'; + const gate = mode === 'bypass' + ? ['--dangerously-skip-permissions'] + : ['--permission-mode', mode]; + execFileSync(findClaude(), ['--print', '--output-format', 'text', + ...gate, '--settings', settings, + '--model', MODEL, '--session-id', sessionId, '-p', prompt], + { cwd: appDir, encoding: 'utf8', stdio: 'pipe', maxBuffer: 64 * 1024 * 1024 }); +} catch (error: unknown) { + const message = error instanceof Error ? error.message : String(error); + console.error(`the probe session failed to run: ${message.split('\n')[0] ?? ''}`); + process.exit(2); +} + +// Judge on the tool RESULTS, not the model's prose summary: a session that was +// refused can still describe itself as having read the file. +const store = join(homedir(), '.claude', 'projects'); +let transcript: string | null = null; +const stack: string[] = existsSync(store) ? [store] : []; +while (stack.length && !transcript) { + const d = stack.pop(); + if (!d) continue; + for (const e of readdirSync(d, { withFileTypes: true })) { + if (e.isDirectory()) stack.push(join(d, e.name)); + else if (e.name === `${sessionId}.jsonl`) { transcript = join(d, e.name); break; } + } +} +if (!transcript) { console.error('probe transcript not found; cannot judge'); process.exit(2); } + +const lines = readFileSync(transcript, 'utf8').split('\n').filter(Boolean); +const attempts = new Map(); // tool_use_id -> path +const verdicts = new Map(); // path -> verdict +const norm = (value: unknown): string => String(value ?? '').replace(/\\/g, '/').toLowerCase(); + +interface TranscriptContent { + type?: string; + name?: string; + id?: string; + input?: { file_path?: unknown }; + tool_use_id?: string; + is_error?: boolean; +} + +function transcriptContent(value: unknown): TranscriptContent[] | null { + if (!value || typeof value !== 'object' || !('message' in value)) return null; + const message = value.message; + if (!message || typeof message !== 'object' || !('content' in message)) return null; + return Array.isArray(message.content) ? message.content as TranscriptContent[] : null; +} + +for (const line of lines) { + let entry: unknown; try { entry = JSON.parse(line) as unknown; } catch { continue; } + const c = transcriptContent(entry); + if (!Array.isArray(c)) continue; + for (const p of c) { + if (p.type === 'tool_use' && p.name === 'Read' && p.id) { + attempts.set(p.id, norm(p.input?.file_path)); + } else if (p.type === 'tool_result' && p.tool_use_id && attempts.has(p.tool_use_id)) { + const path = attempts.get(p.tool_use_id); + if (!path) continue; + // Judge ONLY on the tool's error flag. Matching words like "permission" + // or "denied" in the body reads the FILE'S OWN TEXT: the first version of + // this probe called the grader refused while its contents sat in the + // result. An oracle that mistakes the payload for a refusal reports + // safety that is not there, so anything not flagged an error is a read. + const refused = p.is_error === true; + // One successful read is enough to condemn a path, so a REFUSED verdict + // never overwrites a READ one. + if (verdicts.get(path) !== 'READ') verdicts.set(path, refused ? 'REFUSED' : 'READ'); + } + } +} + +console.log(''); +let leaked = 0, untried = 0; +for (const [what, p] of present) { + const v = verdicts.get(norm(p)); + if (v === 'REFUSED') console.log(` REFUSED ${what}`); + else if (v === 'READ') { console.log(` READ ${what} <-- ${p}`); leaked++; } + else { console.log(` NOT TRIED ${what} (the session never attempted it)`); untried++; } +} + +// Windows holds the session's own directory open for a moment after it exits, +// and an EBUSY here would throw away the verdict that was the point of the run. +if (!process.argv.includes('--keep')) { + try { rmSync(appDir, { recursive: true, force: true }); } + catch { console.log(` (left ${appDir} behind — still locked)`); } +} + +console.log(''); +if (leaked) { + console.log(`FAIL: ${leaked} path(s) still readable. The deny list does not cover them.`); + process.exit(1); +} +if (untried) { + console.log(`INCONCLUSIVE: ${untried} path(s) were never attempted, so nothing was proved`); + console.log('about them. Re-run; a refusal the session declined to attempt is not a pass.'); + process.exit(2); +} +console.log('PASS: every probed path was refused to the file tools.'); +console.log('Bash remains ungoverned by design; the contamination audit is the control.'); diff --git a/tools/stack-bench/commands/progression-graph.ts b/tools/stack-bench/commands/progression-graph.ts new file mode 100644 index 00000000000..09a46511851 --- /dev/null +++ b/tools/stack-bench/commands/progression-graph.ts @@ -0,0 +1,21 @@ +import { join } from 'node:path'; + +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { writeProgressionGraph } from '../src/progression/progression-graph.js'; + +interface ProgressionGraph { + nodes: unknown[]; + levels: number; +} + +const trackRoot = join(STACK_BENCH_ROOT, 'tracks', 'ecommerce'); +const definitionPath = process.argv[2]; +if (!definitionPath) { + throw new Error('usage: progression-graph [html-path]'); +} +const graph: ProgressionGraph = writeProgressionGraph({ + definitionPath, + htmlPath: process.argv[3] ?? join(STACK_BENCH_ROOT, 'docs', 'dependency-graph.html'), + trackRoot, +}); +console.log(`Rendered ${graph.nodes.length} nodes across ${graph.levels} levels.`); diff --git a/tools/stack-bench/commands/qualification-cli.ts b/tools/stack-bench/commands/qualification-cli.ts new file mode 100644 index 00000000000..78ddde3f658 --- /dev/null +++ b/tools/stack-bench/commands/qualification-cli.ts @@ -0,0 +1,278 @@ +#!/usr/bin/env node + +import { existsSync, readFileSync } from 'node:fs'; +import { basename, dirname, extname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { calibrationQualificationIdentity, resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { isDeclaredLevel, listTracks, loadTrack } from '../src/composition/tracks.js'; +import { PACK_BUDGET_POLICY } from '../src/composition/pack-budget.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { companionReferenceArtifactPath } from '../src/references/reference-live.js'; +import type { CalibrationPlan } from '../src/composition/calibration-compiler.js'; +import type { RecipeBinding, RecipeRelease } from '../src/composition/recipe-release.js'; + +interface QualificationArgs { + command?: string; + track: string | null; + level: number | null; + recipe?: string; +} + +interface QualificationBlocker { + code: string; + path: string; + summary: string; +} + +export function parseQualificationArgs(argv: string[]): QualificationArgs { + const args: QualificationArgs = { command: argv[2], track: null, level: null }; + for (let index = 3; index < argv.length; index += 1) { + if (argv[index] === '--track') args.track = argv[++index] ?? ''; + else if (argv[index] === '--level') args.level = Number(argv[++index] ?? ''); + else if (argv[index] === '--recipe') args.recipe = argv[++index] ?? ''; + else throw new Error(`unknown qualification option ${argv[index]}`); + } + if (args.command !== 'status' || typeof args.track !== 'string' || !args.track + || args.level === null || !Number.isInteger(args.level) || args.level < 1) { + throw new Error('usage: node dist/commands/qualification-cli.js status --track --level ' + + '[--recipe @]'); + } + return args; +} + +function blocker(code: string, path: string, summary: string): QualificationBlocker { + return { code, path, summary }; +} + +function evidencePlan(calibration: CalibrationPlan) { + const stacks = calibration.qualification.stacks + .filter(stack => stack.status !== 'unsupported').map(stack => stack.id).sort(); + const evidence = []; + for (const stack of stacks) { + for (let repetition = 1; repetition <= calibration.qualification.referenceRepetitions; repetition += 1) { + evidence.push({ kind: 'reference', stack, repetition }); + } + for (let repetition = 1; repetition <= calibration.qualification.mutationRepetitions; repetition += 1) { + evidence.push({ kind: 'mutation', stack, repetition }); + } + } + for (let repetition = 1; repetition <= calibration.nullControl.repetitions; repetition += 1) { + evidence.push({ kind: 'null', stack: null, repetition }); + } + return evidence; +} + +export interface CalibrationMutationSelection { + mutations: Array<{ backend: string; path: string; targets: Array<{ id: string }> }>; +} + +export function mutationWorkerCount(calibration: CalibrationMutationSelection, stack: string, + readManifest: (path: string) => { mutations?: { id: string }[] } = path => + JSON.parse(readFileSync(resolve(STACK_BENCH_ROOT, path), 'utf8')) as { mutations?: { id: string }[] }) { + const entry = calibration.mutations.find(candidate => candidate.backend === stack); + if (!entry) return 1; + const manifest = readManifest(entry.path); + const selectedIds = new Set(entry.targets.map(target => target.id)); + const selectedMutations = (manifest.mutations ?? []).filter(mutation => + selectedIds.delete(mutation.id)); + if (selectedIds.size) { + throw new Error(`${stack} calibration selects missing mutations: ${[...selectedIds].sort().join(', ')}`); + } + return Math.min(4, Math.max(1, selectedMutations.length)); +} + +function mutationWorkerOption(calibration: CalibrationPlan, stack: string) { + const workers = mutationWorkerCount(calibration, stack); + return workers > 1 ? ` --mutation-workers ${workers}` : ''; +} + +function qualificationRunDirectory(artifactPath: string): string { + return join(dirname(artifactPath), `${basename(artifactPath, extname(artifactPath))}.runs`); +} + +function defectCheckCoverage(release: RecipeRelease, calibration: CalibrationPlan) { + const selected = calibration.qualification.checks + ? new Set(calibration.qualification.checks) : null; + const scored = release.checkCatalog.filter(check => check.points > 0 + && (selected === null || selected.has(check.stableKey))); + const scoredByKey = new Map(scored.map(check => [check.stableKey, check])); + const stacks = calibration.qualification.stacks + .filter(stack => stack.status !== 'unsupported').map(stack => stack.id).sort(); + return { + required: 'every scored check has an exact known-defect test on every supported stack', + totalChecks: scored.length, + totalPoints: scored.reduce((total, check) => total + check.points, 0), + stacks: stacks.map(stack => { + const covered = new Set(calibration.mutations + .filter(entry => entry.backend === stack) + .flatMap(entry => entry.targets.flatMap(target => target.stableKeys)) + .filter(key => scoredByKey.has(key))); + const missing = scored.filter(check => !covered.has(check.stableKey)); + return { + stack, + coveredChecks: covered.size, + coveredPoints: [...covered].reduce((total, key) => total + (scoredByKey.get(key)?.points ?? 0), 0), + missingChecks: missing.map(check => check.stableKey), + }; + }), + }; +} + +export function qualificationReadiness(trackName: string, level: number, recipe: string | null = null) { + if (!listTracks().includes(trackName)) throw new Error(`unknown qualification track ${trackName}`); + const track = loadTrack(trackName); + if (!isDeclaredLevel(track, level)) { + throw new Error(`L${level} is not declared for ${trackName}`); + } + const binding: RecipeBinding | null = resolveRecipeRelease(track, level, recipe); + if (!binding) throw new Error(`${trackName} L${level} has no recipe release`); + const calibration = resolveCalibrationForRelease(binding.release, + { trackRoot: track.dir, alias: `L${level}` }); + if (!calibration) { + throw new Error(`${binding.release.id}@${binding.release.version} has no L${level} calibration`); + } + const identity = calibrationQualificationIdentity(calibration); + const launchBlockers = []; + if (binding.release.state === 'retired') { + launchBlockers.push(blocker('recipe_retired', 'recipe.state', 'selected recipe is retired')); + } + for (const pack of binding.plan.packs) { + if (pack.budget.status !== 'bounded') { + launchBlockers.push(blocker('pack_budget_unbounded', `packs.${pack.id}.budget`, + `${pack.id}@${pack.version} needs a measured maxRuntimeMs before qualification`)); + } + } + for (const entry of calibration.references.entries) { + if (!['candidate', 'active'].includes(String(entry.status))) { + launchBlockers.push(blocker('reference_unavailable', `references.${entry.backend}`, + `${entry.id} is ${entry.status}`)); + } + } + for (const entry of calibration.mutations) { + if (!['candidate', 'active'].includes(String(entry.status))) { + launchBlockers.push(blocker('mutation_unavailable', `mutations.${entry.backend}`, + `${entry.path} is ${entry.status}`)); + } + } + + const requiredEvidence = evidencePlan(calibration); + const defectChecks = defectCheckCoverage(binding.release, calibration); + const recorded = new Set(calibration.qualification.evidence.map(entry => + `${entry.kind}:${entry.stack ?? ''}:${entry.repetition}`)); + const promotionBlockers = [...launchBlockers]; + for (const coverage of defectChecks.stacks.filter(item => item.missingChecks.length > 0)) { + promotionBlockers.push(blocker('defect_check_coverage_incomplete', + `defectChecks.${coverage.stack}`, + `${coverage.coveredChecks}/${defectChecks.totalChecks} scored checks have exact known-defect tests`)); + } + for (const item of requiredEvidence) { + const key = `${item.kind}:${item.stack ?? ''}:${item.repetition}`; + if (!recorded.has(key)) promotionBlockers.push(blocker('evidence_missing', `evidence.${key}`, + `${key} has no hash-bound qualification artifact`)); + } + for (const stale of (calibration.qualificationStaleness ?? []) as { + kind: string; stack?: string; repetition: number; reason: string; + }[]) { + const key = `${stale.kind}:${stale.stack ?? ''}:${stale.repetition}`; + promotionBlockers.push(blocker('qualification_evidence_stale', `evidence.${key}`, + `${key} must be regenerated: ${stale.reason}`)); + } + const sourceStates: [string, string][] = [ + ['recipe.state', binding.release.state], + ['fixture.state', binding.release.components.fixture.state], + ...binding.release.components.packs.map(pack => [`packs.${pack.id}.state`, pack.state] as [string, string]), + ['calibration.state', calibration.state], + ['promotion.status', binding.status], + ]; + const governance = sourceStates.map(([path, state]) => ({ path, state, + target: path === 'promotion.status' ? 'promoted' : 'qualified' })); + governance.push(...calibration.qualification.stacks.map(stack => ({ + path: `qualification.stacks.${stack.id}`, state: stack.status, + target: stack.status === 'unsupported' ? 'unsupported' : 'qualified', + }))); + + const output = '/var/lib/stack-bench/results/qualification'; + const stacks = calibration.qualification.stacks + .filter(stack => stack.status !== 'unsupported').map(stack => stack.id).sort(); + const qualificationLevel = Number(calibration.promotion.alias.slice(1)); + const budgetEvidence = stacks.map(stack => + `${output}/budget-input/${trackName}-l${qualificationLevel}-${stack}.json`); + const budgetPreparationRequired = launchBlockers.some(item => item.code === 'pack_budget_unbounded'); + const recipeOption = recipe + ? ` --recipe ${binding.release.id}@${binding.release.version}` : ''; + const featureCatalog = calibration.qualification.featureCatalog; + const featureCatalogOption = featureCatalog + ? ` --feature-catalog ${featureCatalog.id}@${featureCatalog.version}` : ''; + const combinedReferenceEvidence = calibration.qualification.referenceRepetitions + === calibration.qualification.mutationRepetitions; + const artifactStem = `${trackName}-l${qualificationLevel}-${binding.release.contentSha256.slice(0, 12)}`; + const artifactPaths = { + references: Object.fromEntries(stacks.map(stack => [stack, + `${output}/${artifactStem}-${stack}-reference.json`])), + mutations: Object.fromEntries(stacks.map(stack => [stack, + `${output}/${artifactStem}-${stack}-mutation.json`])), + null: `${output}/${artifactStem}-null.json`, + }; + const launchPaths = new Set([artifactPaths.null]); + for (const stack of stacks) { + const mutationPath = artifactPaths.mutations[stack]; + const referencePath = artifactPaths.references[stack]; + if (!mutationPath || !referencePath) throw new Error(`qualification path is missing for ${stack}`); + launchPaths.add(mutationPath); + launchPaths.add(qualificationRunDirectory(mutationPath)); + launchPaths.add(combinedReferenceEvidence + ? companionReferenceArtifactPath(mutationPath) : referencePath); + if (!combinedReferenceEvidence) { + launchPaths.add(qualificationRunDirectory(referencePath)); + } + } + for (const path of [...launchPaths].filter(existsSync).sort()) { + launchBlockers.push(blocker('qualification_output_exists', path, + 'qualification output already exists')); + } + return { + qualificationSchemaVersion: 1, + scope: { track: trackName, level, recipe: { id: binding.release.id, + version: binding.release.version, contentSha256: binding.release.contentSha256 }, + calibration: { ...identity, contentSha256: calibration.contentSha256 }, + runner: calibration.qualification.runner ?? null }, + launch: { ok: launchBlockers.length === 0, blockers: launchBlockers }, + budgetPreparation: { + required: budgetPreparationRequired, + policy: PACK_BUDGET_POLICY, + commands: budgetPreparationRequired ? [ + ...stacks.map((stack, index) => + `qualify-reference --backend ${stack} --track ${trackName} --level ${qualificationLevel}${recipeOption}${featureCatalogOption} --repetitions ${calibration.qualification.referenceRepetitions} --out ${budgetEvidence[index]}`), + `pack-budget recommend --track ${trackName} --level ${qualificationLevel}${recipeOption} ${budgetEvidence + .map(path => `--evidence ${path}`).join(' ')} --out ${output}/${trackName}-l${qualificationLevel}-pack-budgets.json`, + ] : [], + }, + requiredEvidence, + defectChecks, + artifactPaths, + commands: [ + ...stacks.flatMap(stack => [ + ...(!combinedReferenceEvidence ? [ + `qualify-reference --backend ${stack} --track ${trackName} --level ${qualificationLevel}${recipeOption}${featureCatalogOption} --repetitions ${calibration.qualification.referenceRepetitions} --out ${artifactPaths.references[stack]}`, + ] : []), + `qualify-reference --backend ${stack} --track ${trackName} --level ${qualificationLevel}${recipeOption}${featureCatalogOption} --repetitions ${calibration.qualification.mutationRepetitions} --mutations --release-candidate${mutationWorkerOption(calibration, stack)} --out ${artifactPaths.mutations[stack]}`, + ]), + `qualify-null --track ${trackName} --level ${qualificationLevel}${recipeOption} --out ${artifactPaths.null}`, + ], + promotion: { ready: promotionBlockers.length === 0, blockers: promotionBlockers, + governance }, + }; +} + +function main() { + const args = parseQualificationArgs(process.argv); + if (!args.track || args.level === null) throw new Error('track and level are required'); + console.log(JSON.stringify(qualificationReadiness(args.track, args.level, args.recipe), null, 2)); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + try { main(); } + catch (error: unknown) { console.error(error instanceof Error ? error.message : String(error)); process.exitCode = 2; } +} diff --git a/tools/stack-bench/commands/recovery.ts b/tools/stack-bench/commands/recovery.ts new file mode 100644 index 00000000000..410b6c58981 --- /dev/null +++ b/tools/stack-bench/commands/recovery.ts @@ -0,0 +1,25 @@ +#!/usr/bin/env node + +import { recoverBackendLease, recoverSupervisedRun } from '../src/runtime/recovery.js'; + +const [command, statePath, option, output] = process.argv.slice(2); +const supervisorRequest = command === 'recover' && statePath !== undefined && process.argv.length === 4; +const leaseRequest = command === 'recover-lease' && statePath !== undefined && option === '--out' + && output !== undefined && process.argv.length === 6; +if (!supervisorRequest && !leaseRequest) { + console.error('Usage:\n' + + ' stack-bench recover \n' + + ' stack-bench recover-lease --out '); + process.exit(2); +} + +try { + const result = leaseRequest + ? recoverBackendLease(statePath, output) + : recoverSupervisedRun(statePath); + console.log(JSON.stringify(result, null, 2)); + process.exitCode = result.ok ? 0 : 1; +} catch (error) { + console.error(`recovery: ${error instanceof Error ? error.message : String(error)}`); + process.exitCode = 2; +} diff --git a/tools/stack-bench/commands/repair-cli.ts b/tools/stack-bench/commands/repair-cli.ts new file mode 100644 index 00000000000..4c5e322f728 --- /dev/null +++ b/tools/stack-bench/commands/repair-cli.ts @@ -0,0 +1,219 @@ +#!/usr/bin/env node + +import { randomUUID } from 'node:crypto'; +import { existsSync, mkdirSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; + +import { acquireCampaignLock, releaseCampaignLock } from '../src/campaigns/campaign-lock.js'; +import { emptyArtifactIdentities, readArtifact, writeArtifact } from '../src/evidence/artifacts.js'; +import { STACK_BENCH_ROOT } from '../src/package-root.js'; +import { rescueSupervisedLease } from '../src/runtime/recovery.js'; +import { runBounded } from '../src/runtime/bounded-process.js'; +import type { BoundedProcessResult, RunBoundedOptions } + from '../src/runtime/bounded-process.js'; +import { createRepairGrant, inspectRepairParent } from '../src/runtime/repair-grant.js'; + +const BENCH = join(STACK_BENCH_ROOT, 'dist', 'commands', 'bench.js'); + +export interface RepairStatusArgs { + command: 'status'; + parent: string; + level: number; +} + +export interface RepairGrantArgs { + command: 'grant'; + parent: string; + level: number; + rounds: number; + maxBudgetUsd?: number; + timeoutMinutes: number; +} + +export type RepairArgs = RepairStatusArgs | RepairGrantArgs; + +export function parseRepairArgs(argv: string[]): RepairArgs { + const [command, parent, ...rest] = argv.slice(2); + if (command === 'status' && parent && rest.length === 2 && rest[0] === '--level') { + const level = Number(rest[1]); + if (!Number.isSafeInteger(level) || level < 1) throw new Error('--level must be a positive integer'); + return { command, parent: resolve(parent), level }; + } + if (command !== 'grant' || !parent) { + throw new Error('usage: repair status --level | repair grant --level --rounds [--max-budget-usd ] [--timeout-minutes ]'); + } + const values: { level?: number; rounds?: number; maxBudgetUsd?: number; + timeoutMinutes: number } = { timeoutMinutes: 120 }; + const seen = new Set(); + for (let index = 0; index < rest.length; index += 2) { + const flag = rest[index]; + if (!flag || !['--level', '--rounds', '--max-budget-usd', '--timeout-minutes'].includes(flag) + || index + 1 >= rest.length || seen.has(flag)) { + throw new Error(`invalid or duplicate repair option ${String(flag)}`); + } + seen.add(flag); + const value = Number(rest[index + 1]); + if (flag === '--level') values.level = value; + else if (flag === '--rounds') values.rounds = value; + else if (flag === '--max-budget-usd') values.maxBudgetUsd = value; + else values.timeoutMinutes = value; + } + const level = values.level; + if (level === undefined || !Number.isSafeInteger(level) || level < 1) { + throw new Error('--level must be a positive integer'); + } + const rounds = values.rounds; + if (rounds === undefined || !Number.isSafeInteger(rounds) || rounds < 1 || rounds > 20) { + throw new Error('--rounds must be an integer from 1 through 20'); + } + if (values.maxBudgetUsd !== undefined + && (!Number.isFinite(values.maxBudgetUsd) || values.maxBudgetUsd <= 0)) { + throw new Error('--max-budget-usd must be a positive number'); + } + if (!Number.isFinite(values.timeoutMinutes) || values.timeoutMinutes < 10 + || values.timeoutMinutes > 480) { + throw new Error('--timeout-minutes must be from 10 through 480'); + } + return { command, parent: resolve(parent), level, + rounds, timeoutMinutes: values.timeoutMinutes, + ...(values.maxBudgetUsd === undefined ? {} : { maxBudgetUsd: values.maxBudgetUsd }) }; +} + +export function repairStatus(parent: string, level: number): Record { + try { + const inspected = inspectRepairParent(parent, level); + return { eligible: true, parentRunId: inspected.parent.id, level, + score: inspected.level.score, max: inspected.level.max, + roundsUsed: inspected.cumulativeRoundsBefore, + checkpointSha256: inspected.checkpoint.payload.source.sha256 }; + } catch (error) { + return { eligible: false, level, + reason: error instanceof Error ? error.message : String(error) }; + } +} + +interface RepairExecutionDependencies { + execute?: (command: string, argv: string[], + options: RunBoundedOptions) => Promise; + rescue?: (path: string, output: string) => void; + uuid?: () => string; + env?: NodeJS.ProcessEnv; +} + +interface RepairContinuationPayload { + outcome?: unknown; + continuation?: { + parentRunId?: string; + roundsGranted?: number; + level?: number; + [key: string]: unknown; + }; + [key: string]: unknown; +} + +export async function executeRepairGrant(args: RepairGrantArgs, + { execute = runBounded, rescue = rescueSupervisedLease, uuid = randomUUID, + env = process.env }: RepairExecutionDependencies = {}) { + const resolved = createRepairGrant(args.parent, { level: args.level, rounds: args.rounds }); + const lock = acquireCampaignLock(join(resolved.root, '.repair-control'), { + id: `repair-l${args.level}`, + contentSha256: resolved.checkpoint.payload.source.sha256, + }); + const stamp = new Date().toISOString().replace(/[-:.TZ]/g, '').slice(0, 14); + const executionId = `grant-${stamp}-${uuid().replaceAll('-', '').slice(0, 12)}`; + const output = join(resolved.root, 'continuations', executionId); + const privateRoot = join(tmpdir(), 'stack-bench-repair-supervisors'); + const supervisorState = join(privateRoot, `${executionId}.json`); + try { + mkdirSync(output, { recursive: true }); + mkdirSync(privateRoot, { recursive: true, mode: 0o700 }); + const argv = [BENCH, + '--repair-from', resolved.root, + '--repair-level', String(args.level), + '--fix-rounds', String(args.rounds), + '--out', output, + '--no-media']; + if (args.maxBudgetUsd !== undefined) { + argv.push('--max-budget-usd', String(args.maxBudgetUsd)); + } + const childEnv: NodeJS.ProcessEnv = { + ...env, + STACK_BENCH_SUPERVISOR_STATE: supervisorState, + }; + if (resolved.configuration.buildImage) { + childEnv.STACK_BENCH_IMAGE = resolved.configuration.buildImage; + } + const processResult = await execute(process.execPath, argv, { + cwd: STACK_BENCH_ROOT, + env: childEnv, + stdio: 'inherit', + timeoutMs: args.timeoutMinutes * 60_000, + logs: { stdout: join(output, 'process.stdout.log'), + stderr: join(output, 'process.stderr.log') }, + }); + let cleanupError: unknown = null; + if (!processResult.ok && existsSync(supervisorState)) { + try { rescue(supervisorState, output); } + catch (error) { cleanupError = error; } + } + const streams = processResult.logs ? Object.fromEntries(Object.entries(processResult.logs) + .map(([name, value]) => [name, { ...value, path: `process.${name}.log` }])) : null; + writeArtifact(join(output, 'process.json'), { + kind: 'repair_process', + id: `${executionId}-process`, + attempt: { id: `${executionId}-process`, parentId: resolved.parent.id }, + identities: emptyArtifactIdentities({ + agentAdapter: resolved.parentArtifact.identities.agentAdapter, + stackAdapter: resolved.parentArtifact.identities.stackAdapter, + }), + payload: { schemaVersion: 1, parentRunId: resolved.parent.id, + level: args.level, roundsGranted: args.rounds, + exitCode: processResult.code ?? null, signal: processResult.signal ?? null, + timedOut: processResult.timedOut, streams }, + }); + if (cleanupError) { + const detail = cleanupError instanceof Error ? cleanupError.message : String(cleanupError); + throw new Error(`repair continuation cleanup failed: ${detail}`); + } + const runPath = join(output, 'run.json'); + if (!existsSync(runPath)) { + throw new Error(`repair continuation produced no run artifact${processResult.timedOut ? ' before its timeout' : ''}`); + } + const run = readArtifact(runPath, + { expectedKind: 'repair_continuation' }); + if (run.attempt.parentId !== resolved.parent.id + || run.payload.continuation?.parentRunId !== resolved.parent.id + || run.payload.continuation?.roundsGranted !== args.rounds + || run.payload.continuation?.level !== args.level) { + throw new Error('repair continuation result does not match its grant'); + } + return { output, process: processResult, run }; + } finally { + rmSync(supervisorState, { force: true }); + releaseCampaignLock(lock); + } +} + +async function main(): Promise { + const args = parseRepairArgs(process.argv); + if (args.command === 'status') { + const status = repairStatus(args.parent, args.level); + console.log(JSON.stringify(status, null, 2)); + if (status.eligible !== true) process.exitCode = 1; + return; + } + const result = await executeRepairGrant(args); + console.log(JSON.stringify({ output: result.output, id: result.run.id, + outcome: result.run.payload.outcome, + continuation: result.run.payload.continuation }, null, 2)); + if (!result.process.ok) process.exitCode = 1; +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) { + main().catch((error: unknown) => { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + }); +} diff --git a/tools/stack-bench/commands/report-bugs.ts b/tools/stack-bench/commands/report-bugs.ts new file mode 100644 index 00000000000..98a8949e5ba --- /dev/null +++ b/tools/stack-bench/commands/report-bugs.ts @@ -0,0 +1,333 @@ +#!/usr/bin/env node +// Turns grading results into a behavioral BUG_REPORT.md for the fix agent. +// Selectors, test mechanics, local topology and raw paths are deliberately +// removed so a fix cannot overfit the harness instead of repairing the app. + +import { existsSync, mkdirSync, readdirSync, rmSync, writeFileSync } from 'node:fs'; +import { dirname, join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { redactCredentials, sanitiseConsoleError, + sanitiseDiagnostic } from '../src/evidence/diagnostic-sanitizer.js'; +import { emptyArtifactIdentities, readArtifact, readArtifactPayload, writeArtifact } from '../src/evidence/artifacts.js'; +import { criterionEvidence, evidenceIsRepairable } from '../src/evidence/check-evidence.js'; +import { renderRepairDiagnostic } from '../src/evidence/evidence-presentation.js'; +import type { ActionEvidence } from '../src/actions/action-contract.js'; + +interface RepairHistoryEntry { + round?: number; + beforeScore?: number; + beforeMax?: number; + afterScore?: number; + afterMax?: number; + result?: string; + remainingFailures?: string[]; +} + +interface ReportBugsArgs { + app: string; + out: string; + archive?: string; + history: RepairHistoryEntry[]; + checks: string[] | null; + controls: string[] | null; +} + +interface ParsedArgs { + app?: string; + out?: string; + archive?: string; + history?: unknown; + checks?: unknown; + controls?: unknown; +} + +interface Criterion { + id?: string; + stableKey?: string; + desc?: string; + points?: number; + evidence?: unknown; +} + +interface GradeFeature { + name?: string; + consoleErrors?: string[]; + criteria?: Criterion[]; +} + +interface GradePayload { + url?: string; + features?: GradeFeature[]; +} + +interface ContractResult { + id: string; + status: string; + detail?: string; +} + +interface ContractLintPayload { + url?: string; + results?: ContractResult[]; +} + +interface GradeBundlePayload { + backend?: string; + url?: string; + outcome?: { kind?: string; phase?: string; reason?: string }; +} + +interface RepairBug { + area: string; + actor: string | null; + expected: string; + observed: string; + url: string | null; + consoleErrors: string[]; + contract: boolean; + vague: boolean; +} + +function repairValue(value: unknown, fallback: string): string { + if (value === null || value === undefined) return fallback; + if (typeof value === 'object' && !Array.isArray(value) + && Object.keys(value).length === 1 && 'value' in value) value = value.value; + const text = typeof value === 'string' ? value : JSON.stringify(value); + return sanitiseDiagnostic(text, 500) || fallback; +} + +export function parseReportBugsArgs(argv: string[]): ReportBugsArgs { + const args: ParsedArgs = {}; + for (let i = 2; i < argv.length; i++) { + if (argv[i] === '--app') args.app = argv[++i]; + else if (argv[i] === '--out') args.out = argv[++i]; + else if (argv[i] === '--archive') args.archive = argv[++i]; + else if (argv[i] === '--history-json') { + const value = argv[++i]; + if (value === undefined) throw new Error('--history-json requires a value'); + args.history = JSON.parse(value); + } + else if (argv[i] === '--checks-json') { + const value = argv[++i]; + if (value === undefined) throw new Error('--checks-json requires a value'); + args.checks = JSON.parse(value); + } + else if (argv[i] === '--controls-json') { + const value = argv[++i]; + if (value === undefined) throw new Error('--controls-json requires a value'); + args.controls = JSON.parse(value); + } + else throw new Error(`Unknown argument: ${argv[i]}`); + } + if (!args.app) { + throw new Error('Usage: report-bugs --app [--out ]'); + } + args.out ??= join(args.app, 'BUG_REPORT.md'); + args.history ??= []; + if (!Array.isArray(args.history)) throw new Error('--history-json must contain an array'); + args.checks ??= null; + if (args.checks !== null && (!Array.isArray(args.checks) + || args.checks.some(check => typeof check !== 'string' || !check) + || new Set(args.checks).size !== args.checks.length)) { + throw new Error('--checks-json must contain distinct non-empty strings'); + } + args.controls ??= null; + if (args.controls !== null && (!Array.isArray(args.controls) + || args.controls.some(control => typeof control !== 'string' || !control) + || new Set(args.controls).size !== args.controls.length)) { + throw new Error('--controls-json must contain distinct non-empty strings'); + } + return { app: args.app, out: args.out, archive: args.archive, + history: args.history as RepairHistoryEntry[], checks: args.checks as string[] | null, + controls: args.controls as string[] | null }; +} + +const VAGUE = new Set([ + 'it did not behave as described', + 'the feature could not be reached at all', + 'the app did not respond in time', +]); +export function createBugReport(args: ReportBugsArgs): number { + const resultsDir = join(args.app, 'stack-bench'); + if (!existsSync(resultsDir)) throw new Error(`No grading results in ${resultsDir}`); + + let vagueBugs = 0; + const bugs: RepairBug[] = []; + const selectedChecks = args.checks === null ? null : new Set(args.checks); + const selectedControls = args.controls === null ? null : new Set(args.controls); + + for (const file of readdirSync(resultsDir).filter(name => /^grading-.*\.json$/.test(name))) { + const report = readArtifactPayload(join(resultsDir, file), { expectedKind: 'grade' }); + for (const feature of report.features ?? []) { + // Only typed application failures are sent to a fix round. Inconclusive or + // harness-failure evidence describes the benchmark, not the generated app. + // Zero-point criteria are test-development evidence and never control an + // ordinary repair loop, even when their behavioral observation failed. + for (const criterion of feature.criteria ?? []) { + if (selectedChecks && (!criterion.stableKey + || !selectedChecks.has(criterion.stableKey))) continue; + if (!(Number(criterion.points) > 0)) continue; + const evidence = criterionEvidence(criterion); + if (!evidenceIsRepairable(evidence)) continue; + const observed = renderRepairDiagnostic(evidence); + const actionEntry = evidence.actions.at(-1); + const actionEvidence = actionEntry?.evidence as ActionEvidence | undefined; + const safeDetails = evidence.sensitivity.length === 0 + && (actionEvidence?.sensitivity.length ?? 0) === 0; + const fallbackExpected = sanitiseDiagnostic( + criterion.desc ?? 'the requested behavior', 300); + const expected = safeDetails + ? repairValue(actionEvidence?.expected ?? evidence.expected, fallbackExpected) + : fallbackExpected; + const actual = safeDetails + ? repairValue(actionEvidence?.observation ?? evidence.observation, observed) + : observed; + const vague = VAGUE.has(actual); + if (vague) vagueBugs += 1; + bugs.push({ + area: sanitiseDiagnostic(feature.name, 120), + actor: sanitiseDiagnostic(actionEntry?.actor ?? evidence.actor, 120) || null, + expected, + observed: actual, + url: typeof report.url === 'string' ? redactCredentials(report.url).slice(0, 500) : null, + consoleErrors: (feature.consoleErrors ?? []).slice(0, 3) + .map(sanitiseConsoleError).filter(Boolean), + contract: false, vague, + }); + } + } + } + + // Contract failures are separate because the element id is itself the public + // requirement here. Behavioral failures above must never expose one. + const lintPath = join(resultsDir, 'contract-lint.json'); + if (existsSync(lintPath)) { + const lint = readArtifactPayload(lintPath, { expectedKind: 'contract_lint' }); + for (const result of (lint.results ?? []).filter(item => item.status === 'FAIL' + && (!selectedControls || selectedControls.has(item.id)))) { + bugs.push({ + area: 'Application controls', + actor: null, + expected: `A visible element for "${(result.detail ?? '').split('expected: ').pop()}" must use id="${result.id}"`, + observed: sanitiseDiagnostic(result.detail + ?? `no visible element with id="${result.id}" was found after a clean reset`, 500), + url: typeof lint.url === 'string' ? redactCredentials(lint.url).slice(0, 500) : null, + consoleErrors: [], contract: true, vague: false, + }); + } + } + + const bundlePath = join(resultsDir, 'bundle.json'); + if (existsSync(bundlePath)) { + const bundle = readArtifactPayload(bundlePath, { expectedKind: 'grade_bundle' }); + if (bundle.outcome?.kind === 'app_failure' && bundle.outcome.reason) { + const expectedByPhase: Record = { + 'database-provenance': `The app must use the ${bundle.backend} database and connection supplied for this run.`, + 'application-layout': 'The app must use a project layout that can be built, started, and reset repeatedly.', + 'application-restart': 'The app must provide /app/start.sh. From clean source, it must install dependencies, build, and start the complete application without changing source files.', + }; + const expected = expectedByPhase[bundle.outcome.phase ?? ''] + ?? 'The app must start successfully in the supplied environment.'; + bugs.unshift({ + area: 'Application setup', + actor: null, + expected, + observed: sanitiseDiagnostic(bundle.outcome.reason, 500), + url: typeof bundle.url === 'string' ? redactCredentials(bundle.url).slice(0, 500) : null, + consoleErrors: [], + contract: false, vague: false, + }); + } + } + + if (bugs.length === 0) { + console.log('No failures — no bug report written.'); + return 3; + } + + const repairBugs = bugs.filter(bug => !bug.vague); + const behavioral = repairBugs.filter(bug => !bug.contract); + const contractFailures = repairBugs.filter(bug => bug.contract); + const lines = [ + '# Bug Report', + '', + 'The application has these problems after a clean database reset and a fresh', + 'restart. Fix the behavior, then redeploy.', + 'Do not change behavior that is already correct. A result from existing local', + 'state does not replace the clean result below.', + '', + ]; + + if (args.history.length) { + lines.push('## Earlier work', ''); + lines.push('Earlier changes did not fix the current problems. Use the current source as', + 'the starting point. Do not repeat an earlier approach only because it appeared', + 'to work with existing local state.', ''); + } + + if (behavioral.length) { + lines.push('## Behavior', ''); + behavioral.forEach((bug, index) => { + lines.push(`### Bug ${index + 1}: ${bug.area}`, ''); + if (bug.actor) lines.push(`**Actor/session:** ${bug.actor}`, ''); + lines.push(`**Expected:** ${bug.expected}`, ''); + lines.push(`**Actual:** ${bug.observed}`, ''); + if (bug.url) lines.push(`**Application URL:** \`${bug.url.replaceAll('`', "'")}\``, ''); + if (bug.consoleErrors.length) { + lines.push('**Console or network errors:**', ''); + bug.consoleErrors.forEach(error => lines.push(`- \`${error}\``)); + lines.push(''); + } + }); + } + + if (contractFailures.length) { + lines.push('## Application controls', ''); + lines.push('These required elements were not available in the clean application state:', ''); + contractFailures.forEach(bug => { + lines.push(`- **Expected:** ${bug.expected}`); + lines.push(` **Actual:** ${bug.observed}`); + if (bug.url) lines.push(` **Application URL:** \`${bug.url.replaceAll('`', "'")}\``); + }); + lines.push(''); + } + + const vaguePct = Math.round((vagueBugs / bugs.length) * 100); + try { + const bundle = existsSync(bundlePath) ? readArtifact(bundlePath, { expectedKind: 'grade_bundle' }) : null; + const parentId = bundle?.attempt.id ?? null; + writeArtifact(join(resultsDir, 'bug-report-quality.json'), { + kind: 'bug_report_quality', id: `${parentId ?? 'bugs'}-bug-report-quality`, + attempt: { id: `${parentId ?? 'bugs'}-bug-report-quality`, parentId }, + identities: bundle?.identities ?? emptyArtifactIdentities(), + payload: { bugs: bugs.length, vague: vagueBugs, vaguePct }, + }); + } catch { /* Quality metadata must not block the repair decision. */ } + + console.log(` diagnostic quality: ${repairBugs.length}/${bugs.length} actionable, ${vagueBugs} vague (${vaguePct}%)`); + if (repairBugs.length === 0) { + rmSync(args.out, { force: true }); + console.log('No actionable failures. A paid repair was not started.'); + return 4; + } + + const reportText = lines.join('\n'); + writeFileSync(args.out, reportText); + if (args.archive) { + mkdirSync(dirname(args.archive), { recursive: true }); + writeFileSync(args.archive, reportText); + } + console.log(`Wrote ${repairBugs.length} bug(s) to ${args.out}`); + return 0; +} + +function main(): void { + try { + process.exitCode = createBugReport(parseReportBugsArgs(process.argv)); + } catch (error) { + console.error(error instanceof Error ? error.message : String(error)); + process.exitCode = 2; + } +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/reset-backend.ts b/tools/stack-bench/commands/reset-backend.ts new file mode 100644 index 00000000000..c2815d3b103 --- /dev/null +++ b/tools/stack-bench/commands/reset-backend.ts @@ -0,0 +1,24 @@ +#!/usr/bin/env node + +import { GENERATED_APP_LAYOUT_EXIT_CODE, resetBackend } from '../src/stacks/backend-reset.js'; +import { GeneratedAppLayoutError } from '../src/runtime/spacetime-layout.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; + +const [backend, app] = process.argv.slice(2); +if (!backend || !app) throw new Error('usage: node dist/commands/reset-backend.js '); + +Promise.resolve().then(() => resetBackend({ backend, app })).then(result => { + console.log(result); +}).catch(error => { + if (error instanceof GeneratedAppLayoutError || error?.code === 'generated_app_layout') { + console.error(`GENERATED_APP_LAYOUT: ${error.message}`); + process.exitCode = GENERATED_APP_LAYOUT_EXIT_CODE; + return; + } + const childOutput = [error?.stderr, error?.stdout] + .filter(value => value !== undefined && value !== null && String(value).trim()) + .map(value => String(value).trim()).join('\n'); + if (childOutput) console.error(redactCredentials(childOutput).slice(-2000)); + console.error(error.stack ?? error.message); + process.exitCode = 1; +}); diff --git a/tools/stack-bench/commands/run-suite.ts b/tools/stack-bench/commands/run-suite.ts new file mode 100644 index 00000000000..f2d41b05e9b --- /dev/null +++ b/tools/stack-bench/commands/run-suite.ts @@ -0,0 +1,1144 @@ +#!/usr/bin/env node +// Stack Bench: grade one generated app end to end. +// +// Every manual step in this sequence has produced a wrong result at least once +// (grading a dirty database silently lowers scores; grading the wrong backend +// entirely when two apps collide on a port), so the sequence is automated and +// each precondition is verified rather than assumed. +// +// stop hosted app -> reset database -> verify clean -> contract lint -> feature/invariant/delivery +// suites -> bundle +// +// Usage: +// node dist/commands/run-suite.js --app --url --backend spacetime|postgres|mongodb +// --label [--out ] [--media] [--level 1] [--no-reset] + +import { execFileSync, spawnSync } from 'node:child_process'; +import type { ExecFileSyncOptionsWithStringEncoding, SpawnSyncOptionsWithStringEncoding } from 'node:child_process'; +import { readFileSync, writeFileSync, mkdirSync, existsSync, readdirSync, rmSync } from 'node:fs'; +import { join, resolve } from 'node:path'; +import { pathToFileURL } from 'node:url'; +import { dbName, loadTrack, suitesFor, DEFAULT_TRACK } from '../src/composition/tracks.js'; +import { controlBackend } from '../src/runtime/backend-control.js'; +import { readArtifactPayload, recipeArtifactIdentities, writeArtifact } from '../src/evidence/artifacts.js'; +import { bundleRecipeRelease, resolveRecipeRelease } from '../src/composition/recipe-release.js'; +import { createBoundRecipeTaskRequest, resolveBoundRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import { contractControlIds } from '../src/composition/agent-visible-contract.js'; +import { resolveCalibrationForRelease } from '../src/composition/calibration-compiler.js'; +import { criterionEvidence, evidencePassed } from '../src/evidence/check-evidence.js'; +import { renderEvidenceConsoleLine } from '../src/evidence/evidence-presentation.js'; +import { executeStackCapability } from '../src/stacks/stack-adapter-contract.js'; +import { STACK_ADAPTER_REGISTRY } from '../src/stacks/stack-adapters.js'; +import { aggregatePackRuntime, exceededPackBudgets } from '../src/composition/pack-runtime.js'; +import { hashAppSource } from '../src/runtime/source-snapshot.js'; +import { GENERATED_APP_LAYOUT_EXIT_CODE } from '../src/stacks/backend-reset.js'; +import { readBackendLease } from '../src/runtime/backend-lease.js'; +import { databaseContainerName } from '../src/stacks/database-containers.js'; +import { redactCredentials } from '../src/evidence/diagnostic-sanitizer.js'; +import { canonicalDefinitionJson } from '../src/composition/definition-plan.js'; +import { sha256 } from '../src/evidence/provenance.js'; +import { GRADER_SOURCE_TIMEOUT_MS } from '../src/runtime/grading-timeout.js'; +import type { BackendLease, BackendLeaseExpectation } from '../src/runtime/backend-lease.js'; +import type { CheckEvidence } from '../src/evidence/check-evidence.js'; +import type { AggregatedPackRuntimeEvidence, PackRuntimeEvidence } from '../src/composition/pack-runtime.js'; +import { isModularRecipeTaskRequest } from '../src/composition/recipe-selection.js'; +import type { BoundRecipeTaskRequestResult, RecipeSelection } from '../src/composition/recipe-selection.js'; +import type { RecipeBinding, RecipeCheck } from '../src/composition/recipe-release.js'; +import type { Track, TrackSuite } from '../src/composition/tracks.js'; + +import { STACK_BENCH_ROOT as ROOT, compiledEntrypoint } from '../src/package-root.js'; +const RESET = compiledEntrypoint('commands', 'reset-backend.js'); + +type Observation = 'scored' | 'observed'; +type Selection = { + schemaVersion: number; + recipe: { id: string; version: string; contentSha256: string }; + requested: RecipeSelection['requested']; + sha256: string; + checks: RecipeCheck[]; + scoredPoints: number; + observedChecks?: Array; + observedPoints?: number; + evaluationSha256?: string; + regressionChecks?: Array; + regressionPoints?: number; + observation?: Observation; +}; +type DeclaredSuite = TrackSuite; +type Failure = Error & { stdout?: string; stderr?: string; status?: number | null; signal?: string | null; + code?: string }; +type FailureDetail = { message?: unknown; stderr?: unknown } | null; +type RestartSpec = Record; +type RecipeTaskArgument = { recipe: { id: string; version: string; contentSha256?: string } } & Record; +type RunArguments = { + app: string; + url: string; + backend: string; + label: string; + out: string; + level: string; + reset: boolean; + media: boolean; + runIndex: number; + track: string; + packIds: string[]; + checkKeys: string[]; + observation: Observation; + recipe?: string; + recipeTask?: RecipeTaskArgument; + credentialAliases?: unknown; + regressionChecks: string[]; + sourceSha256?: string; + restartSpec?: RestartSpec; + applicationFailure?: ApplicationFailure; + parentAttemptId?: string; + databaseLease?: BackendLease | null; + databaseContainer?: string | null; + selection?: Selection | null; + bundleArtifactId: string; +}; +type GradeCriterion = { id: string; stableKey?: string; serverCheck?: string; evidence?: CheckEvidence }; +type GradeFeature = { name: string; criteria: GradeCriterion[] }; +type GradePayload = { total: number; max: number; features: GradeFeature[]; + selection?: { checks?: RecipeCheck[] }; packRuntime?: PackRuntimeEvidence }; +type LintPayload = { pass: boolean; counts: { pass: number; fail: number; blocked: number } }; +type ActionsPayload = { missing: string[]; results: unknown[] }; +type RuntimeProvenance = { ok: boolean | null; verified: boolean; reason: string }; +type ApplicationProbeResult = { ok: boolean; detail: string | null }; +type ResetOutcome = { kind: string; phase: string; appFailures?: string[] }; +type ApplicationFailure = ResetOutcome & { kind: 'app_failure'; reason: string }; +type DatabaseProvenance = { ok: boolean; reason: string; url?: string }; +type GradeDatabaseLease = { resources: { container?: { name?: string | null } | null; + database?: string | null } }; +type GradeLeaseReader = (path: string, expected: BackendLeaseExpectation) => GradeDatabaseLease; +type MutationDirectoryEntry = { name: string; isDirectory(): boolean; isFile(): boolean }; +type MutationDirectoryReader = (path: string, options: { withFileTypes: true }) => readonly MutationDirectoryEntry[]; +type GraderChildResult = { status: number | null; signal: string | null; stdout?: unknown; stderr?: unknown; + error?: Error }; +type GraderChildExecutor = (command: string, argv: readonly string[], options: SpawnSyncOptionsWithStringEncoding) => GraderChildResult; +type ProbeResponse = { ok: boolean; status: number }; +type ApplicationFetch = (url: string, init: { signal: AbortSignal }) => Promise; +type DatabaseProvenanceDefinition = Record | null | undefined; +type DatabaseNameLease = { resources: { database?: string | null } }; +type MarkerCriterion = { stableKey?: string; evidence?: CheckEvidence }; +type MarkerReport = { features: Array<{ criteria: MarkerCriterion[] }> }; +type ApplicationFailureSelection = { checks: Array<{ executionId: string; points?: number }> }; +type ContractLintArguments = Pick; +type BundleSelection = Selection & { attemptedChecks: string[]; reportedChecks: string[]; + notRun: Array<{ stableKey: string; reason: string }> }; +type Bundle = { + definitionSchemaVersion: number; + recipeRelease: ReturnType; + calibration: { id: string; version: string; state: string; contentSha256: string } | null; + label: string; track: string; backend: string; url: string; app: string; level: number; + observation: Observation; source?: { sha256: string }; + suites: Record; + totals: Record; + selection: BundleSelection | null; + code?: ReturnType; + error?: string; + outcome?: { kind: string; phase: string; reason?: string; appFailures?: string[] }; + provenance?: DatabaseProvenance & { runtime?: RuntimeProvenance }; + actions?: ActionsPayload | null; + packRuntime?: AggregatedPackRuntimeEvidence; +}; + +const isRecord = (value: unknown): value is Record => + value !== null && typeof value === 'object' && !Array.isArray(value); + +const parseObservation = (value: string): Observation => { + if (value === 'scored' || value === 'observed') return value; + throw new Error('--observation must be scored or observed'); +}; + +export function suitesForRecipe(track: Track, binding: RecipeBinding): DeclaredSuite[] { + if (!binding?.execution?.length) throw new Error('recipe has no typed execution plan'); + return binding.execution.map(entry => ({ + id: entry.id, + spec: resolve(track.dir, entry.source ?? ''), + ...(entry.ownership.kind === 'inherited' + ? { inherited: true, fromLevel: entry.ownership.fromLevel } + : {}), + })); +} + +export function childFailureDetail(failure: FailureDetail = null, stdout = '', limit = 600): string { + const processOutput = [failure?.stderr, stdout] + .filter(value => value !== undefined && value !== null && String(value).trim()) + .join('\n').trim(); + const diagnostic = processOutput || String(failure?.message ?? '').trim(); + const lines = diagnostic.split(/\r?\n/).map(line => line.trim()).filter(Boolean); + if (!lines.length) return ''; + const punctuationOnly = (line: string) => + [...line].every(character => '[]{},'.includes(character)); + const noise = (line: string) => line.startsWith('at ') || /^Node\.js v/.test(line) + || /^node:internal\//.test(line) || /^\^+$/.test(line) || punctuationOnly(line); + const cause = lines.find(line => !noise(line) && /(?:error|failed|timeout|closed|econn|killed)/i.test(line)) + ?? lines.find(line => !noise(line)) ?? lines[0]; + const selected = [cause, ...lines.slice(-4)].filter((line, index, all) => all.indexOf(line) === index); + return selected.join(' | ').slice(0, limit); +} + +export function resetFailureOutcome(error: unknown): ResetOutcome { + const failure = isRecord(error) ? error : {}; + return failure.status === GENERATED_APP_LAYOUT_EXIT_CODE + ? { kind: 'app_failure', phase: 'application-layout', + appFailures: ['application-layout'] } + : failure.code === 'generated_app_not_restartable' + ? { kind: 'app_failure', phase: 'application-restart', + appFailures: ['application-restart'] } + : { kind: 'harness_failure', phase: 'database-reset' }; +} + +export function applicationFailureTotals(selection: ApplicationFailureSelection | null | undefined, + declaredSuites: Array>): Record { + if (!selection?.checks?.length) return {}; + const inherited = new Set(declaredSuites.filter(suite => suite.inherited).map(suite => suite.id)); + const currentMax = selection.checks.filter(check => !inherited.has(check.executionId)) + .reduce((total, check) => total + Number(check.points ?? 0), 0); + const regressionMax = selection.checks.filter(check => inherited.has(check.executionId)) + .reduce((total, check) => total + Number(check.points ?? 0), 0); + return { score: 0, max: currentMax, dirty: false, contractPass: null, + regression: regressionMax ? { score: 0, max: regressionMax } : null }; +} + +export function clearPreviousGradeOutputs(output: string): void { + const generated = existsSync(output) ? readdirSync(output).filter(name => + /^grading-.+\.json$/.test(name) || /^grader-.+\.(?:stdout|stderr)\.log$/.test(name)) : []; + for (const name of ['bundle.json', 'contract-lint.json', 'actions.json', 'media', 'failure-media', + 'database-provenance', ...generated]) { + rmSync(join(output, name), { recursive: true, force: true }); + } +} + +export function runGraderChild(argv: string[], output: string, suiteId: string, { + execute = spawnSync, +}: { execute?: GraderChildExecutor } = {}) { + const options: SpawnSyncOptionsWithStringEncoding = { encoding: 'utf8', cwd: ROOT, timeout: COMMAND_TIMEOUT_MS, + maxBuffer: 64 * 1024 * 1024 }; + const result = execute(process.execPath, argv, options); + const stdout = redactCredentials(String(result.stdout ?? '')); + const stderr = redactCredentials(String(result.stderr ?? '')); + const safeId = String(suiteId).replace(/[^A-Za-z0-9._-]/g, '_'); + const stdoutName = `grader-${safeId}.stdout.log`; + const stderrName = `grader-${safeId}.stderr.log`; + writeFileSync(join(output, stdoutName), stdout); + writeFileSync(join(output, stderrName), stderr); + let failure = result.error ?? null; + if (!failure && result.status !== 0) { + failure = new Error(`grader exited ${result.status ?? result.signal ?? 'without status'}`); + } + if (failure) Object.assign(failure, { stdout, stderr, status: result.status, signal: result.signal }); + return { stdout, stderr, failure, stdoutName, stderrName }; +} + +function gradeLeaseInput(backend: string, env: NodeJS.ProcessEnv): { path: string; + expected: BackendLeaseExpectation } | null { + if (!['mongodb', 'postgres'].includes(backend)) return null; + const path = String(env.STACK_BENCH_LEASE ?? '').trim(); + const token = String(env.STACK_BENCH_LEASE_TOKEN ?? '').trim(); + if (!path && !token) return null; + if (!path || !token) throw new Error('database grading requires both lease path and lease token'); + return { path, expected: { token, backend, active: true } }; +} + +export function databaseContainerForGrading(backend: string, env = process.env, { + readLease = readBackendLease, +}: { readLease?: GradeLeaseReader } = {}) { + const input = gradeLeaseInput(backend, env); + if (!input) return null; + const lease = readLease(input.path, input.expected); + if (lease?.resources.container) return String(lease.resources.container.name); + return databaseContainerName(backend, env); +} + +export function databaseLeaseForGrading(backend: string, env = process.env, { + readLease = readBackendLease, +} = {}) { + const input = gradeLeaseInput(backend, env); + if (!input) return null; + const lease = readLease(input.path, input.expected); + const container = String(lease.resources?.container?.name ?? '').trim(); + if (!container) throw new Error(`active ${backend} lease has no database container`); + return lease; +} + +export function databaseNameForGrading(track: Pick, runIndex: number, + lease: DatabaseNameLease | null = null): string { + if (!lease) return dbName(track, runIndex); + const database = String(lease.resources?.database ?? '').trim(); + if (!database) throw new Error('active database lease has no database name'); + return database; +} + +function parseArgs(argv: string[]): RunArguments { + const a: RunArguments = { app: '', url: '', backend: '', label: '', out: '', level: '1', reset: true, + media: true, runIndex: 0, track: DEFAULT_TRACK, packIds: [], checkKeys: [], observation: 'scored', + regressionChecks: [], bundleArtifactId: '' }; + for (let i = 2; i < argv.length; i++) { + switch (argv[i]) { + case '--app': a.app = argv[++i] ?? ''; break; + case '--url': a.url = argv[++i] ?? ''; break; + case '--backend': a.backend = argv[++i] ?? ''; break; + case '--label': a.label = argv[++i] ?? ''; break; + case '--out': a.out = argv[++i] ?? ''; break; + case '--level': a.level = argv[++i] ?? ''; break; + case '--recipe': a.recipe = argv[++i] ?? ''; break; + case '--recipe-task-json': a.recipeTask = JSON.parse(argv[++i] ?? ''); break; + case '--credential-aliases-json': a.credentialAliases = JSON.parse(argv[++i] ?? ''); break; + case '--regression-checks-json': a.regressionChecks = JSON.parse(argv[++i] ?? ''); break; + case '--observation': a.observation = parseObservation(argv[++i] ?? ''); break; + case '--source-sha256': a.sourceSha256 = argv[++i] ?? ''; break; + case '--no-media': a.media = false; break; + case '--track': a.track = argv[++i] ?? ''; break; + case '--pack': a.packIds.push(...(argv[++i] ?? '').split(',').filter(Boolean)); break; + case '--check': a.checkKeys.push(...(argv[++i] ?? '').split(',').filter(Boolean)); break; + case '--restart-spec': a.restartSpec = JSON.parse(argv[++i] ?? ''); break; + case '--application-failure-json': a.applicationFailure = JSON.parse(argv[++i] ?? ''); break; + case '--run-index': a.runIndex = parseInt(argv[++i] ?? '', 10); break; + case '--no-reset': a.reset = false; break; + case '--parent-attempt-id': a.parentAttemptId = argv[++i]; break; + default: console.error(`Unknown argument: ${argv[i]}`); process.exit(2); + } + } + if (!a.app || !a.url || !a.backend || !a.label) { + console.error('Usage: node dist/commands/run-suite.js --app --url --backend --label [--out ] [--media] [--no-reset]'); + process.exit(2); + } + if (!['scored', 'observed'].includes(a.observation)) { + throw new Error('--observation must be scored or observed'); + } + if (a.observation === 'observed' && !/^[a-f0-9]{64}$/.test(a.sourceSha256 ?? '')) { + throw new Error('observed specifications require --source-sha256'); + } + if (a.sourceSha256 !== undefined && !/^[a-f0-9]{64}$/.test(a.sourceSha256)) { + throw new Error('--source-sha256 must be a SHA-256 digest'); + } + if (a.applicationFailure && (a.applicationFailure.kind !== 'app_failure' + || typeof a.applicationFailure.phase !== 'string' || !a.applicationFailure.phase + || typeof a.applicationFailure.reason !== 'string' || !a.applicationFailure.reason)) { + throw new Error('--application-failure-json must describe an application failure'); + } + a.out ||= join(a.app, 'stack-bench'); + if (!Array.isArray(a.regressionChecks) + || a.regressionChecks.some(key => typeof key !== 'string' || !key)) { + throw new Error('--regression-checks-json must contain stable check keys'); + } + return a; +} + +export function selectObservationScope(selectedTask: BoundRecipeTaskRequestResult | null, + observation: Observation = 'scored'): Selection | null { + if (observation === 'scored') return selectedTask?.selection ?? null; + if (observation !== 'observed') throw new Error(`unknown observation scope ${observation}`); + if (!selectedTask || !isModularRecipeTaskRequest(selectedTask)) { + throw new Error('observed specifications require a modular schema-3 task request'); + } + const selection = selectedTask.selection; + if (!selection.observedChecks.length) throw new Error('observed specification scope is empty'); + return { + ...selection, + observation: 'observed', + checks: selection.observedChecks, + scoredPoints: 0, + observedPoints: selection.observedChecks.reduce((total, check) => total + check.points, 0), + }; +} + +export function attachRegressionScope(selection: Selection | null, recipeBinding: RecipeBinding | null, + declaredSuites: DeclaredSuite[], stableKeys: string[] = []): Selection | null { + if (!stableKeys.length) return selection; + if (!selection || !recipeBinding?.release?.checkCatalog) { + throw new Error('regression checks require a recipe-bound scored selection'); + } + const uniqueKeys = [...new Set(stableKeys)]; + if (uniqueKeys.length !== stableKeys.length) throw new Error('regression checks contain duplicates'); + const currentKeys = new Set(selection.checks.map(check => check.stableKey)); + const catalog = new Map(recipeBinding.release.checkCatalog + .map(check => [check.stableKey, check])); + const inheritedSuites = new Set(declaredSuites.filter(suite => suite.inherited) + .map(suite => suite.id)); + const regressionChecks = uniqueKeys.map(key => { + if (currentKeys.has(key)) throw new Error(`regression check ${key} is already in the current score`); + const check = catalog.get(key); + if (!check) throw new Error(`regression check ${key} is absent from the cumulative recipe`); + if (!inheritedSuites.has(check.executionId)) { + throw new Error(`regression check ${key} does not belong to an inherited execution`); + } + return { ...check, treatment: check.treatment ?? 'regression' }; + }); + const evaluationDocument = { schemaVersion: 1, selectionSha256: selection.sha256, + regressionChecks: uniqueKeys.slice().sort() }; + return { + ...selection, + checks: [...selection.checks, ...regressionChecks], + regressionChecks: regressionChecks.map(check => ({ ...check, treatment: check.treatment ?? 'regression' })), + regressionPoints: regressionChecks.reduce((total, check) => total + check.points, 0), + evaluationSha256: sha256(Buffer.from(canonicalDefinitionJson(evaluationDocument))), + }; +} + +const sleep = (ms: number): Promise => new Promise(resolveSleep => setTimeout(resolveSleep, ms)); +const COMMAND_TIMEOUT_MS = GRADER_SOURCE_TIMEOUT_MS; +const run = (cmd: string, args: readonly string[], opts: Omit = {}): string => + execFileSync(cmd, args, { + encoding: 'utf8', stdio: 'pipe', cwd: ROOT, timeout: COMMAND_TIMEOUT_MS, ...opts, + }); + +export async function verifyApplicationProbe(url: string, { + fetchImpl = fetch, timeoutMs = 5000, +}: { fetchImpl?: ApplicationFetch; timeoutMs?: number } = {}): Promise { + let response; + try { + response = await fetchImpl(url, { signal: AbortSignal.timeout(timeoutMs) }); + } catch (error) { + return { ok: false, + detail: `application did not respond: ${error instanceof Error ? error.message : String(error)}` }; + } + if (!response.ok) { + return { ok: false, detail: `application returned HTTP ${response.status}` }; + } + return { ok: true, detail: null }; +} + +export async function waitForApplicationProbe(url: string, { + attempts = 9, intervalMs = 250, probeTimeoutMs = 1000, + probe = verifyApplicationProbe, sleepImpl = sleep, +}: { attempts?: number; intervalMs?: number; probeTimeoutMs?: number; + probe?: typeof verifyApplicationProbe; + sleepImpl?: (ms: number) => Promise } = {}): Promise { + if (!Number.isInteger(attempts) || attempts < 1) { + throw new Error('application probe attempts must be a positive integer'); + } + let result = null; + for (let attempt = 1; attempt <= attempts; attempt += 1) { + result = await probe(url, { timeoutMs: probeTimeoutMs }); + if (result.ok || attempt === attempts) return result; + await sleepImpl(intervalMs); + } + return result ?? { ok: false, detail: 'application readiness probe did not run' }; +} + +// The benchmark's own database containers. A generated app that connects +// somewhere else is not measuring what we think it is: one Postgres app pointed +// at an unrelated project's container on 5433 and graded "fine" while writing to +// a database the harness could not reset. +export function checkDatabaseProvenance(args: Pick): DatabaseProvenance { + const adapter = STACK_ADAPTER_REGISTRY.get(args.backend); + const expected = executeStackCapability(adapter, 'ports', 'allocations').db; + if (!expected) return { ok: true, reason: 'no external database for this backend' }; + // Neutral guidance does not prescribe project layout. Search the app for the + // connection string instead of assuming it is in server/.env. + const urls: string[] = []; + let usesLeasedEnvironment = false; + const walk = (dir: string): void => { + if (!existsSync(dir)) return; + for (const e of readdirSync(dir, { withFileTypes: true })) { + if (/^(node_modules|dist|\.vite|\.git|module_bindings)$/.test(e.name)) continue; + const p = join(dir, e.name); + if (e.isDirectory()) { walk(p); continue; } + if (!/\.(env|ts|tsx|js|mjs|json|yaml|yml)$|^\.env/.test(e.name)) continue; + try { + const text = readFileSync(p, 'utf8'); + if (/process\.env(?:\.DATABASE_URL|\[['"]DATABASE_URL['"]\])/.test(text)) { + usesLeasedEnvironment = true; + } + const m = executeStackCapability(adapter, 'agent', 'find-database-urls', { text }); + if (Array.isArray(m) && m.every(value => typeof value === 'string')) urls.push(...m); + } catch { /* unreadable file proves nothing */ } + } + }; + walk(args.app); + if (usesLeasedEnvironment) { + return { ok: true, url: 'process.env.DATABASE_URL', + reason: 'app reads the database URL supplied by its authenticated backend lease' }; + } + if (!urls.length) return { ok: false, + reason: 'app neither reads process.env.DATABASE_URL nor contains a database connection string' }; + const matchesExpectedPort = (value: string): boolean => { + try { return Number(new URL(value).port) === Number(expected); } + catch { return false; } + }; + const ok = urls.some(matchesExpectedPort); + return { ok, url: urls[0], + reason: ok ? 'ok' : `app targets ${urls[0]} but the benchmark database is on port ${expected}` }; +} + +export function applicationDatabaseMarker(report: MarkerReport | null, + definition: DatabaseProvenanceDefinition): string | null { + if (!report || !definition) return null; + const check = definition.check; + const actionId = definition.action; + const observationField = definition.observationField; + if (typeof check !== 'string' || typeof actionId !== 'string') return null; + const markers: string[] = []; + for (const feature of report.features ?? []) { + for (const criterion of feature.criteria ?? []) { + if (criterion.stableKey !== check) continue; + const action = criterionEvidence(criterion).actions.find(entry => isRecord(entry.evidence) + && isRecord(entry.evidence.action) && entry.evidence.action.id === actionId + && entry.evidence.status === 'passed'); + const evidence = action?.evidence; + const marker = isRecord(evidence) && isRecord(evidence.observation) + && typeof observationField === 'string' + ? evidence.observation[observationField] : undefined; + if (typeof marker === 'string' && marker) markers.push(marker); + } + } + return new Set(markers).size === 1 ? markers[0] ?? null : null; +} + +export function databaseProvenanceFailure(error: unknown): { kind: string; phase: string; reason: string } { + return { kind: 'harness_failure', phase: 'database-provenance', + reason: `runtime database provenance failed: ${error instanceof Error ? error.message : String(error)}` }; +} + +export function checkRuntimeDatabaseProvenance(args: Pick, + marker: string | null = null): RuntimeProvenance { + if (!['mongodb', 'postgres'].includes(args.backend)) { + return { ok: null, verified: false, + reason: 'exact runtime database marker proof is not implemented for this stack' }; + } + if (!args.databaseLease) { + return { ok: null, verified: false, + reason: 'standalone grading has no authenticated database lease' }; + } + if (typeof marker !== 'string' || !marker) { + return { ok: null, verified: false, + reason: 'the application action did not produce a database marker' }; + } + const result = executeStackCapability(STACK_ADAPTER_REGISTRY.get(args.backend), + 'database', 'prove-use', { lease: args.databaseLease, marker }); + if (!isRuntimeProvenance(result)) throw new Error('database provenance proof returned an invalid result'); + return result; +} + +function isRuntimeProvenance(value: unknown): value is RuntimeProvenance { + return isRecord(value) && (value.ok === true || value.ok === false || value.ok === null) + && typeof value.verified === 'boolean' && typeof value.reason === 'string'; +} + +function isGradePayload(value: GradePayload | LintPayload | null | undefined): value is GradePayload { + return value !== null && value !== undefined && 'total' in value && 'max' in value; +} + +// Report the application size and direct runtime dependency count. +export function codeMetrics(args: Pick): { serverLoc: number; serverFiles: number; + totalLoc: number; totalFiles: number; runtimeDeps: number } { + // Also a prescribed-stack assumption: under minimal guidance an app may put + // its server anywhere, and a missing `server/` reported 0 LOC in 0 files + // rather than admitting it had not found the code. Fall back to everything + // outside the client when the conventional directory is absent. + const adapter = STACK_ADAPTER_REGISTRY.get(args.backend); + const serverDirectory = executeStackCapability(adapter, 'agent', 'server-directory'); + const conventional = typeof serverDirectory === 'string' ? serverDirectory : '.'; + const SERVER_DIR = existsSync(join(args.app, conventional)) ? conventional : '.'; + const walk = (dir: string, out: string[] = []): string[] => { + if (!existsSync(dir)) return out; + for (const e of readdirSync(dir, { withFileTypes: true })) { + if (/^(node_modules|dist|\.vite|module_bindings|drizzle)$/.test(e.name)) continue; + const p = join(dir, e.name); + if (e.isDirectory()) walk(p, out); + // JavaScript counts too: a stack-free app is under no obligation to use + // TypeScript, and one that wrote 17 .js and 10 .jsx files was reported as + // "0 server LOC in 0 files" — a lie about the measurement, not a fact + // about the app. + else if (/\.(ts|tsx|js|jsx|mjs|cjs)$/.test(e.name)) out.push(p); + } + return out; + }; + const count = (files: string[]): number => files.reduce((n, f) => n + readFileSync(f, 'utf8').split('\n').length, 0); + // With no conventional server directory, "server" is everything that is not + // the client — otherwise the fallback counts the client twice and serverLoc + // equals totalLoc, which reads as a much larger backend than was written. + const allFiles = walk(args.app); + const serverFiles = SERVER_DIR === '.' + ? allFiles.filter(f => !/[\\/]client[\\/]/.test(f)) + : walk(join(args.app, SERVER_DIR)); + + let deps = 0; + const packageFiles = new Set([ + resolve(args.app, 'package.json'), + resolve(args.app, SERVER_DIR, 'package.json'), + resolve(args.app, 'client/package.json'), + ]); + for (const p of packageFiles) { + if (!existsSync(p)) continue; + try { deps += Object.keys(JSON.parse(readFileSync(p, 'utf8')).dependencies ?? {}).length; } catch { /* ignore */ } + } + + return { + serverLoc: count(serverFiles), serverFiles: serverFiles.length, + totalLoc: count(allFiles), totalFiles: allFiles.length, + runtimeDeps: deps, + }; +} + +export function findMutationBackups(app: string, { readDir = readdirSync }: + { readDir?: MutationDirectoryReader } = {}): string[] { + const backups: string[] = []; + const walk = (dir: string): void => { + let entries; + try { + entries = readDir(dir, { withFileTypes: true }); + } catch (error) { + // Vite atomically replaces transient dependency directories while the + // app runs. They are not source and may vanish between parent and child + // reads; a missing directory cannot contain a mutation backup. + if (isRecord(error) && error.code === 'ENOENT') return; + throw error; + } + for (const entry of entries) { + if (/^(node_modules|dist|\.vite|\.git|module_bindings)$/.test(entry.name)) continue; + const path = join(dir, entry.name); + if (entry.isDirectory()) walk(path); + else if (entry.isFile() && entry.name.endsWith('.mutation-backup')) backups.push(path); + } + }; + walk(app); + return backups; +} + +function resetDatabase(args: RunArguments): { ok: boolean; detail: string | null; + outcome: { kind: string; phase: string; appFailures?: string[] } | null } { + process.stdout.write(' reset database ... '); + try { + run(process.execPath, [RESET, args.backend, args.app]); + console.log('ok'); + } catch (err) { + console.log('FAILED'); + const failure: Failure = err instanceof Error ? err : new Error(String(err)); + const detail = childFailureDetail(failure, failure.stdout); + console.log(` ${detail}`); + return { ok: false, detail, outcome: resetFailureOutcome(failure) }; + } + return { ok: true, detail: null, outcome: null }; +} + +export function contractLintArgv(args: ContractLintArguments, + selectedTask: BoundRecipeTaskRequestResult | null = null): string[] { + const controls = selectedTask ? contractControlIds(selectedTask.task.contractText) : []; + const out = join(args.out, 'contract-lint.json'); + return [compiledEntrypoint('linter', 'lint.js'), '--url', args.url, '--level', args.level, + '--track', args.track, '--label', args.label, '--out', out, + '--parent-attempt-id', args.bundleArtifactId, + ...(args.credentialAliases + ? ['--credential-aliases-json', JSON.stringify(args.credentialAliases)] : []), + ...controls.flatMap(id => ['--hook', id])]; +} + +function lint(args: RunArguments, selectedTask: BoundRecipeTaskRequestResult | null = null): LintPayload | null { + process.stdout.write(' contract lint ... '); + const out = join(args.out, 'contract-lint.json'); + rmSync(out, { force: true }); + let failure: unknown = null; + try { + run('node', contractLintArgv(args, selectedTask)); + } catch (error) { failure = error; /* hook failures still write a report */ } + if (!existsSync(out)) { + const output = failure && typeof failure === 'object' && 'stdout' in failure + ? String(failure.stdout ?? '') : undefined; + const detail = failure instanceof Error + ? childFailureDetail(failure, output) : null; + throw new Error(`contract lint produced no report${detail ? `: ${detail}` : ''}`); + } + const r = readArtifactPayload(out, { expectedKind: 'contract_lint' }); + console.log(r.pass + ? `PASS (${r.counts.pass} hooks)` + : `FAIL (${r.counts.fail} failed, ${r.counts.blocked} blocked)`); + return r; +} + +// Named write actions let concurrency checks issue authenticated operations +// without prescribing one transport. Missing actions are reported explicitly. +function checkActions(args: RunArguments): ActionsPayload | null { + process.stdout.write(` ${'actions'.padEnd(10)} ... `); + const out = join(args.out, 'actions.json'); + rmSync(out, { force: true }); + try { + run('node', [compiledEntrypoint('commands', 'check-actions.js'), '--backend', args.backend, + '--url', args.url, '--app', args.app ?? '.', '--track', args.track, '--out', out, '--quiet', + '--parent-attempt-id', args.bundleArtifactId]); + } catch { /* non-zero exit means something is missing; the report still lands */ } + if (!existsSync(out)) { console.log('NO REPORT'); return null; } + const r = readArtifactPayload(out, { expectedKind: 'action_check' }); + if (!r.missing.length) { console.log(`all ${r.results.length} present`); return r; } + console.log(`${r.missing.length} MISSING — ${r.missing.join(', ')}`); + return r; +} + +function gradeSuite(args: RunArguments, suite: DeclaredSuite, track: Track, + recipeBinding: RecipeBinding | null, bundleArtifactId: string, selectedChecks: RecipeCheck[] = [], + { recordSelection = true, captureMedia = true, outputDirectory = args.out }: { + recordSelection?: boolean; captureMedia?: boolean; outputDirectory?: string; + } = {}): GradePayload { + process.stdout.write(` ${suite.id.padEnd(10)} ... `); + mkdirSync(outputDirectory, { recursive: true }); + const out = join(outputDirectory, `grading-${suite.id}.json`); + rmSync(out, { force: true }); + const argv = [compiledEntrypoint('grader', 'grade.js'), '--url', args.url, '--level', args.level, + '--label', `${args.label}-${suite.id}`, '--out', out]; + if (suite.spec) argv.push('--spec', suite.spec); + argv.push('--backend', args.backend, '--track', args.track); + if (recipeBinding) argv.push('--expected-recipe-sha256', recipeBinding.release.contentSha256); + const requestedRecipe = args.recipe ?? (args.recipeTask + ? `${args.recipeTask.recipe.id}@${args.recipeTask.recipe.version}` : null); + if (requestedRecipe) argv.push('--recipe', requestedRecipe); + for (const check of selectedChecks) argv.push('--selected-check', check.stableKey); + if (args.credentialAliases) { + argv.push('--credential-aliases-json', JSON.stringify(args.credentialAliases)); + } + if (recordSelection && args.selection?.sha256) { + argv.push('--selection-sha256', args.selection.evaluationSha256 ?? args.selection.sha256); + } + argv.push('--parent-attempt-id', bundleArtifactId); + // The out-of-band write goes straight to this run's database, with no + // app code in the loop; only the harness knows which one that is. + argv.push('--db-name', databaseNameForGrading(track, args.runIndex ?? 0, args.databaseLease)); + if (args.databaseContainer) argv.push('--database-container', args.databaseContainer); + if (args.restartSpec) argv.push('--restart-spec', JSON.stringify(args.restartSpec)); + // The systems criteria run scripts the app itself ships (back-office writes), + // so the grader has to know where the app lives. + if (args.app) argv.push('--app', args.app); + if (captureMedia && args.media) argv.push('--media', join(outputDirectory, 'media'), '--trace'); + else if (captureMedia) argv.push('--failure-media', join(outputDirectory, 'failure-media')); + const child = runGraderChild(argv, outputDirectory, suite.id); + const { stdout, failure } = child; + if (!existsSync(out)) { + console.log('NO REPORT'); + const detail = childFailureDetail(failure, stdout); + throw new Error(`grader produced no report for ${suite.id}${detail ? `: ${detail}` : ''}; ` + + `full diagnostics: ${child.stdoutName}, ${child.stderrName}`); + } + const r = readArtifactPayload(out, { expectedKind: 'grade' }); + if (selectedChecks.length) { + const expected = selectedChecks.map(check => check.stableKey).sort(); + const reported = (r.selection?.checks ?? []).map(check => check.stableKey).sort(); + if (JSON.stringify(reported) !== JSON.stringify(expected)) { + throw new Error(`grader report scope differs from requested suite scope for ${suite.id}`); + } + } + console.log(`${r.total}/${r.max}`); + for (const f of r.features) { + for (const c of f.criteria.filter(c => !evidencePassed(criterionEvidence(c)))) { + console.log(` ${renderEvidenceConsoleLine(criterionEvidence(c), `${f.name} / ${c.id}`, { + includeSummary: false, + })}`); + } + } + // A criterion that PASSED on interface behaviour alone, because its + // server-side check could not be run against this backend, is a weaker result + // than one where the server refused a real request. Saying so on every run is + // the difference between a disclosed limitation and a flattering score. + const uiOnly = r.features.flatMap(f => + f.criteria.filter(c => evidencePassed(criterionEvidence(c)) && c.serverCheck === 'unverified') + .map(c => `${f.name}/${c.id}`)); + if (uiOnly.length) { + console.log(` note: ${uiOnly.length} criterion/criteria passed on interface behaviour only`); + for (const u of uiOnly) console.log(` ${u} — server-side check not runnable on this backend`); + } + return r; +} + +function databaseProvenanceChecks(definition: Track['databaseProvenance'], + recipeBinding: RecipeBinding | null): RecipeCheck[] { + if (!definition || !recipeBinding) return []; + const check = recipeBinding.release.checkCatalog.find(candidate => + candidate.stableKey === definition.check); + if (!check) { + throw new Error(`database provenance check is absent from the bound recipe: ${definition.check}`); + } + return [check]; +} + +async function main() { + const startedAt = new Date().toISOString(); + const args = parseArgs(process.argv); + args.databaseLease = databaseLeaseForGrading(args.backend); + args.databaseContainer = args.databaseLease?.resources.container?.name + ?? (['mongodb', 'postgres'].includes(args.backend) ? databaseContainerName(args.backend) : null); + const track = loadTrack(args.track); + const recipeBinding = resolveRecipeRelease(track, Number(args.level), args.recipeTask?.recipe ?? args.recipe); + if (!recipeBinding && (args.packIds.length || args.checkKeys.length)) { + throw new Error('--pack and --check require a recipe-bound level'); + } + const selectedTask = recipeBinding + ? (args.recipeTask + ? resolveBoundRecipeTaskRequest(recipeBinding, args.recipeTask) + : createBoundRecipeTaskRequest(recipeBinding, args)) + : null; + let selection = selectObservationScope(selectedTask, args.observation); + if (args.sourceSha256) { + const source = hashAppSource(args.app); + if (source.sha256 !== args.sourceSha256) { + throw new Error('live application source differs from the source selected for grading'); + } + } + args.selection = selection; + const declaredSuites = recipeBinding + ? suitesForRecipe(track, recipeBinding) + : suitesFor(track, Number(args.level)); + if (args.observation === 'scored') { + selection = attachRegressionScope(selection, recipeBinding, declaredSuites, + args.regressionChecks); + } else if (args.regressionChecks.length) { + throw new Error('observed grading cannot include regression checks'); + } + if (selection) { + const suiteIds = new Set(declaredSuites.map(suite => suite.id)); + const unmapped = selection.checks.filter(check => !suiteIds.has(check.executionId)); + if (unmapped.length) { + throw new Error(`selected recipe checks do not map to a declared suite: ${ + unmapped.map(check => check.stableKey).join(', ')}`); + } + } + const calibration = resolveCalibrationForRelease(recipeBinding?.release ?? null, { + trackRoot: track.dir, + stackBenchRoot: ROOT, + }); + const observationSuffix = args.observation === 'observed' ? '-observed' : ''; + const bundleArtifactId = `${args.parentAttemptId ?? args.label}-grade-bundle-l${args.level}${observationSuffix}`; + args.bundleArtifactId = bundleArtifactId; + mkdirSync(args.out, { recursive: true }); + // A structural abort can happen before any suite overwrites its old result. + // Remove all outputs from the prior grade. A cumulative level can rename + // inherited suites, so deleting only the current names leaves stale L1 + // evidence in an L2 result package. + clearPreviousGradeOutputs(args.out); + + console.log(`\n=== ${args.label} (${args.backend}) ===`); + console.log(` app: ${args.app}`); + console.log(` url: ${args.url}`); + if (recipeBinding && selection) { + console.log(` recipe: ${recipeBinding.alias} -> ${recipeBinding.release.id}@${recipeBinding.release.version} ` + + `(${recipeBinding.status}, ${recipeBinding.release.contentSha256.slice(0, 12)})`); + console.log(args.observation === 'observed' + ? ` scope: ${selection.checks.length} observed check(s), ${selection.observedPoints} observed point(s), 0 score contribution` + : ` scope: ${selection.checks.length} check(s), ${selection.scoredPoints} point(s)`); + if (selection.requested.packs?.length) console.log(` packs: ${selection.requested.packs.join(', ')}`); + if (selection.requested.features?.length) { + console.log(` features: ${selection.requested.features.join(', ')}`); + } + if (selection.requested.checks.length) console.log(` extra checks: ${selection.requested.checks.join(', ')}`); + } + + const bundle: Bundle = { + definitionSchemaVersion: track.schemaVersion, + recipeRelease: bundleRecipeRelease(recipeBinding), + calibration: calibration ? { id: calibration.id, version: calibration.version, + state: calibration.state, contentSha256: calibration.contentSha256 } : null, + label: args.label, track: args.track, backend: args.backend, url: args.url, app: args.app, + level: Number(args.level), observation: args.observation, + ...(args.sourceSha256 ? { source: { sha256: args.sourceSha256 } } : {}), + suites: {}, totals: {}, + selection: selection ? { ...selection, attemptedChecks: [], reportedChecks: [], notRun: [] } : null, + }; + const selectedPackIds = new Set(selection?.checks.map(check => check.packId) ?? []); + const selectedPackDefinitions = recipeBinding?.plan.packs + .filter(pack => selectedPackIds.has(pack.id)) ?? []; + const writeBundle = () => { + if (args.sourceSha256) { + const current = hashAppSource(args.app); + if (current.sha256 !== args.sourceSha256) { + bundle.error = 'application source changed while grading was in progress'; + bundle.outcome = { kind: 'harness_failure', phase: 'source-provenance', + reason: bundle.error }; + } + } + return writeArtifact(join(args.out, 'bundle.json'), { + kind: 'grade_bundle', + id: bundleArtifactId, + attempt: { id: bundleArtifactId, parentId: args.parentAttemptId ?? null }, + timestamps: { startedAt, completedAt: new Date().toISOString() }, + identities: recipeArtifactIdentities(recipeBinding?.release ?? null, { + calibration: calibration ? { id: calibration.id, version: calibration.version, + sha256: calibration.contentSha256, state: calibration.state } : null, + stackAdapter: { id: args.backend }, + }), + payload: bundle, + }); + }; + const recordApplicationAbort = () => { + bundle.totals = applicationFailureTotals(selection, declaredSuites); + }; + const freshenFailureMessage = () => lastResetOutcome?.phase === 'application-readiness' + ? `application did not become ready after database reset${lastResetFailure ? `: ${lastResetFailure}` : ''}` + : `database reset failed — scores would not be comparable${lastResetFailure ? `: ${lastResetFailure}` : ''}`; + const markRemainingNotRun = (reason: string): void => { + if (!bundle.selection) return; + const accounted = new Set([ + ...bundle.selection.attemptedChecks, + ...bundle.selection.notRun.map(check => check.stableKey), + ]); + bundle.selection.notRun.push(...bundle.selection.checks + .filter(check => !accounted.has(check.stableKey)) + .map(check => ({ stableKey: check.stableKey, reason }))); + }; + + if (args.applicationFailure) { + bundle.error = args.applicationFailure.reason; + bundle.outcome = args.applicationFailure; + recordApplicationAbort(); + markRemainingNotRun(`run aborted: ${bundle.error}`); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + + // Reset before EVERY step, not once per run: the lint and each suite create + // state of their own, so a single up-front reset leaves later suites grading + // dirty state — which silently lowers scores. + let lastResetFailure: string | null = null; + let lastResetOutcome: ResetOutcome = { kind: 'harness_failure', phase: 'database-reset' }; + const freshen = async () => { + if (!args.reset) return true; + const requiresReseed = executeStackCapability(STACK_ADAPTER_REGISTRY.get(args.backend), + 'reset', 'requires-reseed'); + const controlledRestart = track.reseedOnReset && Boolean(args.restartSpec) && requiresReseed; + if (controlledRestart) { + process.stdout.write(' stop application ... '); + try { + await controlBackend(args.restartSpec, 'stop'); + console.log('ok'); + } catch (error) { + const failure: Failure = error instanceof Error ? error : new Error(String(error)); + lastResetFailure = childFailureDetail(failure); + lastResetOutcome = { kind: 'harness_failure', phase: 'application-reset-control' }; + console.log(`FAILED (${lastResetFailure})`); + return false; + } + } + const reset = resetDatabase(args); + lastResetFailure = reset.detail; + lastResetOutcome = reset.outcome ?? { kind: 'harness_failure', phase: 'database-reset' }; + if (!reset.ok) return false; + // Do not grade until the reset application is reachable. + const waitUntilReady = async () => { + const ready = await waitForApplicationProbe(args.url); + if (!ready.ok) { + lastResetFailure = ready.detail; + lastResetOutcome = { kind: 'app_failure', phase: 'application-readiness', + appFailures: ['application-readiness'] }; + console.log(`FAILED (${ready.detail})`); + return false; + } + console.log('ok'); + return true; + }; + if (track.reseedOnReset && args.restartSpec && requiresReseed) { + process.stdout.write(' restart ... '); + // Judge restart success with the readiness probe. The restart command can + // leave a long-running server process behind, so the command also needs a deadline. + try { + // Do not give a background server an inherited pipe that keeps the + // synchronous restart command open. + await controlBackend(args.restartSpec, controlledRestart ? 'start' : 'restart'); + } catch (err) { + const failure: Failure = err instanceof Error ? err : new Error(String(err)); + lastResetOutcome = resetFailureOutcome(failure); + const detail = ((failure.stderr || '') + (failure.stdout || '') + (failure.message || '')) + .toString().trim().split('\n').slice(-3).join(' | ').slice(0, 300); + lastResetFailure = detail || null; + console.log('FAILED (application did not restart)'); + console.log(` control: ${JSON.stringify(args.restartSpec)}`); + console.log(` ${detail}`); + return false; + } + return await waitUntilReady(); + } + process.stdout.write(' ready ... '); + return await waitUntilReady(); + }; + + bundle.code = codeMetrics(args); + console.log(` code ... ${bundle.code.serverLoc} server LOC in ${bundle.code.serverFiles} files, ` + + `${bundle.code.totalLoc} total LOC, ${bundle.code.runtimeDeps} runtime deps`); + + // An interrupted mutation run leaves the app deliberately broken with a + // backup beside it. Grading that produces confident numbers for source + // nobody intended to measure. + const mutated = findMutationBackups(args.app); + if (mutated.length) { + bundle.error = `app still carries mutation backups (${mutated.join(', ')}) — its source is mutated, not the build under test`; + bundle.outcome = { kind: 'harness_failure', phase: 'mutation-cleanup', reason: bundle.error }; + markRemainingNotRun('run aborted because application source is still mutated'); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + + const prov = checkDatabaseProvenance(args); + bundle.provenance = prov; + console.log(` database ... ${prov.ok ? 'benchmark-owned' : `WRONG DATABASE — ${prov.reason}`}`); + if (!prov.ok) { + bundle.error = `app is not using the benchmark database: ${prov.reason}`; + bundle.outcome = { kind: 'app_failure', phase: 'database-provenance', reason: bundle.error, + appFailures: ['database-provenance'] }; + recordApplicationAbort(); + markRemainingNotRun('run aborted because database provenance was invalid'); + writeBundle(); + console.log('\nABORTED: results would not describe the benchmark environment.'); + process.exit(1); + } + + if (args.observation === 'scored') { + if (!(await freshen())) { + bundle.error = freshenFailureMessage(); + bundle.outcome = { ...lastResetOutcome, reason: bundle.error }; + if (bundle.outcome.kind === 'app_failure') recordApplicationAbort(); + markRemainingNotRun(`run aborted: ${bundle.error}`); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + let runtime = checkRuntimeDatabaseProvenance(args); + let proofError = null; + const proof = track.databaseProvenance; + const requiresRuntimeProof = ['mongodb', 'postgres'].includes(args.backend) + && args.databaseLease && args.reset; + if (requiresRuntimeProof && !proof) { + proofError = new Error(`${args.track} does not define a runtime database provenance check`); + } else if (requiresRuntimeProof && proof) { + try { + const checks = databaseProvenanceChecks(proof, recipeBinding); + const report = gradeSuite(args, { id: 'database-provenance', spec: proof.scenario }, + track, recipeBinding, bundleArtifactId, checks, + { recordSelection: false, captureMedia: false, + outputDirectory: join(args.out, 'database-provenance') }); + const marker = applicationDatabaseMarker(report, proof); + runtime = marker + ? checkRuntimeDatabaseProvenance(args, marker) + : { ok: false, verified: false, + reason: 'the configured application action did not produce one database marker' }; + } catch (error) { + proofError = error; + } + + // The proof writes unique data through the application. Remove it before + // linting and scored grading so the proof cannot change the result. + if (!(await freshen())) { + bundle.error = freshenFailureMessage(); + bundle.outcome = { ...lastResetOutcome, reason: bundle.error }; + if (bundle.outcome.kind === 'app_failure') recordApplicationAbort(); + markRemainingNotRun(`run aborted: ${bundle.error}`); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + } else if (['mongodb', 'postgres'].includes(args.backend) && !args.reset) { + runtime = { ok: null, verified: false, + reason: 'runtime marker proof requires database reset to isolate its write' }; + } + + if (proofError) { + bundle.outcome = databaseProvenanceFailure(proofError); + bundle.error = bundle.outcome.reason; + markRemainingNotRun('run aborted because runtime database provenance could not be verified'); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + + bundle.provenance.runtime = runtime; + console.log(` db runtime ... ${runtime.verified + ? runtime.ok ? runtime.reason : `WRONG DATABASE — ${runtime.reason}` + : runtime.reason}`); + if (runtime.ok === false) { + bundle.error = `app did not write its marker to the benchmark database: ${runtime.reason}`; + bundle.outcome = { kind: 'app_failure', phase: 'database-provenance', reason: bundle.error, + appFailures: ['database-provenance'] }; + recordApplicationAbort(); + markRemainingNotRun('run aborted because runtime database provenance failed'); + writeBundle(); + console.log('\nABORTED: application data came from outside the benchmark database.'); + process.exit(1); + } + try { + bundle.suites.lint = lint(args, selectedTask); + } catch (error) { + markRemainingNotRun('run aborted after contract lint failed to produce evidence'); + bundle.error = error instanceof Error ? error.message : String(error); + bundle.outcome = { kind: 'harness_failure', phase: 'contract-lint', reason: bundle.error }; + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + bundle.actions = checkActions(args); + } + + // Two numbers, kept apart on purpose. `score` is this level's own work. + // `regression` is whether the guarantees earned at earlier levels still hold. + // Summing them would hide the finding: an app that adds every L3 feature and + // silently breaks live stock updates from L1 would still read as progress. + let total = 0, max = 0, regTotal = 0, regMax = 0; + const dirty = false; + for (const suite of declaredSuites) { + const selectedChecks = selection?.checks.filter(check => check.executionId === suite.id) ?? []; + if (selection && selectedChecks.length === 0) { + console.log(` ${suite.id.padEnd(10)} ... not selected`); + continue; + } + if (!(await freshen())) { + bundle.error = freshenFailureMessage(); + console.log(` ${suite.id}: SKIPPED (${bundle.error})`); + markRemainingNotRun(`run aborted: ${bundle.error}`); + bundle.outcome = { ...lastResetOutcome, reason: bundle.error }; + if (bundle.outcome.kind === 'app_failure') recordApplicationAbort(); + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + if (bundle.selection) { + bundle.selection.attemptedChecks.push(...selectedChecks.map(check => check.stableKey)); + } + let r; + try { + r = gradeSuite(args, suite, track, recipeBinding, bundleArtifactId, selectedChecks); + } catch (error) { + markRemainingNotRun(`run aborted after ${suite.id} grader failure`); + bundle.error = error instanceof Error ? error.message : String(error); + bundle.outcome = { kind: 'harness_failure', phase: `grade:${suite.id}`, reason: bundle.error }; + writeBundle(); + console.log(`\nABORTED: ${bundle.error}`); + process.exit(1); + } + bundle.suites[suite.id] = r; + if (bundle.selection) bundle.selection.reportedChecks.push(...selectedChecks.map(check => check.stableKey)); + if (selection) { + bundle.packRuntime = aggregatePackRuntime( + Object.values(bundle.suites).filter(isGradePayload), + selectedPackDefinitions); + const exceeded = exceededPackBudgets(bundle.packRuntime); + if (exceeded.length) { + // Runtime budgets qualify the benchmark's known-good references; they + // are not a deadline for generated applications. A broken app can + // legitimately consume several assertion timeouts in one pack. Keep + // grading so it receives a complete repair report, while retaining the + // exceeded measurement for diagnostics and qualification policy. + console.log(` runtime ... ${exceeded.map(pack => + `${pack.id} ${pack.measuredRuntimeMs}ms > ${pack.budget.maxRuntimeMs}ms`) + .join(', ')} [recorded; grading continues]`); + } + } + if (suite.inherited) { regTotal += r.total; regMax += r.max; } + else { total += r.total; max += r.max; } + } + + bundle.totals = { + score: total, max, dirty, contractPass: isGradePayload(bundle.suites.lint) + ? null : bundle.suites.lint?.pass ?? null, + // null rather than 0/0 at L1, where there is nothing earlier to regress. + regression: regMax ? { score: regTotal, max: regMax } : null, + }; + writeBundle(); + + console.log(` ${'TOTAL'.padEnd(10)} ... ${total}/${max}${dirty ? ' [DIRTY]' : ''}`); + if (regMax) { + const kept = regTotal === regMax ? 'all earlier guarantees still hold' : `${regMax - regTotal} EARLIER GUARANTEE(S) LOST`; + console.log(` ${'REGRESSION'.padEnd(10)} ... ${regTotal}/${regMax} — ${kept}`); + } + console.log(` bundle: ${join(args.out, 'bundle.json')}`); +} + +if (process.argv[1] && pathToFileURL(resolve(process.argv[1])).href === import.meta.url) main(); diff --git a/tools/stack-bench/commands/stdb-report.ts b/tools/stack-bench/commands/stdb-report.ts new file mode 100644 index 00000000000..480f1be8a46 --- /dev/null +++ b/tools/stack-bench/commands/stdb-report.ts @@ -0,0 +1,287 @@ +#!/usr/bin/env node +// What did SpacetimeDB cost the model, and where did it get stuck? +// +// The score says which backend finished. It does not say what the model fought +// with on the way, and that is the part SpacetimeDB can act on: an error it hit +// seven times is a documentation or API problem with a name and a fix, where +// "cost more" is only a complaint. +// +// Token figures come from the CLI's own `usage` on each assistant message -- +// real input/output/cache counts, not bytes divided by four. An estimate is +// fine for a rough share; it is not fine for a number somebody is going to +// prioritise engineering work against. +// +// Reads the ARCHIVED transcripts under transcripts/