diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e7badef..50f189f 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -20,6 +20,18 @@ jobs: - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 24 + cache: npm + - name: Configure Socket Firewall for verifier dependencies + uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 + with: + token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} + allow-external-fork-fallback: true + - name: Install verifier dependencies + run: npm ci --ignore-scripts --no-audit --no-fund + - name: Remove Socket Firewall credentials before executing verifier source + uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 + - name: Run offline verifier tests + run: npm run check - name: Check shell style and test coverage policy run: | set -euo pipefail diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c168e71 --- /dev/null +++ b/.gitignore @@ -0,0 +1,4 @@ +node_modules/ +*.inventory.json +reports/* +!reports/.gitkeep diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index c70d0b9..979d31b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -5,7 +5,7 @@ Thanks for helping improve the WorkOS Socket Firewall GitHub Action. ## Development requirements - Bash on Linux or macOS -- Node.js 24 or later, matching the branch-repair JavaScript action runtime +- Node.js 24 or later and npm, matching the branch-repair JavaScript action runtime and CI - Go, for the pinned `shfmt` check - ShellCheck - Passwordless `sudo` and a disposable Linux runner for integration testing that modifies `/etc/hosts` @@ -26,11 +26,31 @@ bash scripts/scrub-lockfile.test.sh node --test scripts/scrub-npm-lockfile.test.mjs scripts/fix-lockfile.test.mjs bash scripts/build-release.test.sh bash scripts/publish-release.test.sh +npm ci --ignore-scripts --no-audit --no-fund +npm run check ``` +`npm run check` runs formatting and pure mocked Node tests offline after dependencies are installed. `npm test` and `npm run test:unit` both run all verifier suites. No credential or live discovery-ref lookup is needed for these tests. The install and pinned Go formatter command above may access the network; they are not offline tests. + CI runs the same static and unit checks on every pull request. The npm scrub smoke matrix tests the internal normalizer and `npm ci --ignore-scripts` against both npm filenames and lockfile versions 1–3 without credentials. Branch-repair tests exercise the action controller against a simulated GitHub API, asserting the target branch, single-file commit, expected-head race protection, no-op, fork/default-branch guards, and denied writes. Release-tree tests run those same controller tests using the packaged runtime. Token-backed GitHub-hosted smoke jobs additionally exercise every supported package manager and a scrubbed npm lockfile with public registry DNS blocked. -The minimum test-coverage policy is one shell test suite for every executable shell source file. Changes to supported package-manager behavior must also include a token-backed frozen-lockfile smoke test. +Existing public/fork secret gates and token-backed smoke jobs are independent of the detector. Never add a live organization audit or release snapshot verification to normal source CI. + +The minimum test-coverage policy is one shell test suite for every executable shell source file. Verifier changes require synthetic Node fixtures for per-download setup/teardown boundaries, opaque execution, immutable-SHA reads, partial failures and private reporting. Preserve REST/GraphQL token-visible inventory reconciliation without claiming it proves organization-wide completeness. Changes to supported package-manager behavior must also include a token-backed frozen-lockfile smoke test. + +## Operator-only commands + +`npm run inventory` and `npm run audit:live` are read-only manual commands using an existing authorized `gh` session. Confirm organization-wide read access separately before making organization-wide claims. No scheduling or remediation is implicit. See README for the scan scope and limitations. + +For recurring agent use, `npm run review:weekly -- --state /durable/private/ledger.json` runs the audit and applies explicit fingerprint-bound decisions. `npm run review -- init|show|record|forget --state ...` manages that private ledger; see README for exact arguments. Never auto-initialize a missing weekly ledger, auto-acknowledge new cases, expire decisions by elapsed time, or publish the ledger/source report in this public repository. The weekly JSON intentionally contains private repository/job metadata; raw audit terminal output remains counts-only. + +Weekly-review changes require deterministic replay tests, unrelated-commit/order stability, new-repository/job and input-drift reopening, immutable repository IDs, durable decisions across visibility loss, and fail-closed missing/corrupt/partial state and concurrent writes. Test the next run after recording/fixing a finding, not only the first clean result. This adds report fingerprints/IDs to schema3 without changing primary or strict classification semantics. + +Audit schema3 separates primary observed configuration from strict `assuranceDispositions`. Acceptance tests must show that arbitrary execution before or after setup, pipelines, package scripts and script-only siblings do not invent missing installs or erase configuration. An actual uncovered install, explicit configuration conflict or unresolved relevant source must remain visible. Cover conditional Corepack state, scoped environment/registry changes, quoted data versus executable commands, executor payload boundaries, composite input binding and snapshot-bound reusable-workflow references. Preserve strict security/opacity diagnostics and source acquisition limits. `integrated` is configuration evidence, not runtime proof. `no-js-ci` is absence of an observed install path, not assurance about script bodies. Do not introduce a historical cohort, coverage percentage or blanket repository allowlist. Approved source exclusions must remain explicitly recorded, pin-checked and separate from protection claims. + +Full results are owner-only, ignored `reports/inventory.json` and `reports/live-audit.json`; terminal JSON contains sanitized counts. Audit `scanErrors`/`scanStatus` distinguish operational failures from discovered gaps. Exit 1 includes partial scans; exit 0 can still include `needs-sfw` or `needs-review`. Do not force-add reports, expose inventory/source in logs, or upload full reports as artifacts. + +`npm run verify-action` is a separate live, strict historical release snapshot check, not a normal CI check. Advancing `v1` or changing the current release manifest can intentionally invalidate it. Mocked release tests use `tools/rollout/fixtures/release-manifest.txt`, preserving integrity regression tests without binding source CI to future manifest changes. Keep the actual runtime manifest and action-only publication boundaries intact. ## Pull request guidelines diff --git a/README.md b/README.md index aa2f46c..780db1c 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,7 @@ It does not route package publication or Python, Java, Go, Ruby, Rust, .NET, pri Consumers must pin the full 40-character SHA of the reviewed action-only `v1` commit. Do not execute a mutable tag, branch, abbreviated SHA, or normal source commit. ```yaml -uses: workos/setup-socket-firewall@ # v1 +uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 ``` The moving `v1` tag and `action-release/v1` branch are for human discovery and Renovate lookup. The repository’s normal source history contains tests and rollout tooling; each action-only release commit contains only the files in `release-manifest.txt`. @@ -37,7 +37,7 @@ Run package-manager setup first, then configure SFW before the first dependency registry-url: https://registry.npmjs.org/ - name: Configure Socket Firewall - uses: workos/setup-socket-firewall@ # v1 + uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 with: token: ${{ secrets.SOCKET_FIREWALL_TOKEN }} @@ -48,7 +48,7 @@ For a Bun dependency-install job, also set `configure-bun: true`. This writes a The token is fail-closed by default. Private, internal, trusted/default-branch, and Dependabot jobs stop before dependency download when the token is absent. -`SOCKET_FIREWALL_TOKEN` is an organization secret. Dependabot uses a separate secret store: provision the same secret there for dependency-update runs that must pass, or accept the intentional fail-closed result. Ask in `#ask-foundation` about repository selection or token delivery. +`SOCKET_FIREWALL_TOKEN` is an organization secret available to private and internal repositories. Public repositories never receive it; an approved public repository is instead individually selected into the separate `PUBLIC_SOCKET_FIREWALL_TOKEN` organization secret and passes that secret to the same `token` input. Dependabot uses a separate secret store: provision the same secret there for dependency-update runs that must pass, or accept the intentional fail-closed result. Ask in `#ask-foundation` about repository selection or token delivery. ### Public external-fork usage @@ -68,7 +68,7 @@ steps: node-version: 22 - name: Configure Socket Firewall - uses: workos/setup-socket-firewall@ # v1 + uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 with: token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} allow-external-fork-fallback: true @@ -134,7 +134,7 @@ When an existing job must both install and publish, run the teardown entrypoint ```yaml - name: Configure Socket Firewall - uses: workos/setup-socket-firewall@ # v1 + uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 with: token: ${{ secrets.SOCKET_FIREWALL_TOKEN }} @@ -142,7 +142,7 @@ When an existing job must both install and publish, run the teardown entrypoint - run: pnpm build - name: Restore public package registry - uses: workos/setup-socket-firewall/teardown@ # v1 + uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 - name: Publish package run: pnpm publish --access public --provenance --no-git-checks @@ -195,7 +195,136 @@ No maintainer runs local release commands. The workflow uses only the repository-scoped `GITHUB_TOKEN` with `contents: write`, serializes releases, skips stale successful commits when `main` has advanced, and can be retried through `workflow_dispatch`. A future breaking release must change the reviewed channel to `v2`; it must not repurpose `v1`. -Never publish the normal source commit as an action release: it contains tests and, after HELP-724 Phase 2, the one-time rollout verifier and report source. +Never publish the normal source commit as an action release: it contains tests and the read-only CI gap detector source. + +## Read-only CI gap detector + +The source branch includes an **operator-run candidate detector**, not runtime proof of protection. It never enters the action release and does not change repositories, create PRs, schedule scans, or diagnose general CI health. + +```bash +npm ci --ignore-scripts --no-audit --no-fund +npm run check # offline formatting and mocked Node tests +npm run inventory # read-only, token-visible WorkOS inventory +npm run audit:live # read-only, default-branch workflow candidate scan +``` + +`inventory` and `audit:live` require an existing authenticated `gh` session. Organization-wide conclusions require independently confirmed read access to **every intended repository**, including private/internal repositories (typically `repo` and `read:org`, with organization/SSO authorization as applicable). REST/GraphQL reconciliation checks consistency of the token-visible inventory; both APIs agreeing does **not** prove hidden repositories are absent. Archived repositories are counted but not scanned. Each repository's tree and workflow/local-action source use one captured immutable default-branch SHA; different repositories are not captured transactionally. + +### Weekly agent review: stable findings and durable decisions + +Use the review command for recurring work, rather than treating every raw `needs-review` row as a new protection request. It runs the same read-only audit, but keeps **unacknowledged protection gaps**, **unacknowledged uncertainty**, **known decisions**, and **unobserved repositories** separate. It creates no schedule, notification, consumer PR, or protection waiver. + +Use a private, durable ledger shared by successive executions. Do not put it in an ephemeral checkout, initialize it every week, or commit it to this public repository. Back it up or version it in your private operations storage. A missing/corrupt ledger fails closed; it never silently restarts with an empty exception list. + +```bash +STATE=/durable/private/socket-firewall-review.json +REPORT=/durable/private/socket-firewall-audit.json + +# Once only. Fails rather than overwriting any existing ledger. +npm run --silent review -- init --state "$STATE" + +# This is the command to give the weekly agent. +npm run --silent review:weekly -- --state "$STATE" --report "$REPORT" +``` + +The JSON result contains `snapshot`, `needsProtection`, `needsReview`, `known`, and `unobserved`. Each case has an opaque stable `id`, immutable GitHub repository ID, repository/workflow/job, reason codes and an input fingerprint. Names and links in this output are private operational metadata; unlike the raw audit CLI's counts-only output, do not post it to a public log. The separate full audit still retains all original classifications and strict assurance diagnostics. + +For a finding the agent has investigated, record a specific reason, evidence URL, and recorder identity against the **snapshot it actually inspected**: + +```bash +npm run --silent review -- record --state "$STATE" \ + --expected '' --case '' \ + --kind known-review \ + --reason 'Known caller-selected source uncertainty; not evidence of a missing setup.' \ + --evidence 'https://github.com/owner/repo/pull/123' --by 'reviewer identity' +``` + +Decision kinds: + +- `known-review`: acknowledge a known uncertainty/limitation without claiming protection. +- `tracked-gap`: retain a real, unfinished gap linked to its remediation work, without presenting it as new every week. +- `exception`: record an explicitly justified policy exception with its approval evidence. The command records a decision; it does **not** grant approval or relabel the underlying audit as integrated. Follow your existing policy authority before using this kind. + +`--repository workos/name` can replace `--case` to record the current findings in that exact repository at the inspected snapshot. It expands to individual fingerprinted cases, **not** a wildcard repository allowlist: a new job or changed case is not included. `review show --state "$STATE"` reads the saved view without GitHub access. `review forget --state "$STATE" --case '' --expected ''` explicitly revokes a decision. Agents should report unacknowledged cases and visibility/scan failures, not repeatedly re-open known cases or automatically acknowledge new findings. + +Determinism and invalidation rules: + +- Stable repository IDs survive renames but do not transfer decisions to a deleted/recreated repository with the same name. Cases are scoped to a workflow/job (or a parse/exclusion finding), never the entire organization or a historical cohort. +- Job inputs, workflow environment/defaults/triggers, declared upstream jobs, consumed local actions, package-manager configuration and relevant local reusable sources bind each decision. Canonical object ordering and YAML parsing remove API completion order/key order and YAML-comment noise. Unrelated commits, README changes, other independent jobs and wall-clock time do not invalidate it. Raw audit timestamps and captured head SHAs are provenance, not exception identities. +- A changed input invalidates its decision durably. Reverting that input later does not silently revive the old decision. Fixing/removing a job and subsequently reintroducing the old gap is a new review event. Decisions have no automatic TTL/calendar expiry. +- A previously tracked finding/decision whose repository is no longer visible is `unobserved`, **not resolved**; it neither expires nor erases a decision. A returning unchanged repository retains it. An unreadable/partial audit or inventory disagreement fails and leaves the last-good ledger untouched. Never use that retained view as proof the failed run was clean. +- Run/record/forget/init serialize through an exclusive `.lock` file; writes replace the ledger atomically with mode `0600`. Concurrent writers fail rather than lose decisions. A crashed writer leaves a lock: verify no writer is active before removing that exact lock. There is no timeout-based lock stealing or automatic ledger reset. + +These are stable acknowledgements of **bounded static findings**, not attestations about arbitrary scripts, remote code or credentials. A known dynamic-source limitation remains a limitation. Default-branch sources can genuinely change between live scans; deterministic output means the same observed inputs and ledger produce the same review list, not that real configuration changes are ignored. Fingerprint format changes require an explicit migration/review, never silently discarding the ledger. + +### Results and privacy + +Terminal output is sanitized JSON counts, not repository names or workflow source. Full inventory and audit JSON go to `reports/inventory.json` and `reports/live-audit.json`, respectively. These paths are ignored; reports are atomically replaced with owner-only (`0600`) permissions. Keep them private: never force-add, publish, or upload reports as CI artifacts. Both commands take no CLI flags; a later run replaces the previous report. + +Audit JSON schema **3** separates primary `dispositions` (observed dependency-install integration) from `assuranceDispositions` (strict execution/security review). Successfully inspected repositories retain both dispositions, their `headSha`, workflow/job operations and violations; read errors remain explicit error rows. Each job's `integration` includes per-download `covered`, `covered-with-exclusion`, `fork-exception`, `gap` or `unresolved` configuration evidence, reason codes and diagnostic notes. `additionalJsPaths` records explicit JS invocations outside the direct-install grammar, distinguishing `setup-observed` from unresolved configuration. `runtimeVerification` is always `not-performed`. `scanStatus: complete` means the token-visible scan completed, **not** that all jobs are protected. `partial` means one or more repositories have an `audit-error` row. CLI exit **0** means the scan completed, even when gaps/review candidates were found; exit **1** means an operational failure (including partial scans). Consumers should inspect dispositions separately from the exit code. Fatal inventory/API/command errors emit a sanitized error and exit 1; they do not refresh the report, so check its timestamp before use. + +| Repository disposition | Meaning | +| --- | --- | +| `needs-sfw` | A recognized download has a definite missing/unsupported setup interval in its own job. | +| `needs-review` | Install configuration, an explicit JS invocation, or relevant workflow/local-action source could not be resolved. Ordinary opaque scripts alone do not cause this result. | +| `integrated` | SFW configuration is observed for the identified install paths, possibly with an explicit public-fork exception. This is not execution or traffic assurance. | +| `integrated-with-exclusions` | Observed integration has an approved, source-pinned exclusion. Inspect the repository's `exclusions`; the excluded source is not inspected by SFW. | +| `audit-error` | A repository read failed or was incomplete; never a clean result. | +| `no-ci`, `no-js-ci`, `empty` | No observed in-scope download candidate; not proof that remote actions or arbitrary code cannot download packages. | + +Primary integration is not a rollout percentage or a certification of every command. For example, setup → `npm ci` → `npm test` can be `integrated` while strict assurance remains `needs-review`. Without setup, the same direct install remains a `needs-sfw` candidate even though the later test is opaque. A new uncovered install job cannot be masked by an integrated sibling. An ordinary script-only sibling does not invent another install path: its code remains unverified in assurance diagnostics. + +Strict job `status` and `assuranceDisposition` retain opaque execution and security findings (`blocked-trust`, `unsafe-publish`, `blocked-yarn`, etc.). **Inspect those findings separately; `integrated` does not dismiss them.** A strict `protected` or `safe-publish` status remains static evidence, not runtime or lifecycle-hook proof. Schema1 consumers must not interpret newer primary dispositions as the old assurance verdicts. Schema2 consumers must handle schema3's `integrated-with-exclusions` disposition and `covered-with-exclusion` download status explicitly, not count them as fully covered. + +### Approved source exclusions + +`tools/rollout/exclusions.mjs` records the approved `tree-sitter-kotlin` GitHub source in `oagen`, `oagen-emitters`, `openapi-spec` and `workos`, including its exact path, version, URL, integrity pin and approval provenance. This archive intentionally downloads outside SFW; npm-registry dependencies still use the firewall. The existing `NPM_CONFIG_REPLACE_REGISTRY_HOST=npmjs` setting is retained because rewriting GitHub archives to the npm proxy breaks installation. + +For the npm repositories, each scan reads the manifest and version-3 lockfile at the same captured repository SHA. Manifest/lock disagreement, workspace or unsupported override configuration, a changed pin or any additional non-npm source makes the exclusion `stale` and keeps the repository in review; an unreadable lockfile is an audit error. OpenAPI's approval allows ordinary npm-registry dependency bumps and flat registry-only version overrides (including npm aliases): it is not tied to unrelated version strings. Nested override forms, Git/URL/file overrides, a changed Kotlin source/version/integrity or changed project registry settings still require review. This override policy does not extend to the other npm repositories. For step-level environment exceptions, only direct, repository-root `npm install`/`npm ci` steps with that exact setting in the recorded workflows receive the exclusion, after one unconditional default-source checkout. Alternate checkout refs/repositories/paths, nested working directories and composite-action installs are not excused. Missing setup, additional gaps, unknown execution contexts and other configuration overrides remain findings. Qualified results use `integrated-with-exclusions`, never an unqualified protection claim. Strict assurance remains unchanged. + +WorkOS's separate approval checks the exact Git override in Rush's `pnpm-config.json` against the pnpm v9 lockfile, including the Kotlin codeload URL, version and integrity. Ordinary registry override versions may change together in the config and lock without invalidating accepted jobs. The fingerprint still covers override names and alias targets, the exact Git source and all other Rush settings (including lifecycle-script policy). This normalization applies only while the source approval matches; additional network sources or a changed approved source require review. Existing local vendored tarballs are not network exceptions. This record never waives a CI step or certifies arbitrary workspace scripts. Large source files omitted by GitHub's Contents API are read via its returned immutable Git blob (up to 10 MiB), with size, encoding and blob-SHA verification; an incomplete read remains an audit error. + +OpenAPI's exception is recorded from its project `.npmrc`, not used to waive any step-level environment override. Unresolved checkout/action provenance and conditional execution remain findings even when the archive pin matches. + +### Static limits + +The primary question is **whether SFW is configured for identified dependency-install paths**, not whether every command can be proved to preserve protection. Arbitrary scripts, remote actions, `curl`/shell bootstraps and package-script bodies remain unverified; they do not erase or synthesize observed setup. A script before setup or after teardown is not presumed to install dependencies. An explicit install in that position is still a gap. `no-js-ci` means no identified dependency-install path in the supported workflow source—not that the repository contains no JavaScript or that its scripts cannot install packages. + +The small grammar recognizes npm/pnpm/Bun installs, download-capable executors, Yarn blockers and setup/teardown ordering. A lexical boundary scanner keeps quoted data and command substitutions inside their containing command rather than inventing installs from text. Pipelines and executor payload arguments do not invalidate configuration simply because their execution is opaque. For `npx`/`bunx`, a literal target after optional `-y`/`--yes` separates installer options from program arguments, including quoted or variable-valued payloads. Pre-target registry overrides remain gaps; unsupported installer options, dynamic installer targets and explicit context changes can still require review. Uninterpreted shell structure can establish configuration presence without proving whether a particular command executes; it cannot establish a definite missing install when the command may only be data. + +Configuration checks retain the approved action SHA, visibility-appropriate token, Bun configuration and public-fork exception rules. Disabled or continued-on-error setup cannot silently cover an install. Conditional setup covers an install only when the install requires every recognized setup predicate. The bounded grammar supports boolean-string output comparisons (dependency jobs or uniquely identified earlier steps) and literal `github.event_name` equalities, with at most two predicates joined by `&&`. A stricter install condition can imply setup, never the reverse. Future/duplicate step IDs, mutable environment guards, status functions, partial expression interpolation, unsupported expressions and unmatched conditions remain unresolved; strict execution assurance is unchanged. Corepack uncertainty survives conditional or targeted controls. The action-exported `SFW_BUN_CONFIG_PATH` is recognized in Bun's `--config` argument when Bun setup is enabled; explicit overrides of that variable remain configuration questions. Whole-value composite input references in `with`/`env` are bound to caller values/defaults without evaluating conditions or interpolating shell text. Missing/malformed local source, cycles and expansion limits remain review findings. + +Known registry writes and teardown invalidate the relevant configuration; a command-local override does not contaminate later installs. npm/pnpm registry setters, configuration-file writes (including `tee`), standalone relevant assignments and GitHub environment-file writes remain findings rather than being treated as generic opacity. Reading a configuration variable is not a mutation. HOME/PATH/NODE_OPTIONS overrides, relevant exported variables within a run block, unsupported environment wrappers, dynamic mappings and job containers remain configuration questions. Ordinary application secrets/settings, version metadata and sibling service containers do not invalidate host setup ordering. Job-level guards affect shared reachability; explicit status predicates and conditional configuration transitions remain conservative. Exact npm `--replace-registry-host=always` preserves the configured registry; other modes require review unless covered by an approved source exclusion. + +Literal repository-relative default working directories and npm `--prefix` arguments after the install verb are treated like explicit run-step working directories. Supported install options include `--include=optional`, `--omit=dev`, `--package-lock=false` and Bun's literal `--os="*" --cpu="*"`; unknown options and dynamic or escaping directory paths remain reviewable. Root local actions (`uses: ./`) are read from `action.yml` or `action.yaml` at the captured SHA. Entirely commented-out workflow files have no active jobs; malformed or missing source still requires review. + +Literal leading npm workspace selectors (`-w`, `--workspace`, and `--workspace=`) retain the underlying command classification: scripts stay unverified script execution, while installs and explicit `npm exec -- ` remain download candidates. Literal npm package selectors in `npx --package`/`-p` are recognized without treating executable payload flags as installer configuration. A workspace selector may contain a matrix component only when every declared alternative is a literal, safe single path component and the matrix has no `include` overrides; this does not verify any script body. Runner-temp tarball arguments such as `"$RUNNER_TEMP"/package-*.tgz` are local inputs to npm/npx, not alternate registry URLs; overriding `RUNNER_TEMP` invalidates that recognition's configuration evidence. Other dynamic selectors, unsupported options, and post-teardown executor downloads remain findings. + +The exact three-line `set -euo pipefail` → `pnpm_package="$(node --print 'require("./package.json").packageManager')"` → `npm install --global "$pnpm_package" --ignore-scripts --no-audit --no-fund` bootstrap can resolve a plain `pnpm@major.minor.patch` from the captured root manifest. It requires one default-source checkout, root working directories, no project `.npmrc`, and no preceding shell/local/mutable action steps. Unsupported pins, source changes, extra commands and configuration overrides remain unresolved. This is snapshot-based configuration evidence, not runtime verification of the JSON reader or preceding pinned actions; strict assurance stays unverified. + +The exact standard shell template `bash --noprofile --norc -euo pipefail {0}` is recognized alongside `bash`/`sh`. A clean-room `env -i` npm invocation is recognized only when it retains the exact inherited HOME, PATH, action registry and npm-config path; missing, changed, duplicated or additional configuration variables remain unresolved. Local `npm version`, argument-free/format-only `npm pack`, and literal Bun script filenames remain unverified execution, not invented download operations. Remote or dynamic `npm pack` targets remain review candidates. + +A setup-only `NPM_CONFIG_USERCONFIG` pointing to a named file under `${{ runner.temp }}` is recognized because the pinned action validates that path and configures both it and `HOME/.npmrc`. Install-step overrides remain unresolved. Public fallback expressions remain recorded as possible `fork-exception` paths: the pinned action validates the value and independently permits fallback only for public external-fork pull requests. Neither recognition is runtime proof or a repository allowlist. + +Local reusable workflows are already read at the same repository SHA and contribute their actual primary result instead of an automatic unknown. Missing targets and cycles remain reviewable. A remote reusable call can be resolved as having no observed JS install only when that workflow was already captured in the same organization's inventory at the requested default branch or exact SHA. A different pin/tag is never replaced with the current body. Remote callees with installs require caller-specific context and remain unresolved; no additional sources are fetched by this cross-reference step. + +Local action lookup understands literal checkout mount paths only when the preceding checkout selects the captured repository and source (default selection, `github.sha`, or the exact captured SHA). A different repository/ref, conditional or sparse checkout, or unknown potentially overlapping destination remains an explicit checkout-provenance finding. The detector does not evaluate environment assignments or reusable-workflow input expressions to guess a destination/ref. In particular, a caller-selected checker revision cannot be replaced with the checker's current default-branch implementation. In reusable workflows, implicit checkout and `github.repository`/`github.sha` refer to the caller; resolving a local source from the callee requires an explicit matching repository and source selection. + +A local setup/teardown entrypoint can supply configuration evidence when all four runtime blobs match the immutable approved release, with a preceding resolved checkout. Changed/missing blobs or different source selections do not inherit this recognition. This is a source-equivalence check, not a self-test exemption or proof that earlier runtime code did not modify the files. Matrix-dependent Bun configuration, intentional direct-public fixture installs and expected Yarn failures remain reviewable; no unapproved test or archive exception is added. + +An adjacent fail-closed `bun install --help` check for `--offline` permits the exact non-composite run-step canonicalization command `bun install --lockfile-only --offline --ignore-scripts --registry=https://registry.npmjs.org/` to be recorded as capability-guarded offline validation, not a registry download. The guard must exit 1 when support is absent; intervening commands, custom shells, relevant environment overrides, composite expansion, missing flags and unguarded/later invocations do not inherit the recognition. A bare `--offline` flag is insufficient because older Bun versions can ignore it. Strict runtime assurance remains unverified. + +Literal quoted `cat` heredoc bodies are data rather than shell commands, including inside command substitutions. Shell-fed/piped heredocs, missing delimiters and GitHub expressions in the body are not granted that recognition. The lexer bounds substitutions per command (20), nesting (100), and emitted commands/words (1,000), so many independent metadata reads do not exhaust a whole job's substitution budget. Exhausted or malformed source always retains a review finding. + +Only top-level `.github/workflows/*.yml|yaml`, `.depot/workflows/*.yml|yaml` and referenced local actions in the captured scope are inspected. The detector does not execute code, inspect arbitrary script/action implementations, validate credentials, prove traffic routing or certify bespoke controls. Security findings and unverified execution remain separate assurance diagnostics. No historical cohort, coverage percentage or blanket repository allowlist is used. + +### Manual release snapshot verification + +```bash +npm run verify-action +``` + +This separate **live, strict snapshot check** validates discovery refs, the approved signed action-only SHA, exact allowlisted tree, and runtime entrypoints against `tools/rollout/constants.mjs` and the local release manifest. It is not part of `npm run check` or ordinary source tests. It is expected to fail when `v1` advances or the source manifest no longer matches that historical snapshot; this alone is not evidence the current release is unsafe. Review/update the snapshot deliberately for a new release, rather than weakening integrity checks. Offline unit tests use an explicit historical manifest fixture so future manifest additions cannot break unrelated source CI. `tools/rollout/fixtures/approved-release.json` also records curated public API responses for the approved immutable commit/tree/action contents and the newer discovery refs observed at capture time. Positive historical tests explicitly synthesize only the old ref targets; separate tests assert that the captured moved refs are rejected. These fixtures are independent of production constants and contain no credential, inventory, or private repository source. ## Contributing diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..e070f21 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,52 @@ +{ + "name": "@workos/setup-socket-firewall-rollout", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@workos/setup-socket-firewall-rollout", + "version": "0.0.0", + "dependencies": { + "yaml": "2.9.0" + }, + "devDependencies": { + "prettier": "3.9.6" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/prettier": { + "version": "3.9.6", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.6.tgz", + "integrity": "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..091e91e --- /dev/null +++ b/package.json @@ -0,0 +1,28 @@ +{ + "name": "@workos/setup-socket-firewall-rollout", + "version": "0.0.0", + "private": true, + "description": "Read-only Socket Firewall CI gap detector", + "type": "module", + "engines": { + "node": ">=22" + }, + "scripts": { + "audit:live": "node tools/rollout/cli.mjs audit", + "review": "node tools/rollout/review-cli.mjs", + "review:weekly": "node tools/rollout/review-cli.mjs run", + "check": "npm run format:check && npm test", + "format": "prettier --write package.json tools/rollout/*.mjs", + "format:check": "prettier --check package.json tools/rollout/*.mjs", + "inventory": "node tools/rollout/cli.mjs inventory", + "test": "node --test tools/rollout/*.test.mjs", + "test:unit": "node --test tools/rollout/*.test.mjs", + "verify-action": "node tools/rollout/cli.mjs verify-action" + }, + "dependencies": { + "yaml": "2.9.0" + }, + "devDependencies": { + "prettier": "3.9.6" + } +} diff --git a/renovate.json b/renovate.json index 2ce3453..a6f8814 100644 --- a/renovate.json +++ b/renovate.json @@ -1,5 +1,13 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": ["github>workos/renovate-config:public"], - "enabledManagers": ["github-actions"] + "enabledManagers": ["github-actions", "npm"], + "packageRules": [ + { + "description": "Update the approved Socket Firewall SHA only with its verifier constant", + "matchManagers": ["github-actions"], + "matchDepNames": ["workos/setup-socket-firewall"], + "enabled": false + } + ] } diff --git a/reports/.gitkeep b/reports/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tools/rollout/audit-cleanup.test.mjs b/tools/rollout/audit-cleanup.test.mjs new file mode 100644 index 0000000..eb87944 --- /dev/null +++ b/tools/rollout/audit-cleanup.test.mjs @@ -0,0 +1,582 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { stringify } from "yaml"; +import { auditRepository } from "./audit.mjs"; +import { classifyJob, classifyWorkflow } from "./classify.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; + +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { + token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}", + "configure-bun": true, + }, +}; +const context = { visibility: "private", path: ".github/workflows/ci.yml" }; +const inspect = (steps, extra = {}) => + classifyJob("build", { steps, ...extra }, context, ["push"]); + +test("workspace matrix selectors require bounded literal values", () => { + const strategy = { matrix: { app: ["research", "incident-watch"] } }; + for (const run of [ + "npm -w apps/${{ matrix.app }} run build", + 'npm --workspace="apps/${{ matrix.app }}" run typecheck', + ]) { + assert.equal( + inspect([setup, { run: "npm ci" }, { run }], { strategy }).integration + .disposition, + "integrated", + ); + assert.equal(inspect([{ run }], { strategy }).status, "unknown"); + for (const matrix of [ + { app: ["research; npm install evil"] }, + { app: ["../elsewhere"] }, + { app: ["--registry=elsewhere"] }, + { app: [] }, + { app: ["research"], include: [{ app: "unsafe value" }] }, + "${{ fromJSON(needs.matrix.outputs.apps) }}", + ]) + assert.equal( + inspect([{ run }], { strategy: { matrix } }).integration.disposition, + "needs-review", + ); + } + const run = "npm -w apps/${{ matrix.app }} install"; + assert.equal( + inspect([{ run }], { strategy }).integration.disposition, + "needs-sfw", + ); + assert.equal( + inspect([setup, { run }], { strategy }).integration.disposition, + "integrated", + ); +}); + +test("runner-temp tarballs stay local, without excusing runner-temp overrides", () => { + for (const run of [ + 'npm install --no-save --ignore-scripts "$RUNNER_TEMP"/example-*.tgz', + 'npx --yes --package "$RUNNER_TEMP"/example-*.tgz example --list', + ]) { + assert.equal( + inspect([setup, { run }]).integration.disposition, + "integrated", + ); + assert.equal(inspect([{ run }]).integration.disposition, "needs-sfw"); + assert.equal( + inspect([setup, { run }], { + env: { RUNNER_TEMP: "https://example.invalid" }, + }).integration.disposition, + "needs-review", + ); + assert.equal( + inspect([setup, { run: `RUNNER_TEMP=https://example.invalid\n${run}` }]) + .integration.disposition, + "needs-review", + ); + } + for (const target of [ + "$ARTIFACTS/example.tgz", + "$RUNNER_TEMP/../example.tgz", + "$RUNNER_TEMP/$PACKAGE.tgz", + ]) + assert.equal( + inspect([setup, { run: `npm install "${target}"` }]).integration + .disposition, + "needs-review", + ); +}); + +test("clean-room npm preserves only the exact inherited registry and config", () => { + const prefix = + 'env -i CI=true HOME="$HOME" PATH="$PATH" NPM_CONFIG_REGISTRY="${NPM_CONFIG_REGISTRY:?}" NPM_CONFIG_USERCONFIG="${NPM_CONFIG_USERCONFIG:-$HOME/.npmrc}"'; + for (const command of ["npm ci", "npm ci --prefix web"]) { + const run = `${prefix} ${command}`; + assert.equal( + inspect([setup, { run }]).integration.disposition, + "integrated", + ); + assert.equal(inspect([{ run }]).integration.disposition, "needs-sfw"); + assert.equal(inspect([setup, { run }]).status, "unknown"); + } + for (const run of [ + `${prefix.replace('PATH="$PATH"', 'PATH="/other"')} npm ci`, + `${prefix.replace('HOME="$HOME"', 'HOME="/other"')} npm ci`, + `${prefix.replace('"${NPM_CONFIG_REGISTRY:?}"', '"https://example.invalid"')} npm ci`, + `${prefix.replace('"${NPM_CONFIG_USERCONFIG:-$HOME/.npmrc}"', '"/other/.npmrc"')} npm ci`, + `${prefix} NODE_OPTIONS=--require=evil.cjs npm ci`, + `${prefix} HOME="$HOME" npm ci`, + "env -i npm ci", + ]) + assert.notEqual( + inspect([setup, { run }]).integration.disposition, + "integrated", + run, + ); +}); + +test("standard explicit Bash and local script commands are not unknown installer options", () => { + const shell = "bash --noprofile --norc -euo pipefail {0}"; + assert.equal( + inspect([setup, { run: "npm ci", shell }]).integration.disposition, + "integrated", + ); + assert.equal( + inspect([{ run: "npm ci", shell }]).integration.disposition, + "needs-sfw", + ); + for (const custom of ["bash --rcfile /other {0}", "python {0}"]) { + assert.equal( + inspect([setup, { run: "npm ci", shell: custom }]).integration + .disposition, + "needs-review", + ); + } + for (const run of [ + 'npm version "$VERSION" --no-git-tag-version', + "npm pack --json", + "packed=$(npm pack --json)", + "bun scripts/build-binary.ts --target $TARGET", + ]) { + const result = inspect([{ run }]); + assert.equal(result.integration.disposition, "no-js-ci", run); + assert.equal(result.status, "unknown", run); + } + for (const run of [ + "npm pack @scope/remote", + "npm pack $PACKAGE", + "npm pack --pack-destination $DIR", + "bun $SCRIPT", + ]) + assert.equal( + inspect([{ run }]).integration.disposition, + "needs-review", + run, + ); +}); + +test("matching immutable output guards cover only the same guarded installs", () => { + const ready = { id: "ready", run: 'echo enabled=true >> "$GITHUB_OUTPUT"' }; + for (const guard of [ + "steps.ready.outputs.enabled == 'true'", + "needs.prepare.outputs.enabled == 'true'", + ]) { + const guardedSetup = { ...setup, if: guard }; + const install = { run: "npm ci", if: `\${{ ${guard} }}` }; + const result = inspect([ready, guardedSetup, install]); + assert.equal(result.integration.disposition, "integrated"); + assert.equal(result.status, "unknown"); + assert.equal( + inspect([ready, guardedSetup, { run: "npm ci" }]).integration.disposition, + "needs-review", + ); + assert.equal( + inspect([ + ready, + guardedSetup, + { ...install, if: guard.replace("'true'", "'false'") }, + ]).integration.disposition, + "needs-review", + ); + assert.equal( + inspect([ready, { ...guardedSetup, "continue-on-error": true }, install]) + .integration.disposition, + "needs-review", + ); + assert.equal( + inspect([ready, { ...guardedSetup, env: { HOME: "/other" } }, install]) + .integration.disposition, + "needs-review", + ); + assert.equal( + inspect([ + ready, + guardedSetup, + { + uses: `workos/setup-socket-firewall/teardown@${APPROVED_RELEASE_SHA}`, + }, + install, + ]).integration.disposition, + "needs-sfw", + ); + } + const guardedSetup = { + ...setup, + if: "steps.ready.outputs.enabled == 'true'", + }; + const install = { run: "npm ci", if: guardedSetup.if }; + for (const steps of [ + [guardedSetup, ready, install], + [ready, guardedSetup, ready, install], + [ + ready, + guardedSetup, + { ...install, if: "steps.ready.outputs.enabled == 't r u e'" }, + ], + ]) + assert.equal(inspect(steps).integration.disposition, "needs-review"); + for (const guard of [ + "env.ENABLED == 'true'", + "always()", + "steps.ready.outputs.enabled == 't r u e'", + ]) { + assert.equal( + inspect([ready, { ...setup, if: guard }, { ...install, if: guard }]) + .integration.disposition, + "needs-review", + ); + } +}); + +test("a stricter stable conjunction implies setup, never the reverse", () => { + const ready = { id: "ready", run: 'echo enabled=true >> "$GITHUB_OUTPUT"' }; + const output = "steps.ready.outputs.enabled == 'true'"; + const event = "github.event_name == 'pull_request'"; + const check = (setupGuard, installGuard, steps = [ready]) => + inspect([ + ...steps, + { ...setup, if: setupGuard }, + { run: "npx tool", if: installGuard }, + ]); + for (const guard of [`${output} && ${event}`, `${event} && ${output}`]) { + const result = check(output, `\${{ ${guard} }}`); + assert.equal(result.integration.disposition, "integrated"); + assert.equal(result.status, "unknown"); + assert.equal(check(guard, output).integration.disposition, "needs-review"); + assert.equal(check(guard, guard).integration.disposition, "integrated"); + assert.equal( + check(output, guard, [ready, ready]).integration.disposition, + "needs-review", + ); + assert.equal( + check(output, guard, []).integration.disposition, + "needs-review", + ); + } + for (const guard of [ + `${output} || ${event}`, + `\${{ ${output} }} && ${event}`, + `${event} && \${{ ${output} }}`, + `${output} && always()`, + `${output} && env.ENABLED == 'true'`, + `${output} && (${event})`, + `${output} && ${event} && needs.other.outputs.ok == 'true'`, + `${output.replace("'true'", "'false'")} && ${event}`, + ]) + assert.equal( + check(output, guard).integration.disposition, + "needs-review", + guard, + ); + assert.equal( + check(`${output} && ${event}`, `${output} && github.event_name == 'push'`) + .integration.disposition, + "needs-review", + ); +}); + +test("literal npm workspaces preserve command classification and teardown gaps", () => { + const teardown = { + uses: `${setup.uses.split("@")[0]}/teardown@${APPROVED_RELEASE_SHA}`, + }; + for (const selector of [ + "-w apps/research", + "--workspace apps/research", + "--workspace=apps/research", + ]) { + const script = { run: `npm ${selector} run build` }; + assert.equal(inspect([script]).integration.disposition, "no-js-ci"); + assert.equal(inspect([script]).status, "unknown"); + for (const verb of ["ci", "exec -- wrangler --env production"]) { + const install = { run: `npm ${selector} ${verb}` }; + assert.equal( + inspect([setup, install]).integration.disposition, + "integrated", + ); + assert.equal(inspect([install]).integration.disposition, "needs-sfw"); + assert.equal( + inspect([setup, teardown, install]).integration.disposition, + "needs-sfw", + ); + } + } + for (const run of [ + "npm -w $APP run build", + "npm -w apps/${{ matrix.app }} run build", + "npm -w ../outside ci", + "npm -w --userconfig ci", + "npm -w apps/research --userconfig other ci", + "npm -w apps/research exec wrangler", + ]) + assert.notEqual( + inspect([setup, { run }]).integration.disposition, + "integrated", + run, + ); + assert.equal( + inspect([ + setup, + { run: "npm -w apps/research ci --registry=https://example.invalid" }, + ]).integration.disposition, + "needs-sfw", + ); +}); + +test("literal npx package selectors retain all download and configuration boundaries", () => { + for (const run of [ + "npx --yes --package renovate@43.257.6 renovate-config-validator --strict default.json", + "npx -p @scope/tool@1.2.3 tool --env $ENVIRONMENT", + "npx --package=tool --package=other tool --registry=https://payload.invalid", + "npm exec -- wrangler --env $ENVIRONMENT", + ]) { + const result = inspect([setup, { run }]); + assert.equal(result.integration.disposition, "integrated", run); + assert.equal(result.status, "unknown", run); + assert.equal(inspect([{ run }]).integration.disposition, "needs-sfw", run); + } + for (const run of [ + "npx --package=$PACKAGE tool", + "npx --package $ARTIFACTS/tool.tgz tool", + "npx --package=https://example.invalid/tool.tgz tool", + "npx --package tool", + "bunx --package tool tool", + "npm exec -- $TOOL", + "npm exec -- https://example.invalid/tool.tgz", + ]) + assert.equal( + inspect([setup, { run }]).integration.disposition, + "needs-review", + run, + ); + assert.equal( + inspect([setup, { run: "npx --package --userconfig tool" }]).integration + .disposition, + "needs-sfw", + ); + assert.equal( + inspect([ + setup, + { run: "npx --package tool --registry=https://example.invalid tool" }, + ]).integration.disposition, + "needs-sfw", + ); + assert.equal( + inspect([ + setup, + { run: "npm exec --registry=https://example.invalid -- tool" }, + ]).integration.disposition, + "needs-sfw", + ); +}); + +test("literal install options preserve configuration, not execution assurance", () => { + for (const run of [ + "npm ci --include=optional", + "npm install --omit=dev --package-lock=false", + "npm ci --prefix web", + "npm ci --prefix=packages/sdk", + 'bun install --frozen-lockfile --os="*" --cpu="*"', + ]) { + assert.equal( + inspect([setup, { run }]).integration.disposition, + "integrated", + run, + ); + assert.equal(inspect([{ run }]).integration.disposition, "needs-sfw", run); + assert.equal( + inspect([setup, { run: `${run} --registry=https://example.invalid` }]) + .integration.disposition, + "needs-sfw", + run, + ); + } + for (const run of [ + "npm ci --prefix $PROJECT", + "npm ci --prefix ../other", + "npm ci --prefix /tmp/project", + "npm ci --prefix --registry=https://example.invalid", + "npm ci --future-option web", + "npm --prefix help ci", + ]) { + assert.notEqual( + inspect([setup, { run }]).integration.disposition, + "integrated", + run, + ); + } +}); + +test("literal default working directories match explicit run-step directories", () => { + for (const directory of ["web", "./packages/chat", "."]) { + const defaults = { run: { shell: "bash", "working-directory": directory } }; + const result = inspect([setup, { run: "npm ci" }], { defaults }); + assert.equal(result.integration.disposition, "integrated"); + assert.equal(result.status, inspect([setup, { run: "npm ci" }]).status); + assert.equal( + inspect([{ run: "npm ci" }], { defaults }).integration.disposition, + "needs-sfw", + ); + const workflow = classifyWorkflow( + JSON.stringify({ + on: "push", + defaults, + jobs: { build: { steps: [setup, { run: "npm ci" }] } }, + }), + context, + ); + assert.equal(workflow.jobs[0].integration.disposition, "integrated"); + } + for (const directory of [ + "${{ inputs.path }}", + "$HOME", + "../other", + "/tmp", + null, + ]) { + assert.equal( + inspect([setup, { run: "npm ci" }], { + defaults: { run: { "working-directory": directory } }, + }).integration.disposition, + "needs-review", + ); + } + assert.equal( + inspect([setup, { run: "npm ci" }], { + defaults: { run: { "working-directory": "web", shell: "python" } }, + }).integration.disposition, + "needs-review", + ); +}); + +test("setup-owned temporary npm config does not invalidate HOME configuration", () => { + const env = { NPM_CONFIG_USERCONFIG: "${{ runner.temp }}/sfw.npmrc" }; + assert.equal( + inspect([{ ...setup, env }, { run: "npm ci" }]).integration.disposition, + "integrated", + ); + for (const steps of [ + [setup, { run: "npm ci", env }], + [{ ...setup, env: { ...env, HOME: "/other" } }, { run: "npm ci" }], + [ + { ...setup, env: { NPM_CONFIG_USERCONFIG: "${{ inputs.config }}" } }, + { run: "npm ci" }, + ], + ]) + assert.equal(inspect(steps).integration.disposition, "needs-review"); +}); + +test("public dynamic fallback stays bounded by the pinned action's fork checks", () => { + const publicSetup = { + ...setup, + with: { + ...setup.with, + token: "${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }}", + "allow-external-fork-fallback": + "${{ github.event_name == 'pull_request' }}", + }, + }; + const steps = [publicSetup, { run: "npm ci" }]; + const result = classifyJob( + "build", + { steps }, + { ...context, visibility: "public" }, + ["pull_request"], + ); + assert.equal(result.integration.disposition, "integrated"); + assert.equal(result.integration.downloads[0].status, "fork-exception"); + assert.equal(result.status, "unknown"); + assert.equal(inspect(steps).integration.disposition, "needs-review"); + assert.equal( + classifyJob( + "build", + { steps: [{ ...publicSetup, "continue-on-error": true }, steps[1]] }, + { ...context, visibility: "public" }, + ).integration.disposition, + "needs-review", + ); +}); + +test("commented-out workflows contain no active jobs; invalid YAML remains reviewable", () => { + const result = classifyWorkflow( + "# name: Disabled\n# on: push\n# jobs:\n# build: npm ci\n", + context, + ); + assert.equal(result.parseError, undefined); + assert.deepEqual(result.jobs, []); + for (const source of ["", "null", "jobs: {}", "# disabled\non: ["]) { + assert.ok(classifyWorkflow(source, context).parseError, source); + } +}); + +test("audit acquires root actions at the captured SHA, including nested composites", async () => { + const sha = "a".repeat(40); + for (const filename of ["action.yml", "action.yaml"]) { + const sources = new Map([ + [ + context.path, + stringify({ + on: "push", + jobs: { build: { steps: [{ uses: "./" }] } }, + }), + ], + [ + filename, + stringify({ + runs: { using: "composite", steps: [setup, { uses: "./nested" }] }, + }), + ], + [ + "nested/action.yml", + JSON.stringify({ + runs: { + using: "composite", + steps: [{ run: "npm ci", shell: "bash" }], + }, + }), + ], + ]); + const reads = []; + const result = await auditRepository( + { + async getRef() { + return { object: { sha } }; + }, + async getTree() { + return { + truncated: false, + tree: [...sources.keys()].map((path) => ({ + path, + type: "blob", + mode: "100644", + })), + }; + }, + async getText(repo, path, ref) { + assert.equal(ref, sha); + reads.push(path); + return sources.get(path); + }, + }, + { name: "fixture", defaultBranch: "main", visibility: "private" }, + ); + assert.deepEqual(reads.sort(), [...sources.keys()].sort()); + assert.equal(result.disposition, "integrated"); + } + const cyclic = new Map([ + [ + "action.yml", + JSON.stringify({ runs: { using: "composite", steps: [{ uses: "./" }] } }), + ], + ]); + assert.equal( + classifyJob( + "cycle", + { steps: [{ uses: "./" }] }, + { ...context, localActions: cyclic }, + ).integration.disposition, + "needs-review", + ); + assert.equal( + inspect([{ uses: "./" }]).integration.disposition, + "needs-review", + ); +}); diff --git a/tools/rollout/audit.mjs b/tools/rollout/audit.mjs new file mode 100644 index 0000000..33c8780 --- /dev/null +++ b/tools/rollout/audit.mjs @@ -0,0 +1,448 @@ +import { mkdtemp, rename, rm, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; + +import { + classifyWorkflow, + PACKAGE_MANAGER_READ, + repositoryDisposition, + resolveLocalActionSource, +} from "./classify.mjs"; +import { parseYamlSource } from "./yaml.mjs"; +import { captureRepositoryInventory } from "./inventory.mjs"; +import { APPROVED_RUNTIME_BLOBS, ORGANIZATION } from "./constants.mjs"; +import { readRegistryExclusions } from "./exclusions.mjs"; +import { fingerprint, fingerprintYaml } from "./fingerprint.mjs"; +import { + integrationDisposition, + resolveLocalWorkflowCalls, + resolveNoInstallWorkflowCalls, +} from "./integration.mjs"; + +const WORKFLOW_PATH_PATTERN = /^\.(?:github|depot)\/workflows\/[^/]+\.ya?ml$/; +const LOCKFILE_PATTERN = + /(?:^|\/)(?:package-lock\.json|npm-shrinkwrap\.json|pnpm-lock\.yaml|yarn\.lock|bun\.lockb?)$/; +const LOCAL_ACTION_PATTERN = /uses:\s*['"]?\.\/([^\s'"#]*)/g; +const AUDIT_CONCURRENCY = 5; +const MAX_WORKFLOWS_PER_REPOSITORY = 200; +const MAX_LOCAL_ACTIONS_PER_REPOSITORY = 200; + +function sortedByName(rows) { + return [...rows].sort((left, right) => + left.name < right.name ? -1 : left.name > right.name ? 1 : 0, + ); +} + +async function mapWithConcurrency(items, limit, mapper) { + const results = new Array(items.length); + let nextIndex = 0; + + async function worker() { + while (nextIndex < items.length) { + const index = nextIndex; + nextIndex += 1; + results[index] = await mapper(items[index], index); + } + } + + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, () => worker()), + ); + return results; +} + +async function isEmptyRepository(client, repository) { + try { + const metadata = await client.api( + `repos/${repository}`, + `read ${repository} metadata`, + ); + return metadata.size === 0; + } catch { + return false; + } +} + +export async function auditRepository(client, repository) { + const fullName = `${ORGANIZATION}/${repository.name}`; + + let headSha; + let tree; + try { + const ref = await client.getRef( + fullName, + `heads/${encodeURIComponent(repository.defaultBranch)}`, + ); + headSha = ref?.object?.sha; + if (typeof headSha !== "string" || !/^[0-9a-f]{40}$/.test(headSha)) { + throw new Error(`unexpected head ref shape for ${fullName}`); + } + tree = await client.getTree(fullName, headSha, true); + } catch (error) { + const status = error.status ?? error.cause?.status; + if ( + headSha === undefined && + (status === 404 || + status === 409 || + /HTTP (404|409)/.test(error.message)) && + (await isEmptyRepository(client, fullName)) + ) { + return { + defaultBranch: repository.defaultBranch, + disposition: "empty", + jobs: { total: 0 }, + lockfiles: [], + name: repository.name, + visibility: repository.visibility, + workflows: [], + }; + } + return { + defaultBranch: repository.defaultBranch, + disposition: "audit-error", + error: error.message, + lockfiles: [], + name: repository.name, + visibility: repository.visibility, + workflows: [], + }; + } + + if ( + tree?.truncated !== false || + !Array.isArray(tree.tree) || + tree.tree.some( + (entry) => + !entry || + typeof entry.path !== "string" || + !entry.path || + !["blob", "tree", "commit"].includes(entry.type) || + (entry.type === "blob" && + !["100644", "100755", "120000"].includes(entry.mode)) || + (entry.type === "tree" && entry.mode !== "040000") || + (entry.type === "commit" && entry.mode !== "160000"), + ) || + new Set(tree.tree.map((entry) => entry.path)).size !== tree.tree.length + ) { + return { + defaultBranch: repository.defaultBranch, + disposition: "audit-error", + error: + "default-branch tree listing is malformed, truncated, or contains unsupported file modes", + headSha, + lockfiles: [], + name: repository.name, + visibility: repository.visibility, + workflows: [], + }; + } + + const blobPaths = new Set( + (tree.tree ?? []) + .filter((entry) => entry.type === "blob") + .map((entry) => entry.path), + ); + const workflowPaths = [...blobPaths] + .filter((path) => WORKFLOW_PATH_PATTERN.test(path)) + .sort(); + const lockfiles = [...blobPaths] + .filter((path) => LOCKFILE_PATTERN.test(path)) + .sort() + .slice(0, 50); + + if (workflowPaths.length > MAX_WORKFLOWS_PER_REPOSITORY) { + return { + defaultBranch: repository.defaultBranch, + disposition: "audit-error", + error: `repository has ${workflowPaths.length} workflow files; refusing unbounded scan`, + headSha, + lockfiles, + name: repository.name, + visibility: repository.visibility, + workflows: [], + }; + } + + try { + const sourceDigests = new Map(); + const readSource = async (path) => { + if (tree.tree.find((entry) => entry.path === path)?.mode === "120000") { + throw new Error("workflow or local action source is a symlink"); + } + const text = await client.getText(fullName, path, headSha); + if (typeof text === "string") sourceDigests.set(path, fingerprint(text)); + return text; + }; + const workflowTexts = await mapWithConcurrency( + workflowPaths, + AUDIT_CONCURRENCY, + (path) => readSource(path), + ); + + if ( + workflowTexts.some((text) => typeof text !== "string" || !text.trim()) + ) { + throw new Error("workflow source read is empty or malformed"); + } + // Fetch only referenced local actions, including nested composites. The set + // bounds cycles in fetching; the classifier separately bounds expansion. + const localActions = new Map(); + const sourceContext = { + repository: fullName, + defaultBranch: repository.defaultBranch, + headSha, + localSfwRelease: Object.entries(APPROVED_RUNTIME_BLOBS).every( + ([path, sha]) => + tree.tree.some( + (entry) => + entry.path === path && + entry.sha === sha && + entry.type === "blob" && + ["100644", "100755"].includes(entry.mode), + ), + ), + }; + const mounts = new Set([""]); + for (const text of workflowTexts) { + let parsed; + try { + parsed = parseYamlSource(text); + } catch { + continue; + } // Classification reports malformed YAML. + for (const job of Object.values(parsed?.jobs ?? {})) { + for (const step of Array.isArray(job?.steps) ? job.steps : []) { + if ( + !step?.uses?.startsWith?.("actions/checkout@") || + typeof step.with?.path !== "string" + ) + continue; + const prefix = step.with.path + .replace(/^\.\//, "") + .replace(/\/+$/, ""); + const resolved = resolveLocalActionSource( + `${prefix}/action-placeholder`, + { ...sourceContext, checkouts: [step] }, + ); + if (resolved.path === "action-placeholder") mounts.add(`${prefix}/`); + } + } + } + if (mounts.size > MAX_LOCAL_ACTIONS_PER_REPOSITORY) + throw new Error( + "too many checkout mounts; refusing unbounded source lookup", + ); + const pendingTexts = [...workflowTexts]; + const localActionPaths = new Set(); + while (pendingTexts.length > 0) { + const paths = []; + for (const text of pendingTexts.splice(0)) { + for (const match of text.matchAll(LOCAL_ACTION_PATTERN)) { + const base = match[1].replace(/\/+$/, ""); + const prefix = base ? `${base}/` : ""; + const candidates = [...mounts] + .filter((mount) => prefix.startsWith(mount)) + .flatMap((mount) => [ + `${prefix.slice(mount.length)}action.yml`, + `${prefix.slice(mount.length)}action.yaml`, + ]); + for (const candidate of candidates) { + if (blobPaths.has(candidate) && !localActionPaths.has(candidate)) { + localActionPaths.add(candidate); + paths.push(candidate); + } + } + } + } + if (localActionPaths.size > MAX_LOCAL_ACTIONS_PER_REPOSITORY) { + throw new Error( + "too many referenced local actions; refusing unbounded scan", + ); + } + await mapWithConcurrency( + paths.sort(), + AUDIT_CONCURRENCY, + async (path) => { + const text = await readSource(path); + if (typeof text !== "string" || !text.trim()) + throw new Error("local action source read is empty or malformed"); + localActions.set(path, text); + pendingTexts.push(text); + }, + ); + } + + // Read the value at the captured SHA only when this exact idiom occurs. + // A project npmrc or an absent manifest leaves the dynamic install unresolved. + const packageManager = + blobPaths.has("package.json") && + !blobPaths.has(".npmrc") && + workflowTexts.some((text) => text.includes(PACKAGE_MANAGER_READ)) + ? JSON.parse(await readSource("package.json"))?.packageManager + : undefined; + const exclusions = ( + await readRegistryExclusions(repository.name, readSource) + ).map((entry) => ({ + ...entry, + reviewFingerprint: fingerprint({ + rule: entry, + inputs: Object.fromEntries( + [entry.manifest ?? "package.json", entry.lockfile, ".npmrc"].map( + (path) => [path, sourceDigests.get(path)], + ), + ), + }), + })); + // Configuration changes matter, but validated registry-only override + // version bumps in the approved Rush source do not change SFW routing. + sourceContext.reviewConfiguration = fingerprint( + tree.tree + .filter((entry) => + /(?:^|\/)(?:\.npmrc|\.?bunfig\.toml|\.yarnrc(?:\.yml)?|pnpm-workspace\.yaml|pnpm-config\.json)$/.test( + entry.path, + ), + ) + .map(({ path, mode, type, sha }) => ({ + path, + mode, + type, + sha: + exclusions.find((entry) => entry.manifest === path) + ?.configurationFingerprint ?? sha, + })) + .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)), + ); + const workflows = resolveLocalWorkflowCalls( + workflowPaths.map((path, index) => ({ + ...classifyWorkflow(workflowTexts[index], { + ...sourceContext, + localActions, + packageManager, + path, + registryExclusion: exclusions.find( + (entry) => + entry.status === "matched" && entry.workflows.includes(path), + ), + visibility: repository.visibility, + }), + reviewFingerprint: fingerprintYaml(workflowTexts[index]), + })), + ); + + const managers = [ + ...new Set( + workflows.flatMap((workflow) => + workflow.jobs.flatMap((job) => job.managers), + ), + ), + ].sort(); + const statusCounts = {}; + for (const workflow of workflows) { + for (const job of workflow.jobs) { + statusCounts[job.status] = (statusCounts[job.status] ?? 0) + 1; + } + } + + return { + defaultBranch: repository.defaultBranch, + disposition: integrationDisposition(workflows, exclusions), + assuranceDisposition: repositoryDisposition(workflows), + exclusions, + headSha, + jobs: { + byStatus: Object.fromEntries( + Object.entries(statusCounts).sort(([a], [b]) => (a < b ? -1 : 1)), + ), + total: Object.values(statusCounts).reduce( + (total, count) => total + count, + 0, + ), + }, + lockfiles, + managers, + name: repository.name, + visibility: repository.visibility, + workflows, + }; + } catch (error) { + return { + defaultBranch: repository.defaultBranch, + disposition: "audit-error", + error: error.message, + headSha, + lockfiles, + name: repository.name, + visibility: repository.visibility, + workflows: [], + }; + } +} + +export async function runAudit(client, options = {}) { + const progress = options.progress ?? (() => {}); + const inventory = await captureRepositoryInventory(client, ORGANIZATION); + + let completed = 0; + const repositories = await mapWithConcurrency( + inventory.repositories, + AUDIT_CONCURRENCY, + async (repository) => { + const row = await auditRepository(client, repository); + completed += 1; + progress(completed, inventory.repositories.length, repository.name); + return { ...row, repositoryId: repository.repositoryId }; + }, + ); + + const rows = sortedByName( + resolveNoInstallWorkflowCalls(repositories, ORGANIZATION), + ); + const dispositions = {}; + const assuranceDispositions = {}; + for (const row of rows) { + dispositions[row.disposition] = (dispositions[row.disposition] ?? 0) + 1; + const assurance = row.assuranceDisposition ?? row.disposition; + assuranceDispositions[assurance] = + (assuranceDispositions[assurance] ?? 0) + 1; + } + + return { + dispositions: Object.fromEntries( + Object.entries(dispositions).sort(([a], [b]) => (a < b ? -1 : 1)), + ), + assuranceDispositions: Object.fromEntries( + Object.entries(assuranceDispositions).sort(([a], [b]) => + a < b ? -1 : 1, + ), + ), + runtimeVerification: "not-performed", + generatedAt: new Date().toISOString(), + inventory: { + activeCount: inventory.activeCount, + archivedCount: inventory.archivedCount, + differences: inventory.differences, + totalCount: inventory.totalCount, + visibility: inventory.visibility, + }, + organization: ORGANIZATION, + scanErrors: dispositions["audit-error"] ?? 0, + scanStatus: dispositions["audit-error"] ? "partial" : "complete", + coverage: + "token-visible repositories only; organization-wide access is an operator prerequisite", + repositories: rows, + schemaVersion: 3, + }; +} + +export async function writeReportAtomically(reportPath, report) { + const directory = dirname(reportPath); + const temporaryDirectory = await mkdtemp(join(directory, ".audit-")); + const temporaryPath = join(temporaryDirectory, "report.json"); + try { + await writeFile(temporaryPath, `${JSON.stringify(report, null, 2)}\n`, { + mode: 0o600, + }); + await rename(temporaryPath, reportPath); + } finally { + await rm(temporaryDirectory, { force: true, recursive: true }); + } +} diff --git a/tools/rollout/classify.mjs b/tools/rollout/classify.mjs new file mode 100644 index 0000000..5ede3b4 --- /dev/null +++ b/tools/rollout/classify.mjs @@ -0,0 +1,1842 @@ +import { parseYamlSource as parse } from "./yaml.mjs"; +import { shellCommands, shellTokens } from "./commands.mjs"; +import { fingerprint } from "./fingerprint.mjs"; + +import { ACTION_REPOSITORY, APPROVED_RELEASE_SHA } from "./constants.mjs"; +import { + certainTeardown, + defaultsUncertain, + environmentUncertain, + literalWorkingDirectory, + observedIntegration, + unresolvedJsInvocation, + SUPPORTED_SHELLS, +} from "./integration.mjs"; + +export const PACKAGE_MANAGER_READ = 'require("./package.json").packageManager'; +const PNPM_BOOTSTRAP = `set -euo pipefail\npnpm_package="$(node --print '${PACKAGE_MANAGER_READ}')"\nnpm install --global "$pnpm_package" --ignore-scripts --no-audit --no-fund`; + +const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/; +const LOCAL_TARBALL = /^\$(?:RUNNER_TEMP|\{RUNNER_TEMP\})\/[\w.*-]+\.tgz$/; +const UNSAFE_PUBLIC_TRIGGERS = new Set([ + "issue_comment", + "pull_request_target", + "repository_dispatch", + "workflow_run", +]); +const CONTRIBUTOR_HEAD_PATTERN = + /github\.event\.pull_request\.head|github\.head_ref|\bhead\.sha\b|\bhead\.ref\b/; + +const NPM_DOWNLOAD = new Set([ + "add", + "audit", + "ci", + "clean-install", + "dedupe", + "exec", + "i", + "install", + "install-ci-test", + "install-test", + "update", + "upgrade", + "x", +]); +// Only literal informational commands are considered transparent. Scripts, +// lifecycle commands and executors can hide dependency installation. +const NPM_NO_NETWORK = new Set(["get", "help", "ls", "whoami"]); +const PNPM_DOWNLOAD = new Set([ + "add", + "dlx", + "fetch", + "i", + "import", + "install", + "install-test", + "up", + "update", +]); +const PNPM_NO_NETWORK = new Set(["list", "ls", "why"]); +const BUN_DOWNLOAD = new Set(["add", "i", "install", "update", "x"]); +const BUN_NO_NETWORK = new Set(); +const YARN_DOWNLOAD = new Set(["add", "dlx", "i", "import", "install", "up"]); +const OTHER_ECOSYSTEM_COMMANDS = new Set([ + "apk", + "apt", + "apt-get", + "brew", + "bundle", + "cargo", + "composer", + "conda", + "dotnet", + "gem", + "go", + "gradle", + "helm", + "mvn", + "nuget", + "pip", + "pip3", + "pipenv", + "poetry", + "terraform", + "uv", +]); +const WRAPPER_COMMANDS = new Set(["just", "make", "mise", "rake", "task"]); +const ORCHESTRATORS = new Set(["lerna", "nx", "rush", "turbo"]); +const SIMPLE_COMMANDS = new Set(["echo", "printf", "pwd", "true", "false"]); +const TRANSPARENT_INSTALL_FLAGS = new Set([ + "--ignore-scripts", + "--no-audit", + "--no-fund", + "--frozen-lockfile", + "--immutable", + "--offline", + "--prefer-offline", + "--no-progress", + "--no-save", + "--package-lock=false", + "--omit=dev", + "--include=optional", + "--os=*", + "--cpu=*", + "--save-exact", + "--save-dev", + "--global", + "-g", + "-D", + "-y", +]); + +function isMapping(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function condition(value) { + if (value === undefined) return "always"; + const expression = normalizeExpression(value).replace( + /^\$\{\{(.*)\}\}$/, + "$1", + ); + if (expression === "false") return "never"; + if (expression === "true") return "always"; + return "uncertain"; +} + +function boundaryUncertain(step) { + return ( + condition(step.if) === "uncertain" || + (step["continue-on-error"] !== undefined && + condition(step["continue-on-error"]) !== "never") || + step.env !== undefined + ); +} + +function normalizeExpression(value) { + return String(value ?? "").replaceAll(/\s+/g, ""); +} + +// An ordinary run-step guard still has GitHub's implicit success() gating. +// It can skip an install/script, but cannot run it after failed setup. Keep +// explicit status predicates conservative and never apply this to uses/composites. +function primaryRunGuard(step) { + if ( + typeof step.run === "string" && + typeof step.if === "string" && + !/\b(?:always|cancelled|failure|success)\s*\(/i.test(step.if) + ) + return undefined; + return step.if; +} + +function expectedTokenExpression(visibility) { + const secret = + visibility === "public" + ? "PUBLIC_SOCKET_FIREWALL_TOKEN" + : "SOCKET_FIREWALL_TOKEN"; + return `\${{secrets.${secret}}}`; +} + +function commandParts(command, context = {}) { + const tokens = shellTokens(command); + const words = tokens.words; + const environment = {}; + let start = 0; + const assignments = (envArguments = false) => { + const values = envArguments ? words : tokens.commands; + while (/^[A-Za-z_][A-Za-z0-9_]*=/.test(values[start] ?? "")) { + // Keep keys for scope checks, never copy values to new diagnostics. + Object.defineProperty(environment, values[start].split("=", 1)[0], { + value: true, + enumerable: true, + configurable: true, + }); + start += 1; + } + }; + assignments(); + let preservedEnvironment = false; + if (words[start] === "env" && words[start + 1] === "-i") { + // This clean-room npm invocation retains the exact action-configured paths + // and registry. Every other env -i shape remains unresolved. + const expected = { + HOME: '"$HOME"', + PATH: '"$PATH"', + NPM_CONFIG_REGISTRY: '"${NPM_CONFIG_REGISTRY:?}"', + NPM_CONFIG_USERCONFIG: '"${NPM_CONFIG_USERCONFIG:-$HOME/.npmrc}"', + }; + const values = new Map(); + let end = start + 2; + let duplicate = false; + while (/^[A-Za-z_][A-Za-z0-9_]*=/.test(words[end] ?? "")) { + const raw = tokens.commands[end]; + const key = raw.slice(0, raw.indexOf("=")); + duplicate ||= values.has(key); + values.set(key, raw.slice(key.length + 1)); + end += 1; + } + if ( + !duplicate && + words[end] === "npm" && + Object.entries(expected).every( + ([key, value]) => values.get(key) === value, + ) && + [...values].every( + ([key, value]) => + Object.hasOwn(expected, key) || (key === "CI" && value === "true"), + ) + ) { + start = end; + preservedEnvironment = true; + } + } + if (words[start] === "env" && !words[start + 1]?.startsWith("-")) { + start += 1; + assignments(true); + } + const args = words.slice(start); + const rawArgs = tokens.commands.slice(start); + // A literal workspace selector changes the package, not the npm command. + // Leave dynamic/escaping selectors and other leading options unresolved. + while (args[0] === "npm") { + const separate = ["-w", "--workspace"].includes(args[1]); + const value = separate + ? args[2] + : args[1]?.startsWith("--workspace=") + ? args[1].slice(12) + : undefined; + const bounded = value?.replace( + /\$\{\{\s*matrix\.([\w-]+)\s*\}\}/g, + (match, key) => + context.literalMatrixKeys?.has(key) ? "matrix-value" : match, + ); + if (!literalWorkingDirectory(bounded) || bounded.startsWith("-")) break; + const count = separate ? 2 : 1; + args.splice(1, count); + rawArgs.splice(1, count); + } + return { words: args, rawWords: rawArgs, environment, preservedEnvironment }; +} + +function commandWords(command, context) { + return commandParts(command, context).words; +} + +// Literal logging only: quoted source, substitution and shell controls are not +// interpreted. These lines are transparent only to observed configuration. +function literalLogging(command) { + // GitHub interpolates expressions before the shell sees even single quotes. + if (command.includes("${{")) return false; + return /^echo(?:\s+(?:"[^"$`\\\\]*"|'[^']*'|[A-Za-z0-9_.,:/@+=-]+))*\s*$/.test( + command, + ); +} + +const REGISTRY_FLAG = + /^--(?:config\.)?(?:(?:@[^:\s=]+:)?(?:reg|regi|regis|regist|registr|registry)|userconfig|globalconfig)(?:=|$)/i; +const REGISTRY_KEY = /^(?:(?:@[^:]+:)?registry|userconfig|globalconfig)$/i; +function registrySetter(words) { + return ( + ["npm", "pnpm"].includes(words[0]) && + words[1] === "config" && + ["set", "delete", "unset"].includes(words[2]) && + REGISTRY_KEY.test((words[3] ?? "").split("=", 1)[0]) + ); +} +function writeTargets(tokens, includeRemovals = true) { + const target = (index) => ({ + value: tokens.words[index] ?? "", + raw: tokens.commands[index] ?? "", + }); + const targets = tokens.commands.flatMap((word, index) => + /^>+$/.test(word) ? [target(index + 1)] : [], + ); + const program = tokens.words[0]; + if ( + program === "tee" || + (includeRemovals && ["rm", "unlink", "mv", "truncate"].includes(program)) || + (["sed", "perl"].includes(program) && + tokens.words.some((word) => /^-[^-]*i|^--in-place(?:=|$)/.test(word))) + ) + return [ + ...targets, + ...tokens.words.slice(1).map((_, index) => target(index + 1)), + ]; + if (["cp", "install", "mv"].includes(program) && tokens.words.length > 1) + return [...targets, target(tokens.words.length - 1)]; + return targets; +} +function expandedTarget(target) { + return target.raw.startsWith('"') && target.raw.endsWith('"') + ? target.raw.slice(1, -1) + : target.raw; +} +function configTarget(target) { + if ( + /(?:^|\/)\.npmrc$/.test(target.value) || + /^\$(?:NPM_CONFIG_USERCONFIG|\{NPM_CONFIG_USERCONFIG(?::\?[^}]*)?\})$/.test( + expandedTarget(target), + ) + ) + return "npm"; + if ( + /(?:^|\/)bunfig\.toml$/.test(target.value) || + /^\$(?:SFW_BUN_CONFIG_PATH|\{SFW_BUN_CONFIG_PATH(?::\?[^}]*)?\})$/.test( + expandedTarget(target), + ) + ) + return "bun"; + return undefined; +} + +function assignmentKeys(words) { + return Object.fromEntries( + words.flatMap((word) => { + const match = word.match(/^([A-Za-z_][A-Za-z0-9_]*)(?:=|<<)/); + return match ? [[match[1], true]] : []; + }), + ); +} +function changesStepEnvironment(command) { + const { words, environment } = commandParts(command); + if (!words.length) return environmentUncertain(environment); + if (["export", "declare", "typeset", "unset"].includes(words[0])) { + const keys = + words[0] === "unset" + ? Object.fromEntries(words.slice(1).map((key) => [key, true])) + : assignmentKeys(words.slice(1)); + return environmentUncertain(keys); + } + return false; +} +function writesEnvironment(tokens) { + const targets = writeTargets(tokens, false).map(expandedTarget); + const keys = assignmentKeys(tokens.words); + return ( + targets.some((path) => + /^\$(?:GITHUB_PATH|\{GITHUB_PATH(?::\?[^}]*)?\})$/.test(path), + ) || + (targets.some((path) => + /^\$(?:GITHUB_ENV|\{GITHUB_ENV(?::\?[^}]*)?\})$/.test(path), + ) && + (!Object.keys(keys).length || environmentUncertain(keys))) + ); +} + +function approvedBunArguments(words, rawWords) { + const indexes = new Set(); + if (words[0] !== "bun") return indexes; + const approvedPath = (value) => + /^\$(?:SFW_BUN_CONFIG_PATH|\{SFW_BUN_CONFIG_PATH(?::\?)?\})$/.test( + value?.startsWith('"') && value.endsWith('"') + ? value.slice(1, -1) + : (value ?? ""), + ); + rawWords.forEach((word, index) => { + if (word.startsWith("--config=") && approvedPath(word.slice(9))) + indexes.add(index); + if (word === "--config" && approvedPath(rawWords[index + 1])) { + indexes.add(index); + indexes.add(index + 1); + } + }); + return indexes; +} + +function directExecutor(words) { + const packageName = /^(?:@[\w.-]+\/)?[A-Za-z0-9_][\w.-]*(?:@[\w.^~*+-]+)?$/; + if (words[0] === "npm") { + // npm parses options after the command too, unless an explicit -- ends them. + return ["exec", "x"].includes(words[1]) && + words[2] === "--" && + (packageName.test(words[3] ?? "") || LOCAL_TARBALL.test(words[3] ?? "")) + ? words.slice(0, 3).join(" ") + : undefined; + } + if (!["npx", "bunx"].includes(words[0])) return undefined; + let target = 1; + while (target < words.length) { + if (["-y", "--yes"].includes(words[target])) target += 1; + else if ( + words[0] === "npx" && + ["-p", "--package"].includes(words[target]) && + (packageName.test(words[target + 1] ?? "") || + LOCAL_TARBALL.test(words[target + 1] ?? "")) + ) + target += 2; + else if ( + words[0] === "npx" && + words[target].startsWith("--package=") && + (packageName.test(words[target].slice(10)) || + LOCAL_TARBALL.test(words[target].slice(10))) + ) + target += 1; + else break; + } + // Installer options end at the literal package/binary target. Arguments to + // that program are payload, not npx/bunx registry configuration. + if ( + !/^[A-Za-z0-9_@][A-Za-z0-9_@./:+-]*$/.test(words[target] ?? "") && + !LOCAL_TARBALL.test(words[target] ?? "") + ) + return undefined; + return words.slice(0, target).join(" "); +} + +function installerArguments(words) { + const prefix = directExecutor(words); + const args = (prefix === undefined ? words : shellTokens(prefix).words).slice( + 1, + ); + const end = args.indexOf("--"); + return end === -1 ? args : args.slice(0, end); +} + +function firstSubcommand(words) { + return words.slice(1).find((word) => !word.startsWith("-")); +} + +export function classifyCommand(command, context) { + const words = commandWords(command, context); + const program = words[0]; + if (!program) { + return { command, kind: "no-network" }; + } + if (program.includes("/")) + return { command, program, kind: "unknown-wrapper" }; + const name = program; + if ( + ["npm", "pnpm", "bun", "yarn", "npx", "bunx", "corepack"].includes(name) && + words.length === 2 && + ["--version", "-v", "--help", "-h"].includes(words[1]) + ) + return { command, kind: "no-network" }; + + if ( + ["npm", "pnpm"].includes(name) && + words[1] === "config" && + ["get", "list", "ls"].includes(words[2]) + ) + return { command, kind: "no-network" }; + + // Do not mistake an option's value (e.g. --prefix help) for the verb. + // Unsupported leading options remain review candidates, not a parsed shell. + if ( + ["npm", "pnpm", "bun", "yarn", "corepack"].includes(name) && + words[1]?.startsWith("-") + ) { + return { command, kind: "unknown-wrapper", manager: name }; + } + + if (name === "npx" || name === "bunx") { + return { + command, + kind: "js-public-download", + manager: name === "npx" ? "npm" : "bun", + }; + } + if (name === "corepack") { + const subcommand = firstSubcommand(words); + if (subcommand === "enable" || subcommand === "disable") { + // Targeted controls may affect Yarn without changing pnpm's shim. + if (words.length !== 2) + return { + command, + program, + corepackControl: subcommand, + kind: "unknown-wrapper", + }; + return { command, corepack: subcommand, kind: "no-network" }; + } + return { + command, + corepack: subcommand ?? "implicit", + kind: "js-public-download", + manager: "npm", + }; + } + if (name === "npm" || name === "pnpm" || name === "bun" || name === "yarn") { + const subcommand = firstSubcommand(words); + if (name === "yarn") { + if ( + subcommand === "publish" || + (subcommand === "npm" && words.includes("publish")) + ) { + return { command, kind: "js-publish", manager: "yarn" }; + } + if (subcommand === undefined || YARN_DOWNLOAD.has(subcommand)) { + return { command, kind: "yarn-blocked", manager: "yarn" }; + } + return { command, kind: "unknown-wrapper", manager: "yarn" }; + } + if (subcommand === "publish") { + return { command, kind: "js-publish", manager: name }; + } + const downloads = + name === "npm" + ? NPM_DOWNLOAD + : name === "pnpm" + ? PNPM_DOWNLOAD + : BUN_DOWNLOAD; + const quiet = + name === "npm" + ? NPM_NO_NETWORK + : name === "pnpm" + ? PNPM_NO_NETWORK + : BUN_NO_NETWORK; + if (subcommand !== undefined && downloads.has(subcommand)) { + if (name === "npm" && subcommand === "audit" && !words.includes("fix")) { + return { command, kind: "no-network" }; + } + return { command, kind: "js-public-download", manager: name }; + } + if (quiet.has(subcommand)) { + return { command, kind: "no-network" }; + } + return { command, kind: "unknown-wrapper", manager: name }; + } + if (OTHER_ECOSYSTEM_COMMANDS.has(name)) { + // Language/package executors can wrap JS installers just like shell scripts. + const executesProgram = words + .slice(1) + .some((word) => ["run", "exec", "generate", "shell"].includes(word)); + return { + command, + kind: executesProgram ? "unknown-wrapper" : "other-ecosystem", + installationCapable: executesProgram, + }; + } + if (ORCHESTRATORS.has(name)) { + if (name === "lerna" && words.includes("bootstrap")) { + return { command, kind: "js-public-download", manager: "npm" }; + } + return { command, program, kind: "unknown-wrapper" }; + } + if ( + WRAPPER_COMMANDS.has(name) || + /\.sh$/.test(name) || + ((name === "bash" || name === "sh" || name === "zsh") && + words.slice(1).some((word) => /\.sh$/.test(word))) + ) { + return { command, program, kind: "unknown-wrapper" }; + } + return { + command, + program, + kind: SIMPLE_COMMANDS.has(name) ? "no-network" : "unknown-wrapper", + }; +} + +function parseUses(uses) { + const value = String(uses); + if (value.startsWith("./")) { + return { + kind: "local", + path: value.replace(/^\.\//, "").replace(/\/+$/, ""), + }; + } + if (value.startsWith("docker://")) { + return { image: value, kind: "docker" }; + } + const atIndex = value.lastIndexOf("@"); + const ref = atIndex === -1 ? "" : value.slice(atIndex + 1); + const target = atIndex === -1 ? value : value.slice(0, atIndex); + const segments = target.split("/"); + return { + kind: "remote", + ref, + repository: segments.slice(0, 2).join("/"), + subpath: segments.slice(2).join("/"), + target, + }; +} + +// Local uses paths are workspace-relative, not relative to the action file. +// Only a checkout selecting this captured source may remap a literal mount. +export function resolveLocalActionSource(path, context = {}) { + for (const step of [...(context.checkouts ?? [])].reverse()) { + if (condition(step.if) === "never") continue; + const input = step.with ?? {}; + if (!isMapping(input)) return { sourceError: "unresolved-checkout-source" }; + const mount = input.path ?? "."; + if (!literalWorkingDirectory(mount)) + return { sourceError: "unresolved-checkout-path" }; + const prefix = mount.replace(/^\.\//, "").replace(/\/$/, ""); + const root = prefix === "." || prefix === ""; + if (!root && path !== prefix && !path.startsWith(`${prefix}/`)) continue; + const repository = input.repository; + const ref = input.ref; + // In a reusable workflow, implicit checkout/github.repository select the + // caller, not necessarily the repository containing this workflow. + const sameRepository = + (typeof repository === "string" && repository === context.repository) || + (!context.reusableWorkflow && + (repository === undefined || + normalizeExpression(repository) === "${{github.repository}}")); + const sameRef = + ref === undefined || + ref === "" || + (context.headSha !== undefined && ref === context.headSha) || + (!context.reusableWorkflow && + normalizeExpression(ref) === "${{github.sha}}"); + if ( + !sameRepository || + !sameRef || + condition(step.if) !== "always" || + condition(step["continue-on-error"] ?? false) !== "never" || + input["sparse-checkout"] !== undefined || + step.env !== undefined + ) + return { + sourceError: "unresolved-checkout-source", + checkoutRepository: repository ?? context.repository, + checkoutRef: ref, + }; + return { + path: root ? path : path.slice(prefix.length).replace(/^\//, ""), + checkedOut: true, + }; + } + return { path }; +} + +// Bind only whole-value composite input references; never evaluate expressions +// or interpolate shell text. This preserves caller token names through helpers. +function bindInputs(value, inputs) { + if (typeof value !== "string") return value; + const match = value.match(/^\$\{\{\s*inputs\.([\w-]+)\s*\}\}$/); + return match + ? Object.hasOwn(inputs, match[1]) + ? inputs[match[1]] + : "" + : value; +} + +function bindStepInputs(step, inputs) { + if (!isMapping(step)) return step; + return Object.fromEntries( + Object.entries(step).map(([key, value]) => [ + key, + ["with", "env"].includes(key) && isMapping(value) + ? Object.fromEntries( + Object.entries(value).map(([name, item]) => [ + name, + bindInputs(item, inputs), + ]), + ) + : value, + ]), + ); +} + +function classifyUsesStep(step, context) { + const uses = parseUses(step.uses); + const withInput = step.with ?? {}; + + if (uses.kind === "local") { + const source = resolveLocalActionSource(uses.path, context); + if (source.sourceError) + return [{ kind: "unknown-local-action", uses: step.uses, ...source }]; + if ( + context.localSfwRelease && + source.checkedOut && + ["", "teardown"].includes(source.path) + ) { + return classifyUsesStep( + { + ...step, + uses: `${ACTION_REPOSITORY}${source.path ? "/teardown" : ""}@${APPROVED_RELEASE_SHA}`, + }, + context, + ).map((operation) => ({ + ...operation, + uses: step.uses, + localRuntimeVerified: true, + uncertain: true, + integrationConfigurationUncertain: false, + })); + } + const stack = context.actionStack ?? []; + if (stack.includes(uses.path) || stack.length >= 20) { + return [ + { + kind: "unknown-local-action", + uses: step.uses, + reason: "cyclic or deeply nested local action", + sourceError: "unresolved-local-action-expansion", + }, + ]; + } + const prefix = source.path ? `${source.path}/` : ""; + const actionText = + context.localActions?.get(`${prefix}action.yml`) ?? + context.localActions?.get(`${prefix}action.yaml`); + context.reviewSources?.set(prefix, actionText ?? null); + if (actionText === undefined) { + return [ + { + kind: "unknown-local-action", + uses: step.uses, + sourceError: "unresolved-local-action-source", + }, + ]; + } + let action; + try { + action = parse(actionText); + } catch { + return [ + { + kind: "unknown-local-action", + uses: step.uses, + sourceError: "local-action-parse-error", + }, + ]; + } + if ( + !isMapping(action) || + !isMapping(action.runs) || + typeof action.runs.using !== "string" || + (action.runs.using === "composite" && + (!Array.isArray(action.runs.steps) || action.runs.steps.length === 0)) + ) { + return [ + { + kind: "unknown-local-action", + uses: step.uses, + sourceError: "malformed-local-action", + }, + ]; + } + const steps = action?.runs?.steps; + if ( + action?.runs?.using !== "composite" || + !Array.isArray(steps) || + steps.length === 0 + ) { + return [{ kind: "unknown-local-action", uses: step.uses }]; + } + const inputs = { + ...Object.fromEntries( + Object.entries(action.inputs ?? {}).map(([key, input]) => [ + key, + input?.default ?? "", + ]), + ), + ...withInput, + }; + return steps.flatMap((inner) => + classifyStep(bindStepInputs(inner, inputs), { + ...context, + actionStack: [...stack, uses.path], + }).map((operation) => ({ + ...operation, + via: step.uses, + })), + ); + } + if (uses.kind === "docker") { + return [{ kind: "unknown", uses: step.uses }]; + } + + if (uses.repository === ACTION_REPOSITORY) { + const kind = uses.subpath === "teardown" ? "sfw-teardown" : "sfw-setup"; + if (uses.subpath !== "" && uses.subpath !== "teardown") { + return [{ kind: "unknown", uses: step.uses }]; + } + return [ + { + configureBun: normalizeExpression( + withInput["configure-bun"] ?? "false", + ), + fallback: normalizeExpression( + withInput["allow-external-fork-fallback"] ?? "false", + ), + kind, + id: step.id, + condition: step.if, + boundaryUncertainWithoutCondition: boundaryUncertain({ + ...step, + if: undefined, + }), + ref: uses.ref, + token: normalizeExpression(withInput.token ?? ""), + uses: step.uses, + }, + ]; + } + if (uses.repository === "actions/setup-node" && uses.subpath === "") { + return [ + { + kind: "setup-node", + registryMutating: withInput["registry-url"] !== undefined, + registryPersisting: withInput["registry-url"] !== undefined, + uses: step.uses, + }, + ]; + } + if (uses.repository === "actions/checkout" && uses.subpath === "") { + return [ + { + kind: "checkout", + uncertain: + withInput.ref !== undefined || withInput.repository !== undefined, + persistCredentials: normalizeExpression( + withInput["persist-credentials"] ?? "true", + ), + ref: String(withInput.ref ?? ""), + repository: String(withInput.repository ?? ""), + uses: step.uses, + }, + ]; + } + if (uses.repository === "pnpm/action-setup" && uses.subpath === "") { + const runInstall = withInput.run_install; + if (runInstall !== undefined && String(runInstall) !== "false") { + return [ + { + kind: "js-public-download", + manager: "pnpm", + uses: step.uses, + uncertain: true, + integrationConfigurationUncertain: ![true, "true"].includes( + runInstall, + ), + }, + ]; + } + return [ + { + kind: "unknown", + uses: step.uses, + }, + ]; + } + return [ + { + kind: "unknown", + uses: step.uses, + }, + ]; +} + +export function classifyStep(step, context = {}) { + if (context.stepBudget && --context.stepBudget.remaining < 0) { + return [ + { + kind: "unknown-local-action", + reason: "local action expansion limit reached", + sourceError: "unresolved-local-action-expansion", + }, + ]; + } + if (!isMapping(step)) return [{ kind: "unknown" }]; + if (condition(step.if) === "never") return []; + if ( + isMapping(step.with) && + Object.values(step.with).some( + (value) => value !== null && typeof value === "object", + ) + ) + return [{ kind: "unknown", sourceError: "malformed-action-input" }]; + // Resolve only this root-manifest bootstrap, not arbitrary shell variables. + // Snapshot evidence does not certify execution of the JSON reader at runtime. + const pinnedBootstrap = + !context.actionStack?.length && + typeof context.packageManager === "string" && + context.packageManager === context.packageManager.trim() && + /^pnpm@(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$/.test( + context.packageManager ?? "", + ) && + [undefined, ".", "./"].includes(step["working-directory"]) && + typeof step.run === "string" && + step.run + .trim() + .split("\n") + .map((line) => line.trim()) + .join("\n") === PNPM_BOOTSTRAP; + if (pinnedBootstrap) + step = { + ...step, + run: `npm install --global ${context.packageManager} --ignore-scripts --no-audit --no-fund`, + }; + const lines = + typeof step.run === "string" + ? step.run + .trim() + .split("\n") + .map((line) => line.trim()) + : []; + const offlineBun = + !context.actionStack?.length && + context.offlineBunAllowed !== false && + !environmentUncertain(step.env) && + (step.shell === undefined || SUPPORTED_SHELLS.has(step.shell)) && + lines[0] === + "if ! bun install --help | grep -F -- '--offline ' >/dev/null; then" && + /^echo '[^'\r\n$`\\]*' >&2$/.test(lines[1] ?? "") && + lines[2] === "exit 1" && + lines[3] === "fi" && + lines[4] === + "bun install --lockfile-only --offline --ignore-scripts --registry=https://registry.npmjs.org/"; + let operations; + let uncertain = boundaryUncertain(step) || pinnedBootstrap; + let integrationEnv = step.env; + if ( + typeof step.uses === "string" && + step.uses.split("@")[0] === ACTION_REPOSITORY && + isMapping(step.env) && + /^\$\{\{runner\.temp\}\}\/[\w.-]+$/.test( + normalizeExpression(step.env.NPM_CONFIG_USERCONFIG), + ) + ) { + // Setup validates this path and configures both it and HOME/.npmrc. + // An override on an install step still requires separate review. + integrationEnv = { ...step.env }; + delete integrationEnv.NPM_CONFIG_USERCONFIG; + } + const registryExclusion = + context.registryExclusion && + !context.actionStack?.length && + [undefined, ".", "./"].includes(step["working-directory"]) && + isMapping(integrationEnv) && + integrationEnv.NPM_CONFIG_REPLACE_REGISTRY_HOST === "npmjs" && + /^\s*npm\s+(?:ci|install)\s*$/.test(step.run ?? "") + ? context.registryExclusion.id + : undefined; + if (registryExclusion) { + integrationEnv = { ...integrationEnv }; + delete integrationEnv.NPM_CONFIG_REPLACE_REGISTRY_HOST; + } + let integrationUncertain = + boundaryUncertain({ ...step, env: undefined, if: primaryRunGuard(step) }) || + environmentUncertain(integrationEnv); + const configurationBoundaryUncertain = integrationUncertain; + if ( + step.uses !== undefined && + step.run === undefined && + typeof step.uses === "string" && + (step.with === undefined || isMapping(step.with)) + ) { + operations = classifyUsesStep(step, context); + } else if (typeof step.run === "string" && step.uses === undefined) { + // Deliberately not a shell interpreter: retain recognizable downloads but + // never certify control flow, substitutions, redirections or custom shells. + const complexShell = (script) => + /[;'"$`|<>(){}\\]|(?:^|\s)(?:if|then|else|fi|for|while|case|eval|source|sudo)(?:\s|$)|(? + /^(?:if|case|for|while|until|select)\b/.test(command), + ); + let offlineCommandPending = offlineBun; + operations = shell.commands.map((command) => { + const offlineValidation = offlineCommandPending && command === lines[4]; + if (offlineValidation) offlineCommandPending = false; + const operation = offlineValidation + ? { + command, + kind: "no-network", + offlineValidation: true, + uncertain: true, + } + : classifyCommand(command, context); + const { words, rawWords, environment, preservedEnvironment } = + commandParts(command, context); + const bunArguments = approvedBunArguments(words, rawWords); + const tokens = shellTokens(command); + if (tokens.limit || tokens.lexError) + operation.sourceError = "unresolved-shell-source"; + const nestedCommands = tokens.substitutions.flatMap( + (body) => shellCommands(body).commands, + ); + const nested = nestedCommands.map((body) => commandParts(body).words); + const nestedEnvironment = nestedCommands.some( + (body, index) => + environmentUncertain(commandParts(body).environment) || + (classifyCommand(body).kind === "unknown-wrapper" && + !classifyCommand(body).manager && + unresolvedJsInvocation(classifyCommand(body))) || + changesStepEnvironment(body) || + (/^(?:env|sudo|command|time|exec)$/.test(nested[index][0] ?? "") && + unresolvedJsInvocation(classifyCommand(body))), + ); + const executorPrefix = directExecutor(words); + // npm's exact 'always' mode rewrites lockfile hosts to the configured + // registry; it does not select another registry. Other modes need review. + const replacementFlags = words.filter((word) => + word.startsWith("--replace-registry-host"), + ); + const configurationWords = installerArguments(words).filter( + (word) => !word.startsWith("--replace-registry-host"), + ); + const supportedReplacement = + words[0] === "npm" && + replacementFlags.every( + (word) => word === "--replace-registry-host=always", + ); + const targets = [ + ...writeTargets(tokens), + ...nestedCommands.flatMap((body) => writeTargets(shellTokens(body))), + ]; + const configKinds = targets.map(configTarget); + const configFileWrite = configKinds.some(Boolean); + const bunFileOnly = configFileWrite && !configKinds.includes("npm"); + const knownRegistryContent = + ["echo", "printf"].includes(words[0]) && + tokens.words.some((word) => + /(?:^|\n)(?:@[^:]+:)?registry\s*=/.test(word), + ); + const setter = registrySetter(words) || nested.some(registrySetter); + const dynamicSetter = [words, ...nested].some( + (args) => + ["npm", "pnpm"].includes(args[0]) && + args[1] === "config" && + ["set", "delete", "unset"].includes(args[2]) && + /[$`]/.test(args.slice(3).join(" ")), + ); + const nestedOverride = nested.some( + (args) => + ["npm", "pnpm", "bun", "yarn", "npx", "bunx"].includes(args[0]) && + installerArguments(args).some((word) => REGISTRY_FLAG.test(word)), + ); + const registryMutating = + !offlineValidation && + (setter || + dynamicSetter || + configFileWrite || + nestedOverride || + (["npm", "pnpm", "bun", "yarn", "npx", "bunx"].includes(words[0]) && + configurationWords.some((word) => REGISTRY_FLAG.test(word))) || + Object.keys(environment).some((key) => + /^(?:npm|pnpm|bun)_CONFIG_/i.test(key), + )); + // A literal npm prefix selects the project directory, just like a + // run-step working-directory. Do not accept dynamic or escaping paths. + const directoryArguments = new Set(); + if (words[0] === "npm" && !words[1]?.startsWith("-")) { + words.forEach((word, index) => { + if ( + word === "--prefix" && + literalWorkingDirectory(words[index + 1]) + ) { + directoryArguments.add(index); + directoryArguments.add(index + 1); + } else if ( + word.startsWith("--prefix=") && + literalWorkingDirectory(word.slice(9)) + ) { + directoryArguments.add(index); + } + }); + } + const unsupportedFlags = + operation.kind === "js-public-download" && + operation.corepack === undefined && + words + .filter( + (word, index) => + !bunArguments.has(index) && + !directoryArguments.has(index) && + !( + supportedReplacement && + word === "--replace-registry-host=always" + ), + ) + .some( + (word) => + word.startsWith("-") && !TRANSPARENT_INSTALL_FLAGS.has(word), + ); + if (registryMutating) { + operation.registryMutating = true; + operation.registryPersisting = + setter || dynamicSetter || configFileWrite; + if (bunFileOnly) operation.registryManager = "bun"; + } else if ( + unsupportedFlags || + (replacementFlags.length > 0 && !supportedReplacement) + ) + operation.uncertain = true; + if (Object.keys(environment).length && !registryMutating) + operation.uncertain = true; + operation.integrationUncertain = + environmentUncertain(environment) || + (replacementFlags.length > 0 && !supportedReplacement) || + (words[0] === "npm" && ["exec", "x"].includes(words[1])) || + (unsupportedFlags && !registryMutating) || + (operation.kind === "js-public-download" && + /[$`]/.test( + words + .filter( + (word, index) => + !bunArguments.has(index) && + !(words[0] === "npm" && LOCAL_TARBALL.test(word)), + ) + .join(" "), + )) || + (configFileWrite && !knownRegistryContent) || + nestedOverride || + dynamicSetter; + if ( + ["npx", "bunx"].includes(words[0]) || + (words[0] === "npm" && ["exec", "x"].includes(words[1])) + ) { + // Strict assurance still sees unknown executor code/configuration flags. + operation.uncertain = true; + operation.integrationExecutor = executorPrefix !== undefined; + operation.integrationUncertain = + environmentUncertain(environment) || + (executorPrefix === undefined && !registryMutating); + operation.integrationUncertain ||= nestedOverride; + // Persistence comes from executable setters/writes, not payload text. + } + if (setter && nested.some(registrySetter)) + operation.integrationUncertain = true; + // Reachability is not a certain state transition: a skipped disable + // cannot clear a persistent Corepack shim established by an earlier step. + if ( + operation.corepackControl || + ["enable", "disable"].includes(operation.corepack) + ) + operation.integrationUncertain ||= + condition(step.if) === "uncertain" || conditionalControl; + if ( + operation.registryMutating && + (conditionalControl || condition(step.if) === "uncertain") + ) + operation.integrationUncertain = true; + operation.commandGroup = context.stepBudget?.remaining; + operation.stepEnvironmentUncertain = changesStepEnvironment(command); + operation.environmentPersisting = + writesEnvironment(tokens) || + nestedCommands.some((body) => writesEnvironment(shellTokens(body))); + const nestedCandidates = nestedCommands.map(classifyCommand); + const nestedInvocations = nestedCandidates.filter((candidate, index) => { + const args = nested[index]; + const configRead = + args[1] === "config" && ["get", "list", "ls"].includes(args[2]); + return ( + candidate.kind === "js-public-download" || + candidate.kind === "yarn-blocked" || + (unresolvedJsInvocation(candidate) && + !configRead && + !registrySetter(args) && + !scriptInvocation(args, candidate.kind)) + ); + }); + operation.explicitJsInvocation = + nestedOverride || nestedInvocations.length > 0; + const directInvocation = + !scriptInvocation(words, operation.kind) && + unresolvedJsInvocation({ ...operation, explicitJsInvocation: false }); + const identifiedInvocations = directInvocation + ? [...nestedInvocations, operation] + : nestedInvocations; + operation.integrationManagers = [ + ...new Set( + identifiedInvocations + .map((candidate) => { + const manager = + candidate.manager ?? + shellTokens(candidate.command).words.find((word) => + ["npm", "pnpm", "bun", "yarn", "npx", "bunx"].includes(word), + ); + return manager === "npx" + ? "npm" + : manager === "bunx" + ? "bun" + : manager; + }) + .filter(Boolean), + ), + ]; + operation.integrationCorepackDownload = nestedInvocations.some( + (candidate) => candidate.corepack, + ); + operation.integrationCorepackUncertain = nestedCandidates.some( + (candidate) => + candidate.corepackControl || + ["enable", "disable"].includes(candidate.corepack), + ); + operation.integrationConfigurationUncertain = + operation.integrationUncertain || + (operation.explicitJsInvocation && nestedEnvironment) || + (directInvocation && !operation.manager) || + (step.shell !== undefined && !SUPPORTED_SHELLS.has(step.shell)) || + (operation.kind === "unknown-wrapper" && + operation.manager !== undefined && + (words[1]?.startsWith("-") || /[$`]/.test(words[1] ?? ""))) || + /^(?![A-Za-z_][A-Za-z0-9_]*=)[^\s=]+=/.test(command) || + /^(?:\.?\.?\/|\/)[^\s]*\/(?:npm|pnpm|npx|bun|bunx|yarn|corepack)\b/.test( + command, + ) || + (!preservedEnvironment && + /^(?:env|sudo|command|time|exec)$/.test(words[0] ?? "") && + /(?:^|\s)(?:npm|pnpm|npx|bun|bunx|yarn|corepack)\b/.test(command)) || + (operation.registryMutating && shell.ambiguous); + operation.integrationSyntaxUncertain = shell.ambiguous; + operation.integrationLiteral = literalLogging(command); + // Script bodies are unverified code, not presumed dependency installs. + // pnpm/Yarn support script-name shorthands; their configuration/toolchain + // commands are not shorthands. Explicit nested JS commands stay reviewable. + operation.integrationScript = scriptInvocation(words, operation.kind); + return operation; + }); + if (shell.limit || shell.lexError) + operations.push({ + kind: "unknown", + sourceError: "unresolved-shell-source", + }); + } else { + operations = [{ kind: "unknown" }]; + } + return operations.map((operation) => ({ + ...operation, + ...(registryExclusion ? { registryExclusion } : {}), + ...(uncertain || operation.uncertain ? { uncertain: true } : {}), + integrationUncertain: + integrationUncertain || + (operation.integrationUncertain ?? operation.uncertain ?? false), + integrationConfigurationUncertain: + configurationBoundaryUncertain || + (operation.integrationConfigurationUncertain ?? + operation.integrationUncertain ?? + operation.uncertain ?? + false), + })); +} + +function scriptInvocation(words, kind) { + return ( + kind === "unknown-wrapper" && + ["npm", "pnpm", "bun", "yarn"].includes(words[0]) && + (/^(?:run|test|start|stop|restart)$/.test(words[1] ?? "") || + (words[0] === "bun" && + (words[1] === "build" || + (literalWorkingDirectory(words[1]) && + /\.[cm]?[jt]sx?$/.test(words[1])))) || + (words[0] === "npm" && + (words[1] === "version" || + (words[1] === "pack" && + words + .slice(2) + .every((arg) => + ["--json", "--ignore-scripts", "--dry-run"].includes(arg), + )))) || + (["pnpm", "yarn"].includes(words[0]) && + /^[A-Za-z0-9_:.+-]+$/.test(words[1] ?? "") && + !/^(?:config|env|setup|set|create|self-update|plugin|policies|patch|patch-commit)$/.test( + words[1], + ) && + !words + .slice(2) + .some((word) => + /^(?:npm|pnpm|npx|bun|bunx|yarn|corepack)$/.test(word), + ))) + ); +} + +function normalizeTriggers(workflow) { + const triggers = workflow?.on ?? workflow?.[true]; + if (typeof triggers === "string") { + return [triggers]; + } + if (Array.isArray(triggers)) { + return triggers.map(String).sort(); + } + if (triggers && typeof triggers === "object") { + return Object.keys(triggers).sort(); + } + return []; +} + +function collectViolations(operations, context, triggers, job) { + const violations = []; + const downloads = []; + const setups = []; + const publishes = []; + const expectedToken = expectedTokenExpression(context.visibility); + let activeSetup; + let lastSetup; + let registryInvalidated = false; + let publishBoundary = true; + let corepackEnabled = false; + + operations.forEach((operation, index) => { + if (operation.corepack === "enable") corepackEnabled = true; + if (operation.corepack === "disable") corepackEnabled = false; + if (operation.kind === "js-public-download") { + downloads.push(index); + if (registryInvalidated || operation.registryMutating) { + violations.push( + `registry-mutating configuration precedes download operation ${index + 1}`, + ); + } + if (operation.manager === "pnpm" && corepackEnabled) { + violations.push( + `Corepack may lazily download pnpm directly from registry.npmjs.org at operation ${index + 1}; install pinned pnpm with npm through Socket Firewall instead`, + ); + } + if ( + activeSetup === undefined || + operation.uncertain || + operation.registryMutating + ) { + violations.push( + `download operation ${index + 1} has no certain active Socket Firewall setup`, + ); + } + if (operation.manager === "bun" && activeSetup?.configureBun !== "true") { + violations.push( + `Bun download operation ${index + 1} requires configure-bun: true`, + ); + } + publishBoundary = false; + } + if (operation.registryMutating === true) { + registryInvalidated = true; + activeSetup = undefined; + } + if (operation.kind === "js-publish") { + publishes.push(index); + if (setups.length > 0 && !publishBoundary) { + violations.push( + "publish follows Socket Firewall setup without a same-SHA teardown boundary", + ); + } + } + if (operation.kind === "sfw-setup") { + lastSetup = operation; + registryInvalidated = false; + setups.push(index); + publishBoundary = false; + activeSetup = + operation.ref === APPROVED_RELEASE_SHA && + operation.token === expectedToken && + !operation.uncertain && + operation.fallback === "false" + ? operation + : undefined; + if (operation.ref !== APPROVED_RELEASE_SHA) { + violations.push( + FULL_SHA_PATTERN.test(operation.ref) + ? `sfw-setup pins unapproved SHA ${operation.ref}` + : `sfw-setup uses mutable or short ref "${operation.ref}"`, + ); + } + if (operation.token !== expectedToken) { + violations.push( + `sfw-setup token must be ${expectedToken} for ${context.visibility} repositories`, + ); + } + if (context.visibility !== "public" && operation.fallback === "true") { + violations.push( + "allow-external-fork-fallback is enabled outside a public repository", + ); + } + } + if (operation.kind === "sfw-teardown") { + registryInvalidated = false; + activeSetup = undefined; + publishBoundary = certainTeardown(operation, lastSetup); + if (operation.ref !== APPROVED_RELEASE_SHA) { + violations.push( + `sfw-teardown ref "${operation.ref}" does not match the approved release SHA`, + ); + } + } + }); + + if ( + operations.some( + (operation) => + operation.corepack !== undefined && + operation.corepack !== "enable" && + operation.corepack !== "disable", + ) + ) { + violations.push( + "Corepack package-manager downloads do not use Socket Firewall npm configuration", + ); + } + + if (downloads.length > 0 && setups.length > 0 && setups[0] > downloads[0]) { + violations.push("Socket Firewall setup runs after the first download"); + } + + if (context.visibility === "public") { + const unsafeTriggers = triggers.filter((trigger) => + UNSAFE_PUBLIC_TRIGGERS.has(trigger), + ); + const installBearing = + downloads.length > 0 || setups.length > 0 || publishes.length > 0; + if (unsafeTriggers.length > 0 && installBearing) { + violations.push( + `install-bearing job is reachable from privileged trigger(s): ${unsafeTriggers.join(", ")}`, + ); + } + for (const operation of operations) { + if ( + operation.kind === "checkout" && + installBearing && + (CONTRIBUTOR_HEAD_PATTERN.test(operation.ref) || + CONTRIBUTOR_HEAD_PATTERN.test(operation.repository)) && + unsafeTriggers.length > 0 + ) { + violations.push( + "privileged trigger checks out contributor-controlled head in an install-bearing job", + ); + } + } + } + + if (job?.secrets === "inherit") { + violations.push("reusable workflow call inherits all secrets"); + } + + return violations; +} + +export function classifyJob(jobName, job, context = {}, triggers = []) { + if (!isMapping(job)) { + return { + job: jobName, + managers: [], + operations: [], + status: "unknown", + violations: ["job definition is not a mapping"], + integration: { + disposition: "needs-review", + downloads: [], + notes: ["malformed-job"], + runtimeVerification: "not-performed", + }, + }; + } + + if (job.uses !== undefined && condition(job.if) === "never") { + return { + job: jobName, + managers: [], + operations: [], + status: "no-in-scope-download", + violations: [], + integration: { + disposition: "no-js-ci", + downloads: [], + notes: [], + runtimeVerification: "not-performed", + }, + }; + } + if (job.uses !== undefined) { + const violations = + job.secrets === "inherit" + ? ["reusable workflow call inherits all secrets"] + : []; + return { + job: jobName, + managers: [], + operations: + typeof job.uses === "string" && + job.steps === undefined && + (job.with === undefined || isMapping(job.with)) && + (job.secrets === undefined || + job.secrets === "inherit" || + isMapping(job.secrets)) + ? [{ kind: "reusable-call", uses: job.uses }] + : [{ kind: "unknown", sourceError: "malformed-reusable-call" }], + status: "reusable-call", + violations, + integration: { + disposition: "needs-review", + downloads: [], + notes: ["unresolved-reusable-workflow"], + runtimeVerification: "not-performed", + }, + }; + } + + const steps = Array.isArray(job.steps) ? job.steps : []; + const matrix = job.strategy?.matrix; + const literalMatrixKeys = new Set( + matrix && + typeof matrix === "object" && + !Array.isArray(matrix) && + matrix.include === undefined + ? Object.entries(matrix) + .filter( + ([, values]) => + Array.isArray(values) && + values.length > 0 && + values.every( + (value) => + typeof value === "string" && + /^[A-Za-z0-9_][\w.-]*$/.test(value), + ), + ) + .map(([key]) => key) + : [], + ); + const checkouts = steps + .map((step, index) => ({ step, index })) + .filter( + ({ step }) => + typeof step?.uses === "string" && + step.uses.startsWith("actions/checkout@"), + ); + const checkout = checkouts[0]; + const defaultCheckout = + checkouts.length === 1 && + condition(checkout.step.if) === "always" && + condition(checkout.step["continue-on-error"] ?? false) === "never" && + ["ref", "repository", "path", "sparse-checkout"].every( + (key) => checkout.step.with?.[key] === undefined, + ); + const stepContext = { + ...context, + reusableWorkflow: triggers.includes("workflow_call"), + offlineBunAllowed: + !environmentUncertain(job.env) && + !defaultsUncertain(job.defaults) && + !context.integrationContextUncertain && + job.container === undefined, + registryExclusion: + defaultCheckout && + context.exclusionRootDirectory !== false && + [undefined, ".", "./"].includes(job.defaults?.run?.["working-directory"]) + ? context.registryExclusion + : undefined, + literalMatrixKeys, + stepBudget: { remaining: 1000 }, + }; + let packageManagerSourceUnchanged = + defaultCheckout && + [undefined, ".", "./"].includes(job.defaults?.run?.["working-directory"]) && + context.exclusionRootDirectory !== false; + const priorCheckouts = []; + const operations = + condition(job.if) === "never" + ? [] + : steps.flatMap((step, index) => { + const afterCheckout = index > (checkout?.index ?? -1); + if ( + typeof step?.uses === "string" && + step.uses.startsWith("actions/checkout@") + ) + priorCheckouts.push(step); + const currentContext = { + ...stepContext, + checkouts: priorCheckouts, + registryExclusion: afterCheckout + ? stepContext.registryExclusion + : undefined, + packageManager: + afterCheckout && packageManagerSourceUnchanged + ? context.packageManager + : undefined, + }; + // Earlier shell/local or mutable action execution can replace the file. + if ( + typeof step?.uses !== "string" || + step.uses.startsWith("./") || + !FULL_SHA_PATTERN.test(step.uses.split("@")[1] ?? "") + ) + packageManagerSourceUnchanged = false; + return classifyStep(step, currentContext).map((operation) => ({ + ...operation, + step: index + 1, + })); + }); + if ( + !Array.isArray(job.steps) || + steps.length === 0 || + job.container !== undefined || + job.services !== undefined || + boundaryUncertain(job) || + context.workflowUncertain || + (job.defaults?.run?.shell !== undefined && + !SUPPORTED_SHELLS.has(job.defaults.run.shell)) + ) { + operations.push({ + kind: "unknown", + reason: "opaque job or workflow execution context", + }); + } + const violations = collectViolations(operations, context, triggers, job); + + const kinds = new Set(operations.map((operation) => operation.kind)); + const managers = [ + ...new Set( + operations + .filter((operation) => operation.kind === "js-public-download") + .map((operation) => operation.manager) + .filter(Boolean), + ), + ].sort(); + + const hasDownload = kinds.has("js-public-download"); + const hasPublish = kinds.has("js-publish"); + const hasSetup = kinds.has("sfw-setup"); + const hasUnknown = + kinds.has("unknown") || + kinds.has("unknown-wrapper") || + kinds.has("unknown-local-action") || + operations.some( + (operation) => + operation.uncertain || + (operation.kind === "sfw-setup" && operation.fallback !== "false"), + ); + const publishUnsafe = violations.some((violation) => + violation.includes("without a same-SHA teardown"), + ); + const trustUnsafe = violations.some( + (violation) => + violation.includes("privileged trigger") || + violation.includes("inherits all secrets"), + ); + + let status; + if (trustUnsafe) { + status = "unsafe-trust"; + } else if (publishUnsafe) { + status = "unsafe-publish"; + } else if (hasUnknown) { + // A conditional setup, opaque wrapper, or supported fork fallback is not + // proof of a missing integration. Retain violations for private review. + status = "unknown"; + } else if (kinds.has("yarn-blocked")) { + status = "blocked-yarn"; + } else if (hasDownload && violations.length > 0) { + status = "unprotected"; + } else if (hasDownload && hasSetup) { + status = "protected"; + } else if (hasDownload) { + status = "unprotected"; + } else if (hasPublish) { + status = hasSetup ? "unsafe-publish" : "safe-publish"; + } else if (kinds.has("other-ecosystem")) { + status = "out-of-scope"; + } else { + status = "no-in-scope-download"; + } + + return { + job: jobName, + managers, + operations, + status, + violations, + integration: observedIntegration(operations, job, context), + }; +} + +export function classifyWorkflow(text, context) { + let workflow; + try { + workflow = parse(text); + } catch (error) { + return { + jobs: [], + parseError: error.message, + path: context.path, + status: "unknown", + triggers: [], + }; + } + if ( + workflow == null && + text.trim().startsWith("#") && + text.split("\n").every((line) => /^\s*(?:#.*)?$/.test(line)) + ) { + return { + jobs: [], + path: context.path, + triggers: [], + status: "no-in-scope-download", + }; + } + if ( + !isMapping(workflow) || + !isMapping(workflow.jobs) || + Object.keys(workflow.jobs).length === 0 + ) { + return { + jobs: [], + parseError: "workflow or jobs is not a nonempty mapping", + path: context.path, + status: "unknown", + triggers: [], + }; + } + + const triggers = normalizeTriggers(workflow); + if (triggers.length === 0) { + return { + jobs: [], + path: context.path, + triggers, + status: "unknown", + parseError: "workflow has no recognized trigger declaration", + }; + } + const jobEntries = Object.entries(workflow.jobs ?? {}).sort(([a], [b]) => + a < b ? -1 : a > b ? 1 : 0, + ); + const jobs = jobEntries.map(([jobName, job]) => { + const reviewSources = new Map(); + const result = classifyJob( + jobName, + job, + { + ...context, + reviewSources, + exclusionRootDirectory: [undefined, ".", "./"].includes( + workflow.defaults?.run?.["working-directory"], + ), + workflowUncertain: + workflow.env !== undefined || workflow.defaults !== undefined, + integrationContextUncertain: + environmentUncertain(workflow.env) || + defaultsUncertain(workflow.defaults), + }, + triggers, + ); + const upstream = Object.create(null); + const pending = [jobName]; + const visited = new Set(); + while (pending.length) { + const name = pending.pop(); + if (visited.has(name)) continue; + visited.add(name); + const parent = Object.hasOwn(workflow.jobs, name) + ? workflow.jobs[name] + : undefined; + if (name !== jobName) upstream[name] = parent ?? null; + const needs = parent?.needs; + pending.push( + ...(typeof needs === "string" + ? [needs] + : Array.isArray(needs) + ? needs.filter((item) => typeof item === "string") + : []), + ); + } + return { + ...result, + reviewDependencies: [...visited] + .filter((name) => name !== jobName) + .sort(), + reviewFingerprint: fingerprint({ + version: 1, + job, + upstream, + workflow: { + on: workflow.on, + env: workflow.env, + defaults: workflow.defaults, + }, + visibility: context.visibility, + defaultBranch: context.defaultBranch, + configuration: context.reviewConfiguration, + packageManager: context.packageManager, + localSfwRelease: context.localSfwRelease, + approvedRelease: APPROVED_RELEASE_SHA, + sources: Object.fromEntries( + [...reviewSources].map(([path, text]) => { + try { + return [path, parse(text)]; + } catch { + return [path, text]; + } + }), + ), + }), + }; + }); + + return { jobs, path: context.path, triggers }; +} + +const STATUS_SEVERITY = [ + "unsafe-trust", + "unsafe-publish", + "blocked-yarn", + "unprotected", + "unknown", + "reusable-call", + "protected", + "safe-publish", + "out-of-scope", + "no-in-scope-download", +]; + +export function repositoryDisposition(workflowResults, evidence = {}) { + const statuses = new Set(); + for (const workflow of workflowResults) { + if (workflow.parseError !== undefined) { + statuses.add("unknown"); + } + for (const job of workflow.jobs) { + statuses.add(job.status); + } + } + + if (evidence.error !== undefined) { + return "audit-error"; + } + for (const status of STATUS_SEVERITY) { + if (statuses.has(status)) { + switch (status) { + case "unsafe-trust": + return "blocked-trust"; + case "unsafe-publish": + return "unsafe-publish"; + case "blocked-yarn": + return "blocked-yarn"; + case "unprotected": + return "needs-sfw"; + case "unknown": + case "reusable-call": + return "needs-review"; + case "protected": + return "protected"; + case "safe-publish": + case "out-of-scope": + return "out-of-scope"; + default: + return "no-js-ci"; + } + } + } + return workflowResults.length === 0 ? "no-ci" : "no-js-ci"; +} diff --git a/tools/rollout/classify.test.mjs b/tools/rollout/classify.test.mjs new file mode 100644 index 0000000..6cb0c87 --- /dev/null +++ b/tools/rollout/classify.test.mjs @@ -0,0 +1,949 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, readdir, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { auditRepository, runAudit, writeReportAtomically } from "./audit.mjs"; +import { + classifyCommand, + classifyJob, + classifyWorkflow, + repositoryDisposition, +} from "./classify.mjs"; +import { main, scanExitCode } from "./cli.mjs"; +import { GitHubClient } from "./github.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; + +const SETUP = `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`; +const TEARDOWN = `workos/setup-socket-firewall/teardown@${APPROVED_RELEASE_SHA}`; +const PRIVATE_TOKEN = "${{ secrets.SOCKET_FIREWALL_TOKEN }}"; +const PUBLIC_TOKEN = "${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }}"; + +function workflow(jobsYaml, triggers = "push") { + return `on: ${triggers}\njobs:\n${jobsYaml}`; +} + +function classify(text, visibility = "private", localActions = new Map()) { + return classifyWorkflow(text, { + localActions, + path: ".github/workflows/ci.yml", + visibility, + }); +} + +test("command grammar", () => { + const cases = [ + ["npm ci --ignore-scripts", "js-public-download"], + ["npm install --no-audit", "js-public-download"], + ["npm audit", "no-network"], + ["npm audit fix", "js-public-download"], + ["npm publish --access public", "js-publish"], + ["npm run build", "unknown-wrapper"], + ["npm test", "unknown-wrapper"], + ["npx changeset publish", "js-public-download"], + ["pnpm install --frozen-lockfile", "js-public-download"], + ["pnpm dlx create-thing", "js-public-download"], + ["pnpm exec vitest run", "unknown-wrapper"], + ["pnpm publish --no-git-checks", "js-publish"], + ["bun install --frozen-lockfile", "js-public-download"], + ["bunx biome check", "js-public-download"], + ["bun run test", "unknown-wrapper"], + ["bun publish", "js-publish"], + ["yarn install --frozen-lockfile", "yarn-blocked"], + ["yarn", "yarn-blocked"], + ["yarn npm publish", "js-publish"], + ["yarn build", "unknown-wrapper"], + ["corepack enable", "no-network"], + ["corepack prepare pnpm@10 --activate", "js-public-download"], + ["pip install -r requirements.txt", "other-ecosystem"], + ["go build ./...", "other-ecosystem"], + ["docker build .", "unknown-wrapper"], + ["make bootstrap", "unknown-wrapper"], + ["./scripts/setup.sh", "unknown-wrapper"], + ["bash scripts/install.sh", "unknown-wrapper"], + ["turbo run lint", "unknown-wrapper"], + ["lerna bootstrap", "js-public-download"], + ["CI=1 npm ci", "js-public-download"], + ["echo done", "no-network"], + ]; + for (const [command, kind] of cases) { + assert.equal(classifyCommand(command).kind, kind, command); + } +}); + +test("classifier: unprotected npm install needs SFW", () => { + const result = classify( + workflow( + ` build:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@sha\n - uses: actions/setup-node@sha\n with: { registry-url: "https://registry.npmjs.org/" }\n - run: npm ci\n`, + ), + ); + assert.equal(result.jobs[0].status, "unprotected"); + assert.deepEqual(result.jobs[0].managers, ["npm"]); + assert.equal(repositoryDisposition([result]), "needs-sfw"); +}); + +test("classifier: correctly ordered pinned setup is protected", () => { + const result = classify( + workflow( + ` build:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@sha\n - uses: actions/setup-node@sha\n with: { registry-url: "https://registry.npmjs.org/" }\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: npm ci\n`, + ), + ); + assert.deepEqual(result.jobs[0].violations, []); + assert.equal(result.jobs[0].status, "protected"); + assert.equal(repositoryDisposition([result]), "protected"); +}); + +test("classifier: registry mutation between setup and download is a violation", () => { + const result = classify( + workflow( + ` build:\n steps:\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - uses: actions/setup-node@sha\n with: { registry-url: "https://registry.npmjs.org/" }\n - run: npm ci\n`, + ), + ); + assert.equal(result.jobs[0].status, "unprotected"); + assert.match(result.jobs[0].violations.join(" "), /registry-mutating/); +}); + +test("classifier: setup after the first download is a violation", () => { + const result = classify( + workflow( + ` build:\n steps:\n - run: npm ci\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n`, + ), + ); + assert.equal(result.jobs[0].status, "unprotected"); + assert.match(result.jobs[0].violations.join(" "), /after the first download/); +}); + +test("classifier: mutable or mismatched refs are violations", () => { + const mutable = classify( + workflow( + ` build:\n steps:\n - uses: workos/setup-socket-firewall@v1\n with: { token: "${PRIVATE_TOKEN}" }\n - run: npm ci\n`, + ), + ); + assert.equal(mutable.jobs[0].status, "unprotected"); + assert.match(mutable.jobs[0].violations.join(" "), /mutable or short ref/); + + const stale = classify( + workflow( + ` build:\n steps:\n - uses: workos/setup-socket-firewall@${"a".repeat(40)}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: npm ci\n`, + ), + ); + assert.equal(stale.jobs[0].status, "unprotected"); + assert.match(stale.jobs[0].violations.join(" "), /unapproved SHA/); +}); + +test("classifier: token must match repository visibility", () => { + const publicWrongSecret = classify( + workflow( + ` build:\n steps:\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: npm ci\n`, + ), + "public", + ); + assert.equal(publicWrongSecret.jobs[0].status, "unprotected"); + assert.match( + publicWrongSecret.jobs[0].violations.join(" "), + /PUBLIC_SOCKET_FIREWALL_TOKEN.*public/, + ); + + const publicRight = classify( + workflow( + ` build:\n steps:\n - uses: ${SETUP}\n with: { token: "${PUBLIC_TOKEN}" }\n - run: npm ci\n`, + ), + "public", + ); + assert.equal(publicRight.jobs[0].status, "protected"); + + const internalWrongSecret = classify( + workflow( + ` build:\n steps:\n - uses: ${SETUP}\n with: { token: "${PUBLIC_TOKEN}" }\n - run: npm ci\n`, + ), + "internal", + ); + assert.match( + internalWrongSecret.jobs[0].violations.join(" "), + /SOCKET_FIREWALL_TOKEN.*internal/, + ); +}); + +test("classifier: publish boundaries", () => { + const cleanPublish = classify( + workflow(` publish:\n steps:\n - run: npm publish\n`), + ); + assert.equal(cleanPublish.jobs[0].status, "safe-publish"); + + const restored = classify( + workflow( + ` release:\n steps:\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: pnpm install --frozen-lockfile\n - uses: ${TEARDOWN}\n - run: pnpm publish --no-git-checks\n`, + ), + ); + assert.equal(restored.jobs[0].status, "protected"); + + const unsafe = classify( + workflow( + ` release:\n steps:\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: pnpm install --frozen-lockfile\n - run: pnpm publish --no-git-checks\n`, + ), + ); + assert.equal(unsafe.jobs[0].status, "unsafe-publish"); + assert.equal(repositoryDisposition([unsafe]), "unsafe-publish"); + + const teardownBeforeFinalDownload = classify( + workflow( + ` release:\n steps:\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - uses: ${TEARDOWN}\n - run: pnpm install --frozen-lockfile\n - run: pnpm publish --no-git-checks\n`, + ), + ); + assert.equal(teardownBeforeFinalDownload.jobs[0].status, "unsafe-publish"); +}); + +test("classifier: yarn installs block", () => { + const result = classify( + workflow(` build:\n steps:\n - run: yarn install\n`), + ); + assert.equal(result.jobs[0].status, "blocked-yarn"); + assert.equal(repositoryDisposition([result]), "blocked-yarn"); +}); + +test("classifier: public privileged trigger with installs is unsafe", () => { + const result = classify( + workflow( + ` build:\n steps:\n - uses: actions/checkout@sha\n with: { ref: "\${{ github.event.pull_request.head.sha }}" }\n - run: npm ci\n`, + "pull_request_target", + ), + "public", + ); + assert.equal(result.jobs[0].status, "unsafe-trust"); + assert.equal(repositoryDisposition([result]), "blocked-trust"); +}); + +test("classifier: reusable call with inherited secrets is flagged", () => { + const result = classify( + `on: push\njobs:\n call:\n uses: workos/shared/.github/workflows/ci.yml@main\n secrets: inherit\n`, + ); + assert.equal(result.jobs[0].status, "reusable-call"); + assert.match(result.jobs[0].violations.join(" "), /inherits all secrets/); + assert.equal(repositoryDisposition([result]), "needs-review"); +}); + +test("classifier: pnpm action-setup run_install downloads", () => { + const result = classify( + workflow( + ` build:\n steps:\n - uses: pnpm/action-setup@sha\n with: { run_install: true }\n`, + ), + ); + assert.equal(result.jobs[0].status, "unknown"); + assert.equal(repositoryDisposition([result]), "needs-review"); +}); + +test("classifier: Corepack pnpm bootstrap bypasses SFW npm configuration", () => { + const lazyCorepack = classify( + workflow( + ` build:\n steps:\n - run: corepack enable\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: pnpm install --frozen-lockfile\n`, + ), + ); + assert.equal(lazyCorepack.jobs[0].status, "unprotected"); + assert.match(lazyCorepack.jobs[0].violations.join(" "), /Corepack.*lazily/); + + const corepackDownload = classify( + workflow( + ` build:\n steps:\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: corepack prepare pnpm@11.20.0 --activate\n`, + ), + ); + assert.equal(corepackDownload.jobs[0].status, "unprotected"); + assert.match( + corepackDownload.jobs[0].violations.join(" "), + /Corepack package-manager downloads/, + ); + + const npmBootstrap = classify( + workflow( + ` build:\n steps:\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: npm install --global pnpm@11.20.0 --ignore-scripts\n - run: pnpm install --frozen-lockfile\n`, + ), + ); + assert.equal(npmBootstrap.jobs[0].status, "protected"); + assert.deepEqual(npmBootstrap.jobs[0].violations, []); +}); + +test("classifier: local composite actions resolve or block", () => { + const localActions = new Map([ + [ + ".github/actions/install/action.yml", + `runs:\n using: composite\n steps:\n - run: npm ci\n shell: bash\n`, + ], + ]); + const resolved = classify( + workflow(` build:\n steps:\n - uses: ./.github/actions/install\n`), + "private", + localActions, + ); + assert.equal(resolved.jobs[0].status, "unprotected"); + assert.equal(resolved.jobs[0].operations[0].via, "./.github/actions/install"); + + const missing = classify( + workflow(` build:\n steps:\n - uses: ./.github/actions/mystery\n`), + ); + assert.equal(missing.jobs[0].status, "unknown"); + assert.equal(repositoryDisposition([missing]), "needs-review"); +}); + +test("classifier: parse errors and scope buckets fail closed", () => { + const broken = classify("on: [push\njobs: {"); + assert.notEqual(broken.parseError, undefined); + assert.equal(repositoryDisposition([broken]), "needs-review"); + + const python = classify( + workflow( + ` build:\n steps:\n - run: pip install -r requirements.txt\n`, + ), + ); + assert.equal(repositoryDisposition([python]), "out-of-scope"); + + assert.equal(repositoryDisposition([]), "no-ci"); + assert.equal(repositoryDisposition([], { error: "boom" }), "audit-error"); +}); + +function stubClient(overrides = {}) { + const restRepositories = [ + { + archived: false, + default_branch: "main", + name: "app", + visibility: "internal", + }, + { + archived: false, + default_branch: "main", + name: "empty-repo", + visibility: "private", + }, + ]; + return { + api: async (endpoint) => { + if (endpoint === "repos/workos/empty-repo") { + return { size: 0 }; + } + throw new Error(`unexpected api call: ${endpoint}`); + }, + getRef: async (repository) => { + if (repository === "workos/empty-repo") { + const error = new Error("Not Found"); + error.status = 404; + throw error; + } + return { object: { sha: "c".repeat(40) } }; + }, + getText: async (repository, path) => { + assert.equal(path, ".github/workflows/ci.yml"); + return workflow(` build:\n steps:\n - run: npm ci\n`); + }, + getTree: async () => ({ + tree: [ + { path: ".github/workflows/ci.yml", type: "blob", mode: "100644" }, + { path: "package-lock.json", type: "blob", mode: "100644" }, + ], + truncated: false, + }), + listGraphqlRepositories: async () => + restRepositories.map((repository) => ({ + isArchived: repository.archived, + name: repository.name, + visibility: repository.visibility, + })), + listRestRepositories: async () => restRepositories, + ...overrides, + }; +} + +test("audit: classifies mocked repositories and writes an atomic report", async (t) => { + const report = await runAudit(stubClient()); + assert.equal(report.inventory.activeCount, 2); + assert.deepEqual(report.dispositions, { empty: 1, "needs-sfw": 1 }); + const app = report.repositories.find((row) => row.name === "app"); + assert.equal(app.disposition, "needs-sfw"); + assert.deepEqual(app.managers, ["npm"]); + assert.deepEqual(app.lockfiles, ["package-lock.json"]); + + const directory = await mkdtemp(join(tmpdir(), "sfw-audit-")); + t.after(() => rm(directory, { recursive: true, force: true })); + const reportPath = join(directory, "report.json"); + await writeReportAtomically(reportPath, report); + assert.equal((await stat(reportPath)).mode & 0o777, 0o600); + const written = JSON.parse(await readFile(reportPath, "utf8")); + assert.deepEqual(written.dispositions, report.dispositions); + assert.deepEqual(await readdir(directory), ["report.json"]); +}); + +test("audit: access failures become audit-error rows, never no-download", async () => { + const client = stubClient({ + getTree: async () => { + const error = new Error("Forbidden"); + error.status = 403; + throw error; + }, + }); + const row = await auditRepository(client, { + defaultBranch: "main", + name: "app", + visibility: "internal", + }); + assert.equal(row.disposition, "audit-error"); + assert.match(row.error, /Forbidden/); +}); + +test("audit: truncated trees fail closed", async () => { + const client = stubClient({ + getTree: async () => ({ tree: [], truncated: true }), + }); + const row = await auditRepository(client, { + defaultBranch: "main", + name: "app", + visibility: "internal", + }); + assert.equal(row.disposition, "audit-error"); + assert.match(row.error, /truncated/); +}); + +test("adapter: transient network failures retry bounded", async () => { + const { GitHubClient } = await import("./github.mjs"); + let attempts = 0; + const client = new GitHubClient({ + execute: async () => { + attempts += 1; + if (attempts < 3) { + throw new Error("dial tcp 140.82.113.6:443: i/o timeout"); + } + return { stdout: "{}" }; + }, + sleep: async () => {}, + }); + await client.api("repos/workos/example", "read example"); + assert.equal(attempts, 3); +}); + +test("adapter: non-transient failures do not retry", async () => { + const { GitHubClient } = await import("./github.mjs"); + let attempts = 0; + const client = new GitHubClient({ + execute: async () => { + attempts += 1; + const error = new Error("gh: Not Found (HTTP 404)"); + error.status = 404; + throw error; + }, + sleep: async () => {}, + }); + await assert.rejects(() => client.api("repos/workos/gone", "read gone")); + assert.equal(attempts, 1); +}); + +const setupStep = { uses: SETUP, with: { token: PRIVATE_TOKEN } }; +const installStep = { run: "npm ci" }; +const teardownStep = { uses: TEARDOWN }; +function jobResult(steps, extra = {}, context = {}) { + return classifyJob( + "fixture", + { steps, ...extra }, + { visibility: "private", ...context }, + ["push"], + ); +} + +test("classifier: every download needs an active, unconditional setup interval", () => { + const cases = [ + ["disabled", [{ ...setupStep, if: false }, installStep]], + [ + "disabled expression", + [{ ...setupStep, if: "${{ false }}" }, installStep], + ], + [ + "conditional", + [{ ...setupStep, if: "github.ref == 'refs/heads/main'" }, installStep], + ], + [ + "continue on error", + [{ ...setupStep, "continue-on-error": true }, installStep], + ], + [ + "dynamic continue on error", + [ + { ...setupStep, "continue-on-error": "${{ matrix.optional }}" }, + installStep, + ], + ], + ["teardown before install", [setupStep, teardownStep, installStep]], + [ + "teardown after first install", + [setupStep, installStep, teardownStep, installStep], + ], + [ + "conditional teardown", + [setupStep, { ...teardownStep, if: "always()" }, installStep], + ], + [ + "later registry action", + [ + setupStep, + installStep, + { + uses: "actions/setup-node@fixture", + with: { "registry-url": "https://registry.npmjs.org/" }, + }, + installStep, + ], + ], + [ + "later registry flag", + [ + setupStep, + installStep, + { run: "npm ci --registry=https://registry.npmjs.org/" }, + ], + ], + [ + "later registry assignment", + [ + setupStep, + installStep, + { run: "NPM_CONFIG_REGISTRY=https://registry.npmjs.org/ npm ci" }, + ], + ], + ["install condition", [setupStep, { ...installStep, if: "always()" }]], + [ + "install continue on error", + [setupStep, { ...installStep, "continue-on-error": true }], + ], + [ + "fallback path", + [ + { + ...setupStep, + with: { ...setupStep.with, "allow-external-fork-fallback": "true" }, + }, + installStep, + ], + ], + ["bun configuration missing", [setupStep, { run: "bun install" }]], + ]; + const ambiguous = new Set([ + "conditional", + "continue on error", + "dynamic continue on error", + "conditional teardown", + "install condition", + "install continue on error", + "fallback path", + ]); + for (const [label, steps] of cases) { + const result = jobResult(steps); + assert.equal( + result.status, + ambiguous.has(label) ? "unknown" : "unprotected", + label, + ); + assert.equal( + repositoryDisposition([{ jobs: [result] }]), + ambiguous.has(label) ? "needs-review" : "needs-sfw", + label, + ); + assert.ok(result.violations.length > 0, label); + } + assert.equal( + jobResult([setupStep, installStep, teardownStep]).status, + "protected", + ); + assert.equal( + jobResult([setupStep, installStep, teardownStep, setupStep, installStep]) + .status, + "protected", + ); + assert.equal( + jobResult([ + { ...setupStep, if: true, "continue-on-error": false }, + installStep, + ]).status, + "protected", + ); + assert.equal( + jobResult([ + { ...setupStep, with: { ...setupStep.with, "configure-bun": "true" } }, + { run: "bun install" }, + ]).status, + "protected", + ); +}); + +test("classifier: opaque operations cannot lose to an otherwise protected install", () => { + const opaque = [ + { run: "npm run bootstrap" }, + { run: "npm test" }, + { run: "pnpm exec custom" }, + { run: "bash -c 'npm ci'" }, + { run: "node scripts/bootstrap.js" }, + { run: "./scripts/bootstrap" }, + { run: "docker build ." }, + { run: "custom-tool" }, + { uses: "docker://fixture/image:latest" }, + { uses: "example/opaque-action@fixture" }, + { uses: "./missing" }, + { run: "npm config set registry https://registry.npmjs.org/" }, + ]; + for (const step of opaque) { + assert.equal(jobResult([step]).status, "unknown", JSON.stringify(step)); + const result = jobResult([setupStep, installStep, step]); + assert.notEqual(result.status, "protected", JSON.stringify(step)); + assert.ok(["unknown", "unprotected"].includes(result.status)); + } + for (const extra of [ + { container: "node:22" }, + { if: "always()" }, + { "continue-on-error": true }, + { env: { NPM_CONFIG_REGISTRY: "custom" } }, + { defaults: { run: { shell: "python" } } }, + ]) { + assert.equal(jobResult([setupStep, installStep], extra).status, "unknown"); + } +}); + +test("classifier: small shell grammar keeps complex constructs uncertain", () => { + for (const run of [ + "npm ci || true", + "npm ci; npm ci", + "if true; then npm ci; fi", + "npm ci $(custom)", + "npm ci > log", + "sudo npm ci", + 'npm ci "$(custom)"', + ]) { + assert.notEqual(jobResult([setupStep, { run }]).status, "protected", run); + } + assert.equal( + jobResult([setupStep, { run: "npm ci && pnpm install" }]).status, + "protected", + ); + assert.equal( + jobResult([setupStep, { run: "npm ci", shell: "python" }]).status, + "unknown", + ); +}); + +test("classifier: local composite nesting, cycles and inner/outer uncertainty", () => { + const localActions = new Map([ + [ + "outer/action.yml", + "runs:\n using: composite\n steps:\n - uses: ./inner\n", + ], + [ + "inner/action.yml", + `runs:\n using: composite\n steps:\n - uses: ${SETUP}\n with: { token: '${PRIVATE_TOKEN}' }\n - run: npm ci\n shell: bash\n`, + ], + ]); + const call = { uses: "./outer" }; + assert.equal(jobResult([call], {}, { localActions }).status, "protected"); + for (const boundary of [{ if: "always()" }, { "continue-on-error": true }]) { + assert.equal( + jobResult([{ ...call, ...boundary }], {}, { localActions }).status, + "unknown", + ); + } + localActions.set( + "inner/action.yml", + `runs:\n using: composite\n steps:\n - uses: ${SETUP}\n if: false\n with: { token: '${PRIVATE_TOKEN}' }\n - run: npm ci\n shell: bash\n`, + ); + assert.equal(jobResult([call], {}, { localActions }).status, "unprotected"); + localActions.set( + "inner/action.yml", + "runs:\n using: composite\n steps:\n - uses: ./outer\n", + ); + assert.equal(jobResult([call], {}, { localActions }).status, "unknown"); + localActions.set( + "inner/action.yml", + "runs: { using: node24, main: index.js }", + ); + assert.equal( + jobResult([setupStep, installStep, call], {}, { localActions }).status, + "unknown", + ); +}); + +test("classifier: malformed workflow/jobs and unknown operations remain review candidates", () => { + for (const text of [ + "{}", + "jobs: []", + "jobs: {}", + "jobs: wrong", + "on: push\njobs: { build: [] }", + "on: push\njobs: { build: {} }", + "on: push\njobs: { build: { steps: wrong } }", + "on: push\njobs: { build: { steps: [null] } }", + ]) { + assert.equal(repositoryDisposition([classify(text)]), "needs-review", text); + } + const result = classify( + workflow(" build:\n steps:\n - run: npm run bootstrap\n"), + ); + assert.equal(repositoryDisposition([result]), "needs-review"); +}); + +const fixtureRepository = { + name: "app", + defaultBranch: "main", + visibility: "private", +}; +const blob = (path) => ({ path, type: "blob", mode: "100644" }); + +test("audit: moving branch cannot change tree or nested source snapshot", async () => { + const sha = "d".repeat(40); + const reads = []; + const sources = new Map([ + [ + ".github/workflows/ci.yml", + workflow(" build:\n steps:\n - uses: ./outer\n"), + ], + [ + "outer/action.yml", + "runs:\n using: composite\n steps:\n - uses: ./inner\n", + ], + [ + "inner/action.yml", + "runs:\n using: composite\n steps:\n - run: npm ci\n shell: bash\n - uses: ./outer\n", + ], + ]); + const row = await auditRepository( + stubClient({ + getRef: async () => ({ object: { sha } }), + getTree: async (_repo, ref, recursive) => { + assert.equal(ref, sha); // The simulated branch now points elsewhere. + assert.equal(recursive, true); + return { truncated: false, tree: [...sources.keys()].map(blob) }; + }, + getText: async (_repo, path, ref) => { + assert.equal(ref, sha); + reads.push(path); + return sources.get(path); + }, + }), + fixtureRepository, + ); + assert.equal(row.headSha, sha); + // The certain install precedes the unresolved cycle; retain both signals. + assert.equal(row.disposition, "needs-sfw"); + assert.equal(row.assuranceDisposition, "needs-review"); + assert.deepEqual(reads.sort(), [...sources.keys()].sort()); + assert.ok( + row.workflows[0].jobs[0].operations.some( + (op) => op.kind === "unknown-local-action", + ), + ); +}); + +test("audit: malformed and partial reads never become clean", async () => { + const overrides = [ + { getRef: async () => ({ object: { sha: "z".repeat(40) } }) }, + ...[ + null, + {}, + { truncated: false }, + { truncated: false, tree: [null] }, + { truncated: false, tree: [{ type: "blob" }] }, + ].map((tree) => ({ getTree: async () => tree })), + ...[undefined, "", {}].map((text) => ({ getText: async () => text })), + { + getText: async () => { + throw new Error("fixture source read failed"); + }, + }, + { + getTree: async () => ({ + truncated: false, + tree: [{ ...blob(".github/workflows/ci.yml"), mode: "120000" }], + }), + }, + ]; + for (const override of overrides) { + assert.equal( + (await auditRepository(stubClient(override), fixtureRepository)) + .disposition, + "audit-error", + ); + } +}); + +test("adapter: reject partial/malformed base64, wrong sizes and invalid UTF-8", async () => { + const valid = { + type: "file", + encoding: "base64", + content: "bmFtZTogZml4dHVyZQo=", + size: 14, + }; + const responses = [ + null, + { ...valid, content: undefined }, + { ...valid, content: "%%%" }, + { ...valid, size: 15 }, + { ...valid, size: undefined }, + { ...valid, content: "/w==", size: 1 }, + ]; + for (const response of responses) { + const client = new GitHubClient({ + execute: async () => ({ stdout: JSON.stringify(response) }), + }); + await assert.rejects( + client.getText("example/fixture", "action.yml", "a".repeat(40)), + ); + } + const client = new GitHubClient({ + execute: async () => ({ stdout: JSON.stringify(valid) }), + }); + assert.equal( + await client.getText("example/fixture", "action.yml", "a".repeat(40)), + "name: fixture\n", + ); +}); + +test("CLI: partial scan errors are distinct from gaps and terminal output is sanitized", async (t) => { + const directory = await mkdtemp(join(tmpdir(), "sfw-cli-")); + t.after(() => rm(directory, { recursive: true, force: true })); + let output = ""; + const reportPath = join(directory, "report.json"); + const report = await main(["audit"], { + client: stubClient({ + getText: async () => { + throw new Error("private fixture detail"); + }, + }), + reportPath, + output: { + write: (text) => { + output += text; + }, + }, + }); + assert.equal(report.scanStatus, "partial"); + assert.equal(report.scanErrors, 1); + assert.equal(scanExitCode(report), 1); + const summary = JSON.parse(output); + assert.equal(summary.scanErrors, 1); + assert.deepEqual(summary.dispositions, { "audit-error": 1, empty: 1 }); + assert.doesNotMatch(output, /private fixture detail|empty-repo|"app"/); + assert.match(await readFile(reportPath, "utf8"), /private fixture detail/); + assert.equal((await stat(reportPath)).mode & 0o777, 0o600); + const gaps = await runAudit(stubClient()); + assert.equal(gaps.scanStatus, "complete"); + assert.equal(scanExitCode(gaps), 0); + const unknowns = await runAudit( + stubClient({ + getText: async () => + workflow(" build:\n steps:\n - run: npm $INSTALL_COMMAND\n"), + }), + ); + assert.equal(unknowns.dispositions["needs-review"], 1); + assert.equal(scanExitCode(unknowns), 0); +}); + +test("classifier: later Corepack activation and alternate checkout are not certified", () => { + const later = jobResult([ + setupStep, + { run: "pnpm install" }, + { run: "corepack enable" }, + { run: "pnpm install" }, + ]); + assert.equal(later.status, "unprotected"); + assert.match(later.violations.join(" "), /Corepack.*operation 4/); + for (const step of [ + { uses: "actions/setup-node/custom@fixture" }, + { uses: "actions/checkout@fixture", with: { ref: "other-branch" } }, + { + uses: "actions/checkout@fixture", + with: { repository: "example/alternate" }, + }, + ]) { + assert.equal(jobResult([step, setupStep, installStep]).status, "unknown"); + } +}); + +test("classifier: branching local composites have a bounded expansion", () => { + const localActions = new Map(); + for (let index = 0; index < 15; index += 1) { + localActions.set( + `action-${index}/action.yml`, + `runs:\n using: composite\n steps:\n - uses: ./action-${index + 1}\n - uses: ./action-${index + 1}\n`, + ); + } + const result = jobResult([{ uses: "./action-0" }], {}, { localActions }); + assert.equal(result.status, "unknown"); + assert.ok(result.operations.length < 1100); + assert.ok( + result.operations.some( + (op) => op.reason === "local action expansion limit reached", + ), + ); +}); + +test("adapter: source paths are escaped independently of immutable ref", async () => { + let endpoint; + const client = new GitHubClient({ + execute: async (args) => { + endpoint = args.at(-1); + return { + stdout: JSON.stringify({ + type: "file", + encoding: "base64", + content: "", + size: 0, + }), + }; + }, + }); + await client.getText( + "example/fixture", + ".github/workflows/ci#fixture.yml", + "e".repeat(40), + ); + assert.equal( + endpoint, + `repos/example/fixture/contents/.github/workflows/ci%23fixture.yml?ref=${"e".repeat(40)}`, + ); +}); + +test("classifier: composite inner boundaries survive expansion", () => { + for (const boundary of ["if: always()", "continue-on-error: true"]) { + const localActions = new Map([ + [ + "inner/action.yml", + `runs:\n using: composite\n steps:\n - uses: ${SETUP}\n ${boundary}\n with: { token: '${PRIVATE_TOKEN}' }\n - run: npm ci\n shell: bash\n`, + ], + ]); + const result = jobResult([{ uses: "./inner" }], {}, { localActions }); + assert.equal(result.status, "unknown", boundary); + assert.equal(result.operations[0].uncertain, true); + } + for (const boundary of [{ if: "always()" }, { "continue-on-error": true }]) { + assert.equal( + jobResult([ + setupStep, + installStep, + { ...teardownStep, ...boundary }, + { run: "npm publish" }, + ]).status, + boundary.if === "always()" ? "unknown" : "unsafe-publish", + ); + } +}); + +test("classifier: registry changes are evaluated at each download, not after the last one", () => { + const registryStep = { + uses: "actions/setup-node@fixture", + with: { "registry-url": "https://registry.npmjs.org/" }, + }; + assert.equal( + jobResult([setupStep, installStep, registryStep]).status, + "protected", + ); + assert.equal( + jobResult([setupStep, installStep, registryStep, installStep]).status, + "unprotected", + ); + assert.equal( + jobResult([setupStep, installStep, registryStep, setupStep, installStep]) + .status, + "protected", + ); +}); diff --git a/tools/rollout/cli.mjs b/tools/rollout/cli.mjs new file mode 100644 index 0000000..bb8ae7b --- /dev/null +++ b/tools/rollout/cli.mjs @@ -0,0 +1,89 @@ +#!/usr/bin/env node + +import { fileURLToPath, pathToFileURL } from "node:url"; + +import { GitHubClient } from "./github.mjs"; +import { captureRepositoryInventory } from "./inventory.mjs"; +import { ORGANIZATION } from "./constants.mjs"; +import { runAudit, writeReportAtomically } from "./audit.mjs"; +import { verifyActionRelease } from "./release.mjs"; + +const COMMANDS = new Set(["audit", "inventory", "verify-action"]); +const REPORT_URL = new URL("../../reports/live-audit.json", import.meta.url); +const INVENTORY_URL = new URL("../../reports/inventory.json", import.meta.url); + +function inventoryCounts(inventory) { + const { activeCount, archivedCount, totalCount, visibility } = inventory; + return { activeCount, archivedCount, totalCount, visibility }; +} + +export function scanExitCode(result) { + return result.scanErrors > 0 ? 1 : 0; +} + +export async function main(argv, options = {}) { + if (argv.length !== 1 || !COMMANDS.has(argv[0])) { + throw new Error("usage: cli.mjs "); + } + + const client = options.client ?? new GitHubClient(); + const output = options.output ?? process.stdout; + const command = argv[0]; + + if (command === "audit") { + const reportPath = options.reportPath ?? fileURLToPath(REPORT_URL); + const report = await runAudit(client, { + progress: (done, total) => { + if (done % 25 === 0 || done === total) { + process.stderr.write(`audited ${done}/${total} repositories\n`); + } + }, + }); + await writeReportAtomically(reportPath, report); + const summary = { + schemaVersion: report.schemaVersion, + dispositions: report.dispositions, + assuranceDispositions: report.assuranceDispositions, + runtimeVerification: report.runtimeVerification, + inventory: inventoryCounts(report.inventory), + scanErrors: report.scanErrors, + scanStatus: report.scanStatus, + coverage: report.coverage, + reportPath, + }; + output.write(`${JSON.stringify(summary, null, 2)}\n`); + return report; + } + + const result = + command === "verify-action" + ? await verifyActionRelease({ client }) + : await captureRepositoryInventory(client, ORGANIZATION); + if (command === "inventory") { + const reportPath = options.reportPath ?? fileURLToPath(INVENTORY_URL); + await writeReportAtomically(reportPath, result); + output.write( + `${JSON.stringify({ ...inventoryCounts(result), scanStatus: "complete", coverage: "token-visible repositories only", reportPath }, null, 2)}\n`, + ); + } else { + output.write(`${JSON.stringify(result, null, 2)}\n`); + } + return result; +} + +const invokedPath = process.argv[1] + ? pathToFileURL(process.argv[1]).href + : undefined; +if (import.meta.url === invokedPath) { + main(process.argv.slice(2)) + .then((result) => { + process.exitCode = scanExitCode(result); + }) + .catch(() => { + // API errors can contain private repository names or workflow source. + process.stderr.write( + `${JSON.stringify({ scanStatus: "failed", scanErrors: 1, error: "verifier failed; check command, access, API availability, and snapshot prerequisites privately" })}\n`, + ); + process.exitCode = 1; + }); +} diff --git a/tools/rollout/commands.mjs b/tools/rollout/commands.mjs new file mode 100644 index 0000000..8033ffb --- /dev/null +++ b/tools/rollout/commands.mjs @@ -0,0 +1,190 @@ +// Lexical boundaries only: no expansion, execution or control-flow evaluation. +// Keep quoted values and substitutions atomic at both command and word level. +function lex(source, words = false) { + const commands = []; + const substitutions = []; + const groups = []; + const backticks = []; + const heredocs = []; + let heredocError = false; + let command = ""; + let quote; + let escaped = false; + let conditional = false; + let limit = false; + let ambiguous = /<<|^\s*(?:function\s|[\w-]+\s*\(\s*\)\s*\{)/m.test(source); + const flush = () => { + if (command.trim()) { + if (commands.length < 1000) commands.push(command.trim()); + else limit = true; + } + command = ""; + }; + const capture = (text) => { + // Word analysis bounds substitutions per command, not across unrelated reads. + if (!words) return; + if (substitutions.length < 20) substitutions.push(text); + else limit = true; + }; + for (let index = 0; index < source.length; index += 1) { + const char = source[index]; + if (escaped) { + command += char === "\n" ? "" : `\\${char}`; + escaped = false; + continue; + } + if (char === "\\" && quote !== "'") { + escaped = true; + continue; + } + if (char === "`" && quote !== "'") { + if (quote === "`") { + const frame = backticks.pop(); + capture(source.slice(frame.start + 1, index)); + quote = frame.quote; + } else { + backticks.push({ start: index, quote }); + quote = "`"; + } + command += char; + continue; + } + if ( + char === "$" && + ["(", "{"].includes(source[index + 1]) && + !["'", "`"].includes(quote) + ) { + const open = source[index + 1]; + groups.push({ + close: open === "(" ? ")" : "}", + quote, + start: index, + capture: open === "(", + }); + quote = undefined; + command += `$${open}`; + index += 1; + continue; + } + if (char === "\n" && !quote && heredocs.length) { + let cursor = index + 1; + for (const { delimiter, tabs } of heredocs.splice(0)) { + const start = cursor; + let found = false; + while (cursor < source.length) { + const end = source.indexOf("\n", cursor); + const lineEnd = end === -1 ? source.length : end; + const line = source.slice(cursor, lineEnd); + if ((tabs ? line.replace(/^\t+/, "") : line) === delimiter) { + heredocError ||= source.slice(start, cursor).includes("${{"); + cursor = lineEnd; + found = true; + break; + } + cursor = lineEnd + 1; + } + heredocError ||= !found; + if (!words || groups.length) command += `\n${delimiter}`; + } + index = cursor - 1; + continue; + } + if (!quote && char === "<" && source[index - 1] !== "<") { + // Only literal cat data, not a heredoc executed by bash/sh or a pipeline. + const match = source + .slice(index) + .match(/^<<(-?)[ \t]*(['"])([A-Za-z_][A-Za-z0-9_]*)\2[ \t]*(?=\n|$)/); + if ( + match && + /(?:^|\$\()cat(?:[ \t]+>{1,2}[ \t]+[\w./-]+)?[ \t]*$/.test( + words ? source.slice(0, index) : command, + ) + ) + heredocs.push({ delimiter: match[3], tabs: match[1] === "-" }); + } + if (quote) { + command += char; + if (char === quote) quote = undefined; + continue; + } + if (["'", '"'].includes(char)) { + quote = char; + command += char; + continue; + } + if ( + char === "#" && + groups.at(-1)?.close !== "}" && + (!command || /\s$/.test(command)) + ) { + while (index < source.length && source[index] !== "\n") index += 1; + if (groups.length) command += "\n"; + else flush(); + continue; + } + if (char === "(") groups.push({ close: ")" }); + else if (groups.at(-1)?.close === char) { + const frame = groups.pop(); + if (frame.capture) capture(source.slice(frame.start + 2, index)); + quote = frame.quote; + } else if (char === ")" && !groups.length) ambiguous = true; + if (groups.length > 100) + return { + commands: [source], + substitutions: [], + ambiguous: true, + limit: true, + conditional, + }; + if (words && !groups.length && [">", "<"].includes(char)) { + flush(); + let operator = char; + while (source[index + 1] === char) operator += source[++index]; + commands.push(operator); + } else if ( + !groups.length && + (words ? /\s/.test(char) : ["\n", ";", "|", "&"].includes(char)) && + !(char === "&" && /[<>]$/.test(command)) + ) { + if (["&", "|"].includes(char) && source[index + 1] === char) + conditional = true; + flush(); + } else command += char; + } + if (escaped) command += "\\"; + flush(); + const lexError = Boolean( + quote || groups.length || escaped || heredocError || heredocs.length, + ); + return { + commands, + substitutions, + conditional, + lexError, + limit, + ambiguous: ambiguous || lexError || limit, + }; +} + +export const shellCommands = (source) => lex(source); +function wordValue(word) { + let quote; + let value = ""; + for (let index = 0; index < word.length; index += 1) { + const char = word[index]; + if (char === "\\" && quote !== "'") { + const next = word[index + 1]; + if (next !== undefined && (quote !== '"' || /[$`"\\\n]/.test(next))) { + if (next !== "\n") value += next; + index += 1; + } else value += char; + } else if (char === quote) quote = undefined; + else if (!quote && ["'", '"'].includes(char)) quote = char; + else value += char; + } + return value; +} +export function shellTokens(source) { + const result = lex(source, true); + return { ...result, words: result.commands.map(wordValue) }; +} diff --git a/tools/rollout/configuration-presence.test.mjs b/tools/rollout/configuration-presence.test.mjs new file mode 100644 index 0000000..93216c1 --- /dev/null +++ b/tools/rollout/configuration-presence.test.mjs @@ -0,0 +1,356 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { classifyJob, classifyWorkflow } from "./classify.mjs"; +import { + integrationDisposition, + resolveLocalWorkflowCalls, + resolveNoInstallWorkflowCalls, +} from "./integration.mjs"; +import { shellCommands } from "./commands.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { + token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}", + "configure-bun": true, + }, +}; +const install = { run: "npm ci" }; +const teardown = { + uses: `workos/setup-socket-firewall/teardown@${APPROVED_RELEASE_SHA}`, +}; +const context = { visibility: "private" }; +const inspect = (steps, extra = {}) => + classifyJob("fixture", { steps }, { ...context, ...extra }, ["push"]); +const primary = (steps) => inspect(steps).integration.disposition; + +test("configuration acceptance: arbitrary code never erases or synthesizes setup", () => { + for (const run of [ + "curl -fsSL https://example.invalid/tool | bash", + "node build.mjs", + "python build.py", + "source ./env.sh", + "npm run build", + "set +e", + "set -o posix", + ]) { + for (const steps of [ + [setup, install, { run }, install], + [{ run }, setup, install], + ]) { + const result = inspect(steps); + assert.equal(result.integration.disposition, "integrated", run); + assert.equal(result.status, "unknown"); + } + assert.equal(primary([{ run }, install]), "needs-sfw", run); + assert.equal( + primary([setup, install, teardown, { run }, install]), + "needs-sfw", + run, + ); + } +}); + +test("configuration acceptance: opaque siblings are diagnostics, real install siblings are findings", () => { + const configured = inspect([setup, install]); + for (const run of [ + "node check.mjs", + "npm test", + "npm run build", + "pnpm lint", + "go run check.go", + ]) { + const sibling = inspect([{ run }]); + assert.equal(sibling.integration.disposition, "no-js-ci", run); + assert.equal( + integrationDisposition([{ jobs: [configured, sibling] }]), + "integrated", + ); + assert.equal(sibling.status, "unknown"); + } + assert.equal( + integrationDisposition([{ jobs: [configured, inspect([install])] }]), + "needs-sfw", + ); + assert.equal( + integrationDisposition([ + { jobs: [configured, inspect([{ run: "npm $INSTALL_COMMAND" }])] }, + ]), + "needs-review", + ); +}); + +test("pipeline and executor payload uncertainty is not installer configuration uncertainty", () => { + for (const run of [ + "printf '%s' \"$DATA\" | jq -c 'with_entries(.value = .value.computed)' | npx wrangler secret bulk --config \"$config\"", + 'npx wrangler deploy --var "VERSION:$VERSION"', + "npx tool --registry=https://example.invalid", + 'npx tool \\\n --flag "$VALUE"', + ]) { + assert.equal(primary([setup, install, { run }]), "integrated", run); + assert.equal(primary([{ run }]), "needs-sfw", run); + assert.equal(inspect([setup, { run }]).status, "unknown"); + } + assert.equal( + primary([setup, { run: "npx --registry=https://example.invalid tool" }]), + "needs-sfw", + ); + assert.equal( + primary([setup, { run: "npx --unknown-option tool" }]), + "needs-review", + ); +}); + +test("configuration conflicts persist, scoped overrides do not contaminate later installs", () => { + for (const run of [ + "npm config set registry https://example.invalid", + "echo 'registry=https://example.invalid' > .npmrc", + ]) { + assert.equal(primary([setup, { run }, install]), "needs-sfw"); + } + assert.equal( + primary([ + setup, + { + run: "npm config set registry https://example.invalid", + if: "inputs.override", + }, + install, + ]), + "needs-review", + ); + const result = inspect([ + setup, + { run: "npm ci --registry=https://example.invalid" }, + install, + ]); + assert.deepEqual( + result.integration.downloads.map((path) => path.status), + ["gap", "covered"], + ); + assert.equal( + primary([setup, { run: 'test -n "${NPM_CONFIG_USERCONFIG:-}"' }, install]), + "integrated", + ); + assert.equal( + primary([setup, { run: "export HOME=/tmp\nnpm ci" }]), + "needs-review", + ); + assert.equal( + primary([setup, { run: "export HOME=/tmp" }, install]), + "integrated", + ); + assert.equal(primary([setup, { run: "HOME=/tmp npm ci" }]), "needs-review"); +}); + +test("lexical boundaries do not turn quoted data into installs", () => { + for (const run of [ + "echo 'npm ci; npm ci'", + 'printf "%s" "example: npm ci; npm install"', + "echo '${{ inputs.payload }}; npm ci; npm install'", + "echo '$(npm ci)'", + ]) { + assert.equal(inspect([{ run }]).integration.downloads.length, 0, run); + assert.notEqual(primary([{ run }]), "needs-sfw", run); + } + assert.deepEqual(shellCommands('echo "a;b|c" | npx tool # npm ci').commands, [ + 'echo "a;b|c"', + "npx tool", + ]); + assert.deepEqual( + shellCommands( + 'VALUE="$(node --print \'require("./package.json").name\')"\nnpm ci', + ).commands, + ['VALUE="$(node --print \'require("./package.json").name\')"', "npm ci"], + ); + assert.equal(shellCommands("cat <<'EOF'\nnpm ci\nEOF").ambiguous, true); + assert.equal(primary([{ run: "cat <<'EOF'\nnpm ci\nEOF" }]), "no-js-ci"); +}); + +test("unparsed explicit JS invocation retains setup evidence, not execution assurance", () => { + for (const run of [ + "RESULT=$(npx tool --output json)", + "npm pack package-name", + ]) { + const result = inspect([setup, { run }]); + assert.equal(result.integration.disposition, "integrated", run); + assert.equal( + result.integration.additionalJsPaths[0].status, + "setup-observed", + ); + assert.equal(result.status, "unknown"); + assert.equal(primary([{ run }]), "needs-review"); + } + for (const run of [ + "uv run npm ci", + "env -i npm ci", + "npm --future-option ci", + "npm $COMMAND", + ]) + assert.equal(primary([setup, { run }]), "needs-review", run); +}); + +const workflow = (path, jobs) => + classifyWorkflow(JSON.stringify({ on: ["workflow_call"], jobs }), { + ...context, + path, + }); +test("local reusable calls use the already-read snapshot and preserve gap precedence", () => { + for (const steps of [ + [setup, install], + [install], + [setup, { run: "npm --future-option ci" }], + ]) { + const target = workflow(".github/workflows/shared.yml", { + build: { steps }, + }); + const caller = workflow(".github/workflows/ci.yml", { + call: { uses: "./.github/workflows/shared.yml" }, + own: { steps: [setup, install] }, + }); + const resolved = resolveLocalWorkflowCalls([caller, target]); + assert.equal( + resolved[0].jobs[0].integration.disposition, + target.jobs[0].integration.disposition, + ); + assert.equal(resolved[0].jobs[0].status, "reusable-call"); + assert.equal( + resolved[0].jobs[0].integration.referencedWorkflow, + target.path, + ); + } +}); + +test("missing, remote, cyclic and malformed reusable sources stay unresolved", () => { + const missing = workflow(".github/workflows/missing.yml", { + call: { uses: "./.github/workflows/absent.yml" }, + }); + const a = workflow(".github/workflows/a.yml", { + call: { uses: "./.github/workflows/b.yml" }, + }); + const b = workflow(".github/workflows/b.yml", { + call: { uses: "./.github/workflows/a.yml" }, + }); + const remote = workflow(".github/workflows/remote.yml", { + call: { uses: "example/shared/.github/workflows/ci.yml@main" }, + }); + const invalid = workflow(".github/workflows/invalid.yml", { + call: { uses: "./.github/workflows/leaf.yml", steps: [install] }, + }); + const leaf = workflow(".github/workflows/leaf.yml", { + build: { steps: [setup, install] }, + }); + for (const result of resolveLocalWorkflowCalls([ + missing, + a, + b, + remote, + invalid, + leaf, + ]).slice(0, -1)) + assert.equal(integrationDisposition([result]), "needs-review"); + const chain = Array.from({ length: 22 }, (_, n) => + workflow( + `.github/workflows/depth-${n}.yml`, + n === 21 + ? { build: { steps: [setup, install] } } + : { call: { uses: `./.github/workflows/depth-${n + 1}.yml` } }, + ), + ); + assert.equal( + integrationDisposition([resolveLocalWorkflowCalls(chain)[0]]), + "integrated", + ); +}); + +test("remote no-install calls resolve only against an exactly captured ref and source", () => { + const sha = "a".repeat(40); + const body = workflow(".github/workflows/guard.yml", { + guard: { steps: [{ run: "git diff --exit-code" }] }, + }); + const target = { + name: "shared", + defaultBranch: "main", + headSha: sha, + disposition: "no-js-ci", + workflows: [body], + }; + const caller = (ref, owner = "example") => ({ + name: "app", + disposition: "needs-review", + assuranceDisposition: "needs-review", + workflows: [ + workflow(".github/workflows/ci.yml", { + install: { steps: [setup, install] }, + guard: { uses: `${owner}/shared/.github/workflows/guard.yml@${ref}` }, + }), + ], + }); + for (const ref of [sha, "main"]) { + const result = resolveNoInstallWorkflowCalls( + [caller(ref), target], + "example", + )[0]; + assert.equal(result.disposition, "integrated"); + assert.equal(result.assuranceDisposition, "needs-review"); + assert.equal( + result.workflows[0].jobs.find((job) => job.job === "guard").integration + .referencedHeadSha, + sha, + ); + } + for (const input of [ + [caller("b".repeat(40)), target], + [caller("v1"), target], + [caller("main", "other"), target], + [caller("main")], + [caller("main"), { ...target, disposition: "audit-error" }], + [ + caller("main"), + { + ...target, + workflows: [ + workflow(body.path, { build: { steps: [setup, install] } }), + ], + }, + ], + ]) { + assert.equal( + resolveNoInstallWorkflowCalls(input, "example")[0].disposition, + "needs-review", + ); + } +}); + +test("whole-value composite inputs preserve actual caller configuration", () => { + const localActions = new Map([ + [ + "helper/action.yml", + JSON.stringify({ + inputs: { token: {} }, + runs: { + using: "composite", + steps: [ + { ...setup, with: { token: "${{ inputs.token }}" } }, + { ...install, shell: "bash" }, + ], + }, + }), + ], + ]); + const call = (token) => + inspect([{ uses: "./helper", with: { token } }], { localActions }); + assert.equal( + call("${{ secrets.SOCKET_FIREWALL_TOKEN }}").integration.disposition, + "integrated", + ); + assert.equal(call("wrong").integration.disposition, "needs-sfw"); + assert.equal( + call("${{ secrets[inputs.name] }}").integration.disposition, + "needs-review", + ); + assert.equal( + inspect([{ uses: "./helper" }], { localActions }).integration.disposition, + "needs-sfw", + ); +}); diff --git a/tools/rollout/configuration-regressions.test.mjs b/tools/rollout/configuration-regressions.test.mjs new file mode 100644 index 0000000..15edcf3 --- /dev/null +++ b/tools/rollout/configuration-regressions.test.mjs @@ -0,0 +1,348 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { classifyJob } from "./classify.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { + token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}", + "configure-bun": true, + }, +}; +const install = { run: "npm ci" }; +const inspect = (steps, extra = {}, job = {}) => + classifyJob( + "fixture", + { steps, ...job }, + { visibility: "private", ...extra }, + ["push"], + ); +const primary = (steps) => inspect(steps).integration.disposition; + +test("quoted assignments and config text never fabricate installs or mutations", () => { + for (const run of [ + 'NOTE="example npm ci "', + "NOTE='example npm ci '", + "NOTE=${VALUE:-example npm ci }", + "NOTE=$(printf '%s' 'npm ci')", + "VERSION=$(bun --version)", + "printf '%s' 'npm config set registry https://example.invalid'", + "printf '%s' 'HOME=/tmp > $GITHUB_ENV'", + "printf '%s' 'registry=https://example.invalid' '>' '.npmrc'", + "printf '%s' 'HOME=/tmp' '>' '$GITHUB_ENV'", + "echo 'HOME=/tmp' > '$GITHUB_ENV'", + ]) { + assert.equal(primary([{ run }]), "no-js-ci", run); + assert.equal(primary([setup, { run }, install]), "integrated", run); + } + assert.equal( + primary([ + setup, + { run: "npx tool 'npm config set registry https://example.invalid'" }, + install, + ]), + "integrated", + ); + assert.equal( + primary([ + setup, + { run: "RESULT=$(npx tool --registry=https://example.invalid)" }, + install, + ]), + "integrated", + ); +}); + +test("transient overrides on unparsed and nested JS installers remain findings", () => { + for (const run of [ + "npm --registry=https://example.invalid ci", + "pnpm --registry=https://example.invalid install", + "RESULT=$(npm --registry=https://example.invalid ci)", + "RESULT=$(env -i npm ci)", + "RESULT=$(HOME=/tmp npm ci)", + ]) { + const result = inspect([setup, install, { run }, install]); + assert.equal(result.integration.disposition, "needs-review", run); + assert.equal(result.integration.downloads.at(-1).status, "covered"); + } +}); + +test("standalone relevant assignments affect the same run, not later run blocks", () => { + for (const key of [ + "HOME", + "NPM_CONFIG_REGISTRY", + "NPM_CONFIG_USERCONFIG", + "PATH", + ]) { + assert.equal( + primary([setup, { run: `${key}=/tmp\nnpm ci` }]), + "needs-review", + key, + ); + assert.equal( + primary([setup, { run: `${key}=/tmp` }, install]), + "integrated", + key, + ); + } + assert.equal( + primary([setup, { run: 'NOTE="HOME=/tmp"\nnpm ci' }]), + "integrated", + ); + assert.equal( + primary([setup, { run: 'export NOTE="HOME=/tmp"\nnpm ci' }]), + "integrated", + ); +}); + +test("explicit persistent environment-file changes reach later installs", () => { + for (const run of [ + 'echo "HOME=/tmp" >> "$GITHUB_ENV"', + 'echo "NPM_CONFIG_USERCONFIG=/tmp/other" >> "${GITHUB_ENV:?}"', + 'echo "/tmp/bin" >> "$GITHUB_PATH"', + 'RESULT=$(echo "HOME=/tmp" >> "$GITHUB_ENV")', + ]) + assert.equal(primary([setup, { run }, install]), "needs-review", run); + assert.equal( + primary([ + setup, + { run: 'echo "APP_MODE=production" >> "$GITHUB_ENV"' }, + install, + ]), + "integrated", + ); + assert.equal( + primary([setup, { run: 'cat "$GITHUB_ENV"' }, install]), + "integrated", + ); + assert.equal( + inspect( + [setup, install], + {}, + { env: { GITHUB_ENV: "/tmp/not-the-runner-env" } }, + ).integration.disposition, + "needs-review", + ); +}); + +test("package-manager and file registry writes remain configuration conflicts", () => { + for (const run of [ + "pnpm config set @example:registry https://example.invalid", + "npm config set registry https://example.invalid", + ]) + assert.equal(primary([setup, { run }, install]), "needs-sfw", run); + for (const run of [ + "echo 'registry=https://example.invalid' | tee .npmrc", + 'tee "$NPM_CONFIG_USERCONFIG" < input.txt', + "RESULT=$(npm config set registry https://example.invalid)", + "npm config set registry $REGISTRY", + "npm config set $KEY $VALUE", + ]) + assert.equal(primary([setup, { run }, install]), "needs-review", run); + const bunWrite = { + run: "printf \"[install]\\nregistry='https://example.invalid'\\n\" > bunfig.toml", + }; + assert.notEqual( + primary([setup, bunWrite, { run: "bun install" }]), + "integrated", + ); + assert.equal(primary([setup, bunWrite, install]), "integrated"); +}); + +test("explicit config-file mutations are distinct from reading or copying a source", () => { + for (const run of [ + 'rm "$NPM_CONFIG_USERCONFIG"', + "mv .npmrc old-config", + "sed -i 's/registry/other/' .npmrc", + 'cp input.txt "$NPM_CONFIG_USERCONFIG"', + ]) + assert.equal(primary([setup, { run }, install]), "needs-review", run); + for (const run of [ + 'cat "$NPM_CONFIG_USERCONFIG"', + 'cp "$NPM_CONFIG_USERCONFIG" artifact.txt', + ]) + assert.equal(primary([setup, { run }, install]), "integrated", run); +}); + +test("composite condition references are not replaced with a different expression type", () => { + const localActions = new Map([ + [ + "helper/action.yml", + JSON.stringify({ + inputs: { enabled: { default: "false" } }, + runs: { + using: "composite", + steps: [ + { if: "${{ inputs.enabled }}", run: "npm ci", shell: "bash" }, + ], + }, + }), + ], + ]); + // Composite inputs are strings: the nonempty string "false" is not the + // literal boolean expression false. The install must not disappear. + const result = inspect([{ uses: "./helper" }], { localActions }); + assert.notEqual(result.integration.disposition, "no-js-ci"); + assert.equal(result.integration.downloads.length, 1); +}); + +test("the action-exported Bun config path is recognized without trusting overrides", () => { + for (const flag of [ + '--config="${SFW_BUN_CONFIG_PATH:?}"', + '--config "$SFW_BUN_CONFIG_PATH"', + "--config=$SFW_BUN_CONFIG_PATH", + ]) { + const run = `bun install --frozen-lockfile ${flag}`; + assert.equal(primary([setup, { run }]), "integrated"); + assert.equal( + primary([ + { ...setup, with: { ...setup.with, "configure-bun": false } }, + { run }, + ]), + "needs-sfw", + ); + assert.equal( + primary([setup, { run, env: { SFW_BUN_CONFIG_PATH: "/tmp/other" } }]), + "needs-review", + ); + } + for (const flag of [ + "--config='$SFW_BUN_CONFIG_PATH'", + '--config="${SFW_BUN_CONFIG_PATH:-other}"', + "--config=./custom.toml", + ]) + assert.equal( + primary([setup, { run: `bun install ${flag}` }]), + "needs-review", + ); +}); + +test("nested JS paths retain manager-specific configuration requirements", () => { + const npmOnly = { ...setup, with: { ...setup.with, "configure-bun": false } }; + for (const run of ["RESULT=$(bun install)", "RESULT=$(uv run bun install)"]) + assert.equal(primary([npmOnly, { run }]), "needs-review"); + assert.equal( + primary([npmOnly, { run: "RESULT=$(npx tool --argument bun)" }]), + "integrated", + ); + assert.equal( + primary([setup, { run: "RESULT=$(yarn install)" }]), + "needs-review", + ); + assert.equal( + primary([ + { run: "corepack enable" }, + setup, + { run: "RESULT=$(pnpm install)" }, + ]), + "needs-review", + ); + assert.equal( + primary([setup, { run: "RESULT=$(corepack prepare pnpm@10)" }]), + "needs-review", + ); + assert.equal( + primary([ + setup, + { run: "RESULT=$(corepack enable)" }, + { run: "pnpm install" }, + ]), + "needs-review", + ); +}); + +test("explicit nested installs survive outer script and mutation roles", () => { + for (const run of [ + 'npm run build "$(npm ci)"', + "RESULT=$(npm ci; npm config set registry https://example.invalid)", + ]) { + assert.equal(primary([{ run }]), "needs-review"); + assert.equal(primary([{ run }, setup, install]), "needs-review"); + } + assert.equal( + primary([setup, { run: 'npm run build "$(npm ci)"' }]), + "integrated", + ); + assert.equal( + primary([{ run: 'npm run build "$(printf npm)"' }, setup, install]), + "integrated", + ); +}); + +test("configuration-file mutations remain visible inside substitutions", () => { + for (const run of [ + 'RESULT=$(echo "@example:registry=https://example.invalid" > .npmrc)', + 'RESULT=$(rm "$NPM_CONFIG_USERCONFIG")', + "RESULT=$(tee .npmrc < input.txt)", + ]) + assert.equal(primary([setup, { run }, install]), "needs-review", run); + assert.equal( + primary([ + setup, + { run: "RESULT=$(printf '%s' 'registry=example' '>' '.npmrc')" }, + install, + ]), + "integrated", + ); +}); + +test("unmodeled direct JS launchers do not inherit guessed configuration", () => { + for (const run of [ + "uv run bun install", + "uv run npx --registry=https://example.invalid tool", + "uv run npm config set registry https://example.invalid", + 'bash -c "bun install"', + ]) { + assert.equal(primary([setup, { run }, install]), "needs-review", run); + assert.equal( + primary([setup, { run: `RESULT=$(${run})` }, install]), + "needs-review", + run, + ); + } +}); + +test("literal word quoting and escapes do not hide registry options", () => { + for (const run of [ + 'n"pm" ci --registry=https://example.invalid', + 'npm ci --reg""istry=https://example.invalid', + String.raw`npm ci \--registry=https://example.invalid`, + ]) + assert.equal(primary([setup, { run }]), "needs-sfw", run); + assert.equal( + primary([setup, { run: "npm ci --registration=unused" }]), + "needs-review", + ); + assert.equal( + primary([ + setup, + { run: 'npm run build -- --registry=example "$(npm ci)"' }, + ]), + "integrated", + ); +}); + +test("line continuations remove the newline without inventing a word boundary", () => { + assert.equal(primary([{ run: "NOTE=hello\\\nnpm ci" }]), "no-js-ci"); + assert.equal(primary([{ run: "n\\\npm ci" }]), "needs-sfw"); + assert.equal(primary([setup, { run: "n\\\npm ci" }]), "integrated"); + assert.equal( + primary([setup, { run: "npm ci --reg\\\nistry=https://example.invalid" }]), + "needs-sfw", + ); +}); + +test("unresolved lexical structure or a lexical limit cannot disappear after a covered install", () => { + for (const run of [ + "NOTE='unterminated", + `NOTE=${"$(".repeat(105)}echo ok${")".repeat(105)}`, + ]) { + const result = inspect([setup, install, { run }]); + assert.equal(result.integration.disposition, "needs-review"); + assert.ok( + result.operations.some( + (operation) => operation.sourceError === "unresolved-shell-source", + ), + ); + } +}); diff --git a/tools/rollout/constants.mjs b/tools/rollout/constants.mjs new file mode 100644 index 0000000..8eb3d87 --- /dev/null +++ b/tools/rollout/constants.mjs @@ -0,0 +1,35 @@ +export const ORGANIZATION = "workos"; +export const ACTION_REPOSITORY = "workos/setup-socket-firewall"; +export const RELEASE_CHANNEL = "v1"; +export const RELEASE_BRANCH = `action-release/${RELEASE_CHANNEL}`; +export const APPROVED_RELEASE_SHA = "ca93dd8aa351f54f4729fe3377a9be23c631c25d"; + +// Runtime blobs read from the immutable approved release, not mutable discovery refs. +export const APPROVED_RUNTIME_BLOBS = Object.freeze({ + "action.yml": "156de46f25b2facc56cd7a5b2ea9e78b3de4ea1d", + "scripts/configure.sh": "155d6aab883d58211a4bb3fbfa70b27a2d96dae2", + "scripts/teardown.sh": "a1ddc2f06a65c607f2af2ae0395bada78e236913", + "teardown/action.yml": "d47a0abeaaa0d6717e5ce9cdbc45c3356ea3726c", +}); + +export const EXPECTED_RELEASE_TREE = Object.freeze([ + Object.freeze({ mode: "100644", path: "LICENSE", type: "blob" }), + Object.freeze({ mode: "100644", path: "action.yml", type: "blob" }), + Object.freeze({ mode: "040000", path: "scripts", type: "tree" }), + Object.freeze({ + mode: "100755", + path: "scripts/configure.sh", + type: "blob", + }), + Object.freeze({ + mode: "100755", + path: "scripts/teardown.sh", + type: "blob", + }), + Object.freeze({ mode: "040000", path: "teardown", type: "tree" }), + Object.freeze({ + mode: "100644", + path: "teardown/action.yml", + type: "blob", + }), +]); diff --git a/tools/rollout/exclusions.mjs b/tools/rollout/exclusions.mjs new file mode 100644 index 0000000..9ea1c75 --- /dev/null +++ b/tools/rollout/exclusions.mjs @@ -0,0 +1,229 @@ +import { isDeepStrictEqual } from "node:util"; +import { parseYamlSource } from "./yaml.mjs"; +import { fingerprint } from "./fingerprint.mjs"; + +// Approved source exclusions, not repository exemptions. A changed archive, +// integrity pin, or additional non-npm source must be reviewed again. +export const REGISTRY_EXCLUSIONS = [ + "oagen", + "oagen-emitters", + "openapi-spec", +].map((repository) => ({ + id: `${repository}-tree-sitter-kotlin`, + repository, + lockfile: "package-lock.json", + packagePath: "node_modules/tree-sitter-kotlin", + version: "0.4.0", + resolved: + "https://github.com/fwcd/tree-sitter-kotlin/archive/f66d2908542e93c0204c6c241f794afe4e9cd5d1.tar.gz", + integrity: + "sha512-7pk1Tg/gXh+6hM4E0F2rPKeLyA/bNlYyqVu6T1Md/AV/vloakbvEZG0R4CYwUfVtgFAMZRbOtA8JBzX1SFatBA==", + workflows: [ + ".github/workflows/ci.yml", + ".github/workflows/lint.yml", + ".github/workflows/release.yml", + ], + reason: + "The pinned tree-sitter-kotlin GitHub archive intentionally downloads outside Socket Firewall; npm-registry dependencies still use SFW.", + approvedBy: "matt.peake@workos.com", + approvalRequestId: "24cb3145-70c0-471a-86b6-3f5d7b5860a0", + approvalUrl: + "https://tars.workos.tools/conversations/pi_ae1489a3e5ee424ebd2cac9260cbad13", + ...(repository === "openapi-spec" + ? { + approvalRequestId: "bf1c082e-4112-484a-8683-f36854138690", + // Record the project-config exception; do not waive any step overrides. + workflows: [], + projectNpmrc: [ + "omit-lockfile-registry-resolved=true", + "replace-registry-host=npmjs", + ], + registryOverrides: true, + registryOverridesApprovalRequestId: + "61c52b1b-8081-48fa-9c8d-e2462e931ef7", + } + : {}), +})); + +const workosTarball = + "https://codeload.github.com/fwcd/tree-sitter-kotlin/tar.gz/f66d2908542e93c0204c6c241f794afe4e9cd5d1"; +REGISTRY_EXCLUSIONS.push({ + ...REGISTRY_EXCLUSIONS[0], + id: "workos-tree-sitter-kotlin", + repository: "workos", + manifest: "common/config/rush/pnpm-config.json", + lockfile: "common/config/rush/pnpm-lock.yaml", + specifier: + "github:fwcd/tree-sitter-kotlin#f66d2908542e93c0204c6c241f794afe4e9cd5d1", + resolved: workosTarball, + packagePath: `tree-sitter-kotlin@${workosTarball}`, + workflows: [], + approvalRequestId: "61c52b1b-8081-48fa-9c8d-e2462e931ef7", +}); + +const mapping = (value) => + value && typeof value === "object" && !Array.isArray(value); +const registrySpec = (value) => + typeof value === "string" && + value.trim().length > 0 && + ![".", "..", "~"].includes(value) && + !/\.t(?:gz|ar(?:\.gz)?)$/i.test(value) && + /^(?:npm:(?:@[\w.-]+\/)?[\w.-]+@)?[\w.*^~|<>= +-]+$/.test(value); + +async function readWorkosSource(rule, readSource) { + // Rush JSONC: preserve quoted strings, replace comments with whitespace so + // malformed adjacent tokens cannot become valid JSON by concatenation. + const manifest = JSON.parse( + (await readSource(rule.manifest)).replace( + /"(?:\\.|[^"\\])*"|\/\*[\s\S]*?\*\/|\/\/[^\r\n]*/g, + (part) => (part.startsWith('"') ? part : " "), + ), + ); + const lock = parseYamlSource(await readSource(rule.lockfile)); + const entry = lock?.packages?.[rule.packagePath]; + const matched = + manifest?.globalOverrides?.["tree-sitter-kotlin"] === rule.specifier && + lock?.lockfileVersion === "9.0" && + isDeepStrictEqual(manifest.globalOverrides, lock.overrides) && + Object.entries(lock.overrides).every(([name, spec]) => + name === "tree-sitter-kotlin" + ? spec === rule.specifier + : registrySpec(spec), + ) && + mapping(lock.packages) && + entry?.version === rule.version && + isDeepStrictEqual(entry.resolution, { + gitHosted: true, + integrity: rule.integrity, + tarball: rule.resolved, + }) && + Object.entries(lock.packages).every(([path, item]) => { + if (path === rule.packagePath) return true; + const resolution = item?.resolution; + // Other locked sources must be registry packages or local vendored files, + // not another network exception. This does not certify workspace scripts. + return ( + mapping(resolution) && + typeof resolution.integrity === "string" && + Object.keys(resolution).every((key) => + ["integrity", "tarball"].includes(key), + ) && + (resolution.tarball === undefined || + (typeof resolution.tarball === "string" && + (resolution.tarball.startsWith("https://registry.npmjs.org/") || + /^file:(?!\/)[\w./@-]+\.tgz$/.test(resolution.tarball)))) + ); + }); + return [ + { + ...rule, + status: matched ? "matched" : "stale", + ...(matched + ? { + // Only validated registry-version values are irrelevant to SFW routing. + // Retain override names/alias targets, the exact Git pin and every other + // Rush setting, including lifecycle-script policy and package extensions. + configurationFingerprint: fingerprint({ + ...manifest, + globalOverrides: Object.fromEntries( + Object.entries(manifest.globalOverrides).map(([name, spec]) => [ + name, + name === "tree-sitter-kotlin" + ? spec + : `${spec.startsWith("npm:") ? spec.slice(0, spec.lastIndexOf("@") + 1) : ""}`, + ]), + ), + }), + } + : {}), + }, + ]; +} + +export async function readRegistryExclusions(repository, readSource) { + const rule = REGISTRY_EXCLUSIONS.find( + (entry) => entry.repository === repository, + ); + if (!rule) return []; + if (repository === "workos") return readWorkosSource(rule, readSource); + // Read errors are audit errors, not permission to apply an exclusion. + const lock = JSON.parse(await readSource(rule.lockfile)); + const manifest = JSON.parse(await readSource("package.json")); + const packages = lock?.packages; + const configMatches = + !rule.projectNpmrc || + isDeepStrictEqual( + (await readSource(".npmrc")) + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line && !/^[#;]/.test(line)) + .toSorted(), + rule.projectNpmrc.toSorted(), + ); + // npm install can change the lock: do not excuse an unrecorded manifest edit. + const manifestMatches = + manifest && + typeof manifest === "object" && + !Array.isArray(manifest) && + !manifest.workspaces && + (rule.registryOverrides + ? manifest.overrides === undefined || + (mapping(manifest.overrides) && + Object.values(manifest.overrides).every(registrySpec)) + : manifest.overrides === undefined) && + [ + "dependencies", + "devDependencies", + "optionalDependencies", + "peerDependencies", + "peerDependenciesMeta", + "bundledDependencies", + "bundleDependencies", + ].every((key) => + isDeepStrictEqual(manifest[key] ?? {}, packages?.[""]?.[key] ?? {}), + ); + const valid = + lock?.lockfileVersion === 3 && + manifestMatches && + configMatches && + packages && + typeof packages === "object" && + !Array.isArray(packages) && + Object.values(packages).every( + (entry) => + entry && + typeof entry === "object" && + !Array.isArray(entry) && + (entry.resolved === undefined || typeof entry.resolved === "string") && + [ + "dependencies", + "devDependencies", + "optionalDependencies", + "peerDependencies", + ].every( + (key) => + entry[key] === undefined || + (entry[key] && + typeof entry[key] === "object" && + !Array.isArray(entry[key]) && + Object.values(entry[key]).every( + (spec) => spec === rule.resolved || registrySpec(spec), + )), + ), + ); + const external = valid + ? Object.entries(packages).filter( + ([, entry]) => + entry.resolved !== undefined && + !entry.resolved.startsWith("https://registry.npmjs.org/"), + ) + : []; + const match = + valid && + external.length === 1 && + external[0][0] === rule.packagePath && + ["version", "resolved", "integrity"].every( + (key) => external[0][1][key] === rule[key], + ); + return [{ ...rule, status: match ? "matched" : "stale" }]; +} diff --git a/tools/rollout/exclusions.test.mjs b/tools/rollout/exclusions.test.mjs new file mode 100644 index 0000000..150824c --- /dev/null +++ b/tools/rollout/exclusions.test.mjs @@ -0,0 +1,314 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { stringify } from "yaml"; +import { auditRepository } from "./audit.mjs"; +import { REGISTRY_EXCLUSIONS } from "./exclusions.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +import { resolveNoInstallWorkflowCalls } from "./integration.mjs"; + +const rule = REGISTRY_EXCLUSIONS[0]; +const sha = "a".repeat(40); +const checkout = { uses: "actions/checkout@v7" }; +const dependencies = { ordinary: "1.0.0", "tree-sitter-kotlin": rule.resolved }; +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { token: "${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }}" }, +}; +const install = { + run: "npm install", + env: { NPM_CONFIG_REPLACE_REGISTRY_HOST: "npmjs" }, +}; +async function scan({ + repository = "oagen", + entry = {}, + extraPackages = {}, + jobs, + defaults, + path = rule.workflows[0], + failRead = false, + manifest = { dependencies }, + npmrc, + lockDependencies = dependencies, +} = {}) { + const lock = { + lockfileVersion: 3, + packages: { + "": { dependencies: lockDependencies }, + "node_modules/ordinary": { + version: "1.0.0", + resolved: "https://registry.npmjs.org/ordinary/-/ordinary-1.0.0.tgz", + }, + [rule.packagePath]: { + version: rule.version, + resolved: rule.resolved, + integrity: rule.integrity, + ...entry, + }, + ...extraPackages, + }, + }; + const sources = new Map([ + [ + path, + stringify({ + on: "push", + defaults, + jobs: jobs ?? { check: { steps: [checkout, setup, install] } }, + }), + ], + [rule.lockfile, JSON.stringify(lock)], + ["package.json", JSON.stringify(manifest)], + ...(npmrc === undefined ? [] : [[".npmrc", npmrc]]), + ]); + return auditRepository( + { + async getRef() { + return { object: { sha } }; + }, + async getTree() { + return { + truncated: false, + tree: [...sources.keys()].map((path) => ({ + path, + type: "blob", + mode: "100644", + })), + }; + }, + async getText(repo, path, ref) { + assert.equal(ref, sha); + assert.equal(repo, `workos/${repository}`); + if (!sources.has(path) || (failRead && path === rule.lockfile)) + throw new Error("source unavailable"); + return sources.get(path); + }, + }, + { name: repository, defaultBranch: "main", visibility: "public" }, + ); +} + +test("approved archives are reported separately from fully integrated repositories", async () => { + for (const repository of ["oagen", "oagen-emitters"]) { + const result = await scan({ repository }); + assert.equal(result.disposition, "integrated-with-exclusions"); + assert.equal(result.assuranceDisposition, "needs-review"); + assert.equal(result.exclusions[0].status, "matched"); + assert.equal( + result.exclusions[0].approvalRequestId, + "24cb3145-70c0-471a-86b6-3f5d7b5860a0", + ); + const download = result.workflows[0].jobs[0].integration.downloads[0]; + assert.equal(download.status, "covered-with-exclusion"); + assert.equal(download.exclusionId, `${repository}-tree-sitter-kotlin`); + assert.equal( + resolveNoInstallWorkflowCalls([result], "workos")[0].disposition, + result.disposition, + ); + } +}); + +test("OpenAPI records the approved archive without waiving other findings", async () => { + const overrides = { "js-yaml": "^5.4.1", lodash: "^4.17.23" }; + const npmrc = + "# Preserve the archive\nreplace-registry-host=npmjs\nomit-lockfile-registry-resolved=true\n"; + const steps = [checkout, setup, { run: "npm ci" }, { run: "npm test" }]; + const options = { + repository: "openapi-spec", + manifest: { dependencies, overrides }, + npmrc, + jobs: { check: { steps } }, + }; + const result = await scan(options); + assert.equal(result.exclusions[0].status, "matched"); + assert.equal( + result.exclusions[0].approvalRequestId, + "bf1c082e-4112-484a-8683-f36854138690", + ); + assert.deepEqual(result.exclusions[0].workflows, []); + assert.equal(result.disposition, "integrated-with-exclusions"); + assert.equal(result.assuranceDisposition, "needs-review"); + assert.equal( + ( + await scan({ + ...options, + jobs: { check: { steps: [...steps, { uses: "./missing" }] } }, + }) + ).disposition, + "needs-review", + ); + assert.equal( + ( + await scan({ + ...options, + jobs: { check: { steps }, uncovered: { steps: [{ run: "npm ci" }] } }, + }) + ).disposition, + "needs-sfw", + ); + assert.equal( + ( + await scan({ + ...options, + jobs: { check: { steps: [checkout, setup, install] } }, + }) + ).disposition, + "needs-review", + ); + for (const changes of [ + { npmrc: npmrc.replace("=npmjs", "=never") }, + { npmrc: `${npmrc}@other:registry=https://example.invalid/\n` }, + { manifest: { dependencies, overrides: { extra: "other/archive" } } }, + { manifest: { dependencies, overrides: { extra: "file:archive.tgz" } } }, + { manifest: { dependencies, overrides: { extra: "1.tgz" } } }, + { manifest: { dependencies, overrides: { extra: { nested: "1.0.0" } } } }, + { + manifest: { + dependencies, + overrides: { ...overrides, extra: "https://example.invalid/other.tgz" }, + }, + }, + { entry: { integrity: "sha512-changed" } }, + { entry: { resolved: "https://example.invalid/other.tgz" } }, + { + extraPackages: { + "node_modules/another": { + resolved: "https://example.invalid/other.tgz", + }, + }, + }, + ]) { + const changed = await scan({ ...options, ...changes }); + assert.equal(changed.exclusions[0].status, "stale"); + assert.equal(changed.disposition, "needs-review"); + } + assert.equal( + (await scan({ ...options, npmrc: undefined })).disposition, + "audit-error", + ); + for (const nextOverrides of [ + undefined, + { ...overrides, "js-yaml": "^5.4.2" }, + { ...overrides, lodash: "^5.0.0", another: "npm:@scope/alias@^2.0.0" }, + ]) { + const nextDependencies = { ...dependencies, ordinary: "^2.0.0" }; + const updated = await scan({ + ...options, + manifest: { dependencies: nextDependencies, overrides: nextOverrides }, + lockDependencies: nextDependencies, + extraPackages: { + "node_modules/ordinary": { + version: "2.0.0", + resolved: "https://registry.npmjs.org/ordinary/-/ordinary-2.0.0.tgz", + }, + }, + }); + assert.equal(updated.exclusions[0].status, "matched"); + assert.equal(updated.disposition, "integrated-with-exclusions"); + assert.equal( + updated.workflows[0].jobs[0].reviewFingerprint, + result.workflows[0].jobs[0].reviewFingerprint, + "registry-only dependency bumps do not invalidate reviewed jobs", + ); + } + // OpenAPI's registry-only override policy does not authorize other repos. + assert.equal( + (await scan({ manifest: { dependencies, overrides } })).exclusions[0] + .status, + "stale", + ); +}); + +test("changed pins and additional external sources invalidate the exclusion", async () => { + for (const entry of [ + { version: "0.5.0" }, + { resolved: rule.resolved.replace("f66d290", "aaaaaaa") }, + { integrity: "sha512-other" }, + ]) { + const result = await scan({ entry }); + assert.equal(result.exclusions[0].status, "stale"); + assert.equal(result.disposition, "needs-review"); + } + const result = await scan({ + extraPackages: { + "node_modules/another": { resolved: "https://example.invalid/other.tgz" }, + }, + }); + assert.equal(result.exclusions[0].status, "stale"); + assert.equal(result.disposition, "needs-review"); + assert.equal((await scan({ failRead: true })).disposition, "audit-error"); + for (const manifest of [ + { dependencies: { extra: "https://example.invalid/extra.tgz" } }, + { overrides: { extra: "https://example.invalid/extra.tgz" } }, + { workspaces: ["packages/*"] }, + ]) + assert.equal((await scan({ manifest })).exclusions[0].status, "stale"); + assert.equal( + ( + await scan({ + entry: { dependencies: { extra: "github:other/unapproved" } }, + }) + ).exclusions[0].status, + "stale", + ); +}); + +test("source exclusions cannot hide other gaps, configuration changes, or repositories", async () => { + const missing = await scan({ + jobs: { check: { steps: [checkout, install] } }, + }); + assert.equal(missing.disposition, "needs-sfw"); + const sibling = await scan({ + jobs: { + check: { steps: [checkout, setup, install] }, + uncovered: { steps: [{ run: "npm ci" }] }, + }, + }); + assert.equal(sibling.disposition, "needs-sfw"); + for (const changed of [ + { ...install, env: { ...install.env, HOME: "/elsewhere" } }, + { ...install, "working-directory": "other" }, + { ...install, run: "npm install --registry=https://example.invalid" }, + { ...install, env: { NPM_CONFIG_REPLACE_REGISTRY_HOST: "never" } }, + ]) + assert.equal( + (await scan({ jobs: { check: { steps: [checkout, setup, changed] } } })) + .disposition, + "needs-review", + ); + assert.equal( + (await scan({ path: ".github/workflows/new.yml" })).disposition, + "needs-review", + ); + for (const changedCheckout of [ + { ...checkout, with: { ref: "other" } }, + { ...checkout, with: { repository: "other/repo" } }, + { ...checkout, with: { path: "other" } }, + { ...checkout, if: false }, + ]) + assert.equal( + ( + await scan({ + jobs: { check: { steps: [changedCheckout, setup, install] } }, + }) + ).disposition, + "needs-review", + ); + const defaults = { run: { "working-directory": "other" } }; + assert.equal((await scan({ defaults })).disposition, "needs-review"); + assert.equal( + ( + await scan({ + jobs: { check: { defaults, steps: [checkout, setup, install] } }, + }) + ).disposition, + "needs-review", + ); + assert.equal( + (await scan({ jobs: { check: { steps: [setup, install, checkout] } } })) + .disposition, + "needs-review", + ); + const unrelated = await scan({ repository: "unapproved" }); + assert.deepEqual(unrelated.exclusions, []); + assert.equal(unrelated.disposition, "needs-review"); +}); diff --git a/tools/rollout/final-detector.test.mjs b/tools/rollout/final-detector.test.mjs new file mode 100644 index 0000000..f7ac332 --- /dev/null +++ b/tools/rollout/final-detector.test.mjs @@ -0,0 +1,115 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { classifyJob } from "./classify.mjs"; +import { shellCommands, shellTokens } from "./commands.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; + +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { + token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}", + "configure-bun": true, + }, +}; +const inspect = (steps, extra = {}) => + classifyJob("test", { steps, ...extra }, { visibility: "private" }); +const offline = `if ! bun install --help | grep -F -- '--offline ' >/dev/null; then + echo 'Offline support is required.' >&2 + exit 1 +fi +bun install --lockfile-only --offline --ignore-scripts --registry=https://registry.npmjs.org/`; + +test("Bun offline canonicalization requires its adjacent fail-closed capability guard", () => { + const result = inspect([ + setup, + { run: "bun install --frozen-lockfile" }, + { run: offline }, + ]); + assert.equal(result.integration.disposition, "integrated"); + assert.equal(result.integration.downloads.length, 1); + assert.equal(result.status, "unknown"); + assert.equal(inspect([{ run: offline }]).integration.disposition, "no-js-ci"); + for (const run of [ + offline.split("\n").at(-1), + offline.replace("exit 1", "exit 0"), + offline.replace("--offline --ignore-scripts", "--offline"), + offline.replace("fi\nbun", "fi\nexport PATH=/elsewhere\nbun"), + offline.replace("--offline '", "--offline'"), + offline.replace("'Offline support is required.'", '"$(npm ci)"'), + ]) { + assert.notEqual( + inspect([setup, { run }]).integration.disposition, + "integrated", + run, + ); + } + assert.notEqual( + inspect([setup, { run: offline, env: { PATH: "/elsewhere" } }]).integration + .disposition, + "integrated", + ); + assert.notEqual( + inspect([setup, { run: offline }], { env: { PATH: "/elsewhere" } }) + .integration.disposition, + "integrated", + ); + assert.notEqual( + inspect([setup, { run: offline, shell: "python" }]).integration.disposition, + "integrated", + ); + assert.equal( + inspect([ + setup, + { run: offline + "\nexport PATH=/other\n" + offline.split("\n").at(-1) }, + ]).integration.downloads.length, + 1, + ); +}); + +test("substitution budget is per command, not every independent read in a job", () => { + const run = Array.from( + { length: 30 }, + (_, i) => `value${i}=$(printf value)`, + ).join("\n"); + assert.equal(shellCommands(run).limit, false); + assert.equal(inspect([{ run }]).integration.disposition, "no-js-ci"); + // A single command exceeding the bound must not hide its last substitution. + const many = `echo ${"$(printf value) ".repeat(20)}$(npm ci)`; + assert.equal(shellTokens(many).limit, true); + assert.equal( + inspect([setup, { run: many }]).integration.disposition, + "needs-review", + ); +}); + +test("quoted heredoc bodies are data, including quotes and apparent installers", () => { + for (const delimiter of ["'EOF'", '"EOF"']) { + const run = `prompt=$(cat <<${delimiter}\nIt's a prompt: npm ci\n$(npm install fake)\n\`bun install\`\n((((\nEOF\n)\necho "$prompt"`; + assert.equal(shellCommands(run).lexError, false); + assert.equal(inspect([{ run }]).integration.disposition, "no-js-ci"); + assert.equal( + inspect([{ run: run + "\nnpm ci" }]).integration.disposition, + "needs-review", + ); + assert.equal( + inspect([setup, { run: run + "\nnpm ci" }]).integration.downloads.length, + 1, + ); + } + assert.equal( + inspect([{ run: "cat <<-'EOF'\n\tnpm ci\n\tEOF\n" }]).integration + .disposition, + "no-js-ci", + ); + for (const run of [ + "cat <<'EOF'\nunclosed", + "cat <<'EOF'\n${{ inputs.untrusted }}\nEOF", + "cat < + item && typeof item === "object" && !Array.isArray(item) + ? Object.fromEntries( + Object.entries(item).sort(([a], [b]) => + a < b ? -1 : a > b ? 1 : 0, + ), + ) + : item, + ) ?? "null" + ); +} + +export function fingerprint(value) { + return createHash("sha256").update(canonicalJson(value)).digest("hex"); +} + +export function fingerprintYaml(text) { + try { + return fingerprint(parseYamlSource(text)); + } catch { + return fingerprint(text); + } +} diff --git a/tools/rollout/fixtures/approved-release.json b/tools/rollout/fixtures/approved-release.json new file mode 100644 index 0000000..abd9993 --- /dev/null +++ b/tools/rollout/fixtures/approved-release.json @@ -0,0 +1,100 @@ +{ + "provenance": { + "repository": "workos/setup-socket-firewall", + "approvedSha": "ca93dd8aa351f54f4729fe3377a9be23c631c25d", + "capturedAt": "2026-09-15T19:58:45.254612+00:00", + "note": "Curated public API fields only. Discovery refs were newer at capture time; positive historical verification supplies synthetic refs to approvedSha, never production constants." + }, + "discovery": { + "branch": { + "ref": "refs/heads/action-release/v1", + "object": { + "sha": "195977cfa38ab4d70363af5403a106dbc54a61eb", + "type": "commit" + } + }, + "tag": { + "ref": "refs/tags/v1", + "object": { + "sha": "195977cfa38ab4d70363af5403a106dbc54a61eb", + "type": "commit" + } + } + }, + "commit": { + "sha": "ca93dd8aa351f54f4729fe3377a9be23c631c25d", + "tree": { + "sha": "5cdbe39b0edafee9457767134320d95c61d91a60" + }, + "verification": { + "verified": true, + "reason": "valid" + } + }, + "tree": { + "sha": "5cdbe39b0edafee9457767134320d95c61d91a60", + "truncated": false, + "tree": [ + { + "path": "LICENSE", + "mode": "100644", + "type": "blob", + "sha": "c0cf74558bfb8eb6ebf3f0f24fb5fc180583efde" + }, + { + "path": "action.yml", + "mode": "100644", + "type": "blob", + "sha": "156de46f25b2facc56cd7a5b2ea9e78b3de4ea1d" + }, + { + "path": "scripts", + "mode": "040000", + "type": "tree", + "sha": "4ad0c03537d69113174ad3365a27ccaa303d158b" + }, + { + "path": "scripts/configure.sh", + "mode": "100755", + "type": "blob", + "sha": "155d6aab883d58211a4bb3fbfa70b27a2d96dae2" + }, + { + "path": "scripts/teardown.sh", + "mode": "100755", + "type": "blob", + "sha": "a1ddc2f06a65c607f2af2ae0395bada78e236913" + }, + { + "path": "teardown", + "mode": "040000", + "type": "tree", + "sha": "02b1d5554d43317d3aaface6c52c44fe9c73e428" + }, + { + "path": "teardown/action.yml", + "mode": "100644", + "type": "blob", + "sha": "d47a0abeaaa0d6717e5ce9cdbc45c3356ea3726c" + } + ] + }, + "contents": { + "action.yml": { + "type": "file", + "path": "action.yml", + "size": 1708, + "sha": "156de46f25b2facc56cd7a5b2ea9e78b3de4ea1d", + "encoding": "base64", + "content": "bmFtZTogU2V0dXAgU29ja2V0IEZpcmV3YWxsCmRlc2NyaXB0aW9uOiA+LQog\nIFJvdXRlIHB1YmxpYyBucG0tY29tcGF0aWJsZSBkZXBlbmRlbmN5IGRvd25s\nb2FkcyB0aHJvdWdoIHRoZSBXb3JrT1MgU29ja2V0CiAgRmlyZXdhbGwgYW5k\nIEROUy1udWxsLXJvdXRlIGtub3duIHB1YmxpYyBKYXZhU2NyaXB0IHJlZ2lz\ndHJpZXMuCmlucHV0czoKICB0b2tlbjoKICAgIGRlc2NyaXB0aW9uOiA+LQog\nICAgICBTb2NrZXQgRmlyZXdhbGwgYXV0aCB0b2tlbi4gUGFzcyBzZWNyZXRz\nLlNPQ0tFVF9GSVJFV0FMTF9UT0tFTiBkaXJlY3RseQogICAgICB0byB0aGlz\nIGFjdGlvbjsgZG8gbm90IGV4cG9zZSBpdCBhdCB3b3JrZmxvdyBvciBqb2Ig\nc2NvcGUuCiAgICByZXF1aXJlZDogZmFsc2UKICAgIGRlZmF1bHQ6ICIiCiAg\nYWxsb3ctZXh0ZXJuYWwtZm9yay1mYWxsYmFjazoKICAgIGRlc2NyaXB0aW9u\nOiA+LQogICAgICBQZXJtaXQgcHVibGljLXJlZ2lzdHJ5IGZhbGxiYWNrIG9u\nbHkgZm9yIGFuIGV4dGVybmFsIHB1bGwgcmVxdWVzdCBhZ2FpbnN0CiAgICAg\nIGEgcHVibGljIHJlcG9zaXRvcnkuIFRoZSBhY3Rpb24gaW5kZXBlbmRlbnRs\neSB2YWxpZGF0ZXMgdGhlIGV2ZW50IGNvbnRleHQuCiAgICByZXF1aXJlZDog\nZmFsc2UKICAgIGRlZmF1bHQ6ICJmYWxzZSIKICBjb25maWd1cmUtYnVuOgog\nICAgZGVzY3JpcHRpb246ID4tCiAgICAgIENvbmZpZ3VyZSBCdW4ncyB1c2Vy\nLWxldmVsIHJlZ2lzdHJ5IHRva2VuIGZpbGUuIEVuYWJsZSBvbmx5IGZvciBC\ndW4KICAgICAgZGVwZW5kZW5jeS1pbnN0YWxsIGpvYnMuCiAgICByZXF1aXJl\nZDogZmFsc2UKICAgIGRlZmF1bHQ6ICJmYWxzZSIKb3V0cHV0czoKICBhY3Rp\ndmU6CiAgICBkZXNjcmlwdGlvbjogV2hldGhlciBTb2NrZXQgRmlyZXdhbGwg\ncHJvdGVjdGlvbiBpcyBhY3RpdmUuCiAgICB2YWx1ZTogJHt7IHN0ZXBzLmNv\nbmZpZ3VyZS5vdXRwdXRzLmFjdGl2ZSB9fQpydW5zOgogIHVzaW5nOiBjb21w\nb3NpdGUKICBzdGVwczoKICAgIC0gbmFtZTogQ29uZmlndXJlIFNvY2tldCBG\naXJld2FsbAogICAgICBpZDogY29uZmlndXJlCiAgICAgIHNoZWxsOiBiYXNo\nCiAgICAgIGVudjoKICAgICAgICBTRldfVE9LRU46ICR7eyBpbnB1dHMudG9r\nZW4gfX0KICAgICAgICBTRldfQUxMT1dfRVhURVJOQUxfRk9SS19GQUxMQkFD\nSzogJHt7IGlucHV0cy5hbGxvdy1leHRlcm5hbC1mb3JrLWZhbGxiYWNrIH19\nCiAgICAgICAgU0ZXX0NPTkZJR1VSRV9CVU46ICR7eyBpbnB1dHMuY29uZmln\ndXJlLWJ1biB9fQogICAgICAgIFNGV19FVkVOVF9OQU1FOiAke3sgZ2l0aHVi\nLmV2ZW50X25hbWUgfX0KICAgICAgICBTRldfUkVQT1NJVE9SWV9QUklWQVRF\nOiAke3sgZ2l0aHViLmV2ZW50LnJlcG9zaXRvcnkucHJpdmF0ZSB9fQogICAg\nICAgIFNGV19IRUFEX1JFUE9TSVRPUlk6ICR7eyBnaXRodWIuZXZlbnQucHVs\nbF9yZXF1ZXN0LmhlYWQucmVwby5mdWxsX25hbWUgfX0KICAgICAgICBTRldf\nQkFTRV9SRVBPU0lUT1JZOiAke3sgZ2l0aHViLmV2ZW50LnB1bGxfcmVxdWVz\ndC5iYXNlLnJlcG8uZnVsbF9uYW1lIH19CiAgICAgICAgIyBUZXN0LW9ubHkg\nb3ZlcnJpZGUgaXMgY2xlYXJlZCBmb3IgcmVhbCBhY3Rpb24gY29uc3VtZXJz\nLgogICAgICAgIFNGV19IT1NUU19GSUxFOiAiIgogICAgICBydW46IGJhc2gg\nIiRHSVRIVUJfQUNUSU9OX1BBVEgvc2NyaXB0cy9jb25maWd1cmUuc2giCg==\n" + }, + "teardown/action.yml": { + "type": "file", + "path": "teardown/action.yml", + "size": 642, + "sha": "d47a0abeaaa0d6717e5ce9cdbc45c3356ea3726c", + "encoding": "base64", + "content": "bmFtZTogVGVhcmRvd24gU29ja2V0IEZpcmV3YWxsCmRlc2NyaXB0aW9uOiA+\nLQogIFJlbW92ZSBvbmx5IFNvY2tldCBGaXJld2FsbC1vd25lZCBucG0gYW5k\nIEROUyBjb25maWd1cmF0aW9uLCB0aGVuIHJlc3RvcmUKICBwdWJsaWMgbnBt\nLWNvbXBhdGlibGUgcmVnaXN0cnkgYWNjZXNzIGJlZm9yZSBwYWNrYWdlIHB1\nYmxpY2F0aW9uLgpvdXRwdXRzOgogIGFjdGl2ZToKICAgIGRlc2NyaXB0aW9u\nOiBXaGV0aGVyIFNvY2tldCBGaXJld2FsbCBwcm90ZWN0aW9uIHJlbWFpbnMg\nYWN0aXZlLgogICAgdmFsdWU6ICR7eyBzdGVwcy50ZWFyZG93bi5vdXRwdXRz\nLmFjdGl2ZSB9fQpydW5zOgogIHVzaW5nOiBjb21wb3NpdGUKICBzdGVwczoK\nICAgIC0gbmFtZTogUmVzdG9yZSBwdWJsaWMgcGFja2FnZSByZWdpc3RyeQog\nICAgICBpZDogdGVhcmRvd24KICAgICAgc2hlbGw6IGJhc2gKICAgICAgZW52\nOgogICAgICAgICMgVGVzdC1vbmx5IG92ZXJyaWRlIGlzIGNsZWFyZWQgZm9y\nIHJlYWwgYWN0aW9uIGNvbnN1bWVycy4KICAgICAgICBTRldfSE9TVFNfRklM\nRTogIiIKICAgICAgICBTRldfUk9MTEJBQ0tfT05MWTogImZhbHNlIgogICAg\nICBydW46IGJhc2ggIiRHSVRIVUJfQUNUSU9OX1BBVEgvLi4vc2NyaXB0cy90\nZWFyZG93bi5zaCIK\n" + } + } +} diff --git a/tools/rollout/fixtures/release-manifest.txt b/tools/rollout/fixtures/release-manifest.txt new file mode 100644 index 0000000..5cebab8 --- /dev/null +++ b/tools/rollout/fixtures/release-manifest.txt @@ -0,0 +1,5 @@ +LICENSE +action.yml +scripts/configure.sh +scripts/teardown.sh +teardown/action.yml diff --git a/tools/rollout/github.mjs b/tools/rollout/github.mjs new file mode 100644 index 0000000..0fe63ca --- /dev/null +++ b/tools/rollout/github.mjs @@ -0,0 +1,258 @@ +import { execFile as execFileCallback } from "node:child_process"; +import { promisify } from "node:util"; +import { createHash } from "node:crypto"; +import { setTimeout as sleep } from "node:timers/promises"; + +const execFile = promisify(execFileCallback); +const DEFAULT_MAX_ATTEMPTS = 4; +const DEFAULT_PAGE_SIZE = 100; +const MAX_REPOSITORY_COUNT = 10_000; +const MAX_REPOSITORY_PAGES = MAX_REPOSITORY_COUNT / DEFAULT_PAGE_SIZE; + +export function statusFromText(text) { + const match = String(text).match(/\bHTTP ([0-9]{3})(?:\)|:)/); + return match ? Number(match[1]) : undefined; +} + +function retryAfterFromText(text) { + const match = String(text).match(/retry-after:\s*([0-9]+)/i); + return match ? Number(match[1]) * 1_000 : undefined; +} + +function parseJson(stdout, description) { + try { + return JSON.parse(stdout); + } catch (error) { + throw new Error(`${description} returned invalid JSON: ${error.message}`); + } +} + +export class GhCommandError extends Error { + constructor(message, options = {}) { + super(message, options); + this.name = "GhCommandError"; + this.status = options.status; + this.retryAfterMs = options.retryAfterMs; + this.exitCode = options.exitCode; + } +} + +export async function executeGh(args) { + try { + return await execFile("gh", args, { + encoding: "utf8", + env: process.env, + maxBuffer: 50 * 1024 * 1024, + }); + } catch (error) { + const stderr = String(error.stderr ?? ""); + const stdout = String(error.stdout ?? ""); + const status = statusFromText(stderr) ?? statusFromText(stdout); + const retryAfterMs = + retryAfterFromText(stderr) ?? retryAfterFromText(stdout); + const detail = + stderr.trim() || `gh exited with code ${error.code ?? "unknown"}`; + throw new GhCommandError(detail, { + cause: error, + exitCode: error.code, + retryAfterMs, + status, + }); + } +} + +export class GitHubClient { + constructor(options = {}) { + this.execute = options.execute ?? executeGh; + this.sleep = options.sleep ?? sleep; + this.maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS; + this.defaultRetryDelayMs = options.defaultRetryDelayMs ?? 1_000; + this.transientRetryDelayMs = options.transientRetryDelayMs ?? 5_000; + + if (!Number.isInteger(this.maxAttempts) || this.maxAttempts < 1) { + throw new Error("maxAttempts must be a positive integer"); + } + } + + async run(args, description) { + for (let attempt = 1; attempt <= this.maxAttempts; attempt += 1) { + try { + return await this.execute(args); + } catch (error) { + const rateLimited403 = + error.status === 403 && + (error.retryAfterMs !== undefined || + /rate.?limit/i.test(error.message)); + const transientNetwork = + error.status === undefined && + /timeout|timed out|connection re|(?:^|unexpected |:\s*)EOF\b|temporary failure/i.test( + error.message, + ); + const retryable = + error.status === 429 || rateLimited403 || transientNetwork; + if (!retryable || attempt === this.maxAttempts) { + throw new Error(`${description} failed: ${error.message}`, { + cause: error, + }); + } + + const baseDelayMs = transientNetwork + ? this.transientRetryDelayMs + : this.defaultRetryDelayMs; + const delay = + error.retryAfterMs ?? baseDelayMs * 2 ** Math.max(0, attempt - 1); + await this.sleep(delay); + } + } + + throw new Error(`${description} failed without an attempt`); + } + + async api(endpoint, description = `GitHub API ${endpoint}`) { + const { stdout } = await this.run( + ["api", "--method", "GET", endpoint], + description, + ); + return parseJson(stdout, description); + } + + async getRef(repository, ref) { + return this.api( + `repos/${repository}/git/ref/${ref}`, + `read ${repository} ref ${ref}`, + ); + } + + async getCommit(repository, sha) { + return this.api( + `repos/${repository}/git/commits/${sha}`, + `read ${repository} commit ${sha}`, + ); + } + + async getTree(repository, sha, recursive = false) { + const suffix = recursive ? "?recursive=1" : ""; + return this.api( + `repos/${repository}/git/trees/${sha}${suffix}`, + `read ${repository} tree ${sha}`, + ); + } + + async getText(repository, path, ref) { + let response = await this.api( + `repos/${repository}/contents/${path.split("/").map(encodeURIComponent).join("/")}?ref=${encodeURIComponent(ref)}`, + `read ${repository}/${path} at ${ref}`, + ); + let blobSha; + // Contents omits bytes above 1 MiB (the Rush lockfile is ~2 MiB). + // Resolve only the returned immutable blob; never fall back to another ref. + if ( + response?.type === "file" && + response.encoding === "none" && + response.content === "" && + Number.isSafeInteger(response.size) && + response.size > 1024 * 1024 && + response.size <= 10 * 1024 * 1024 && + typeof response.sha === "string" && + /^[0-9a-f]{40}$/.test(response.sha) + ) { + blobSha = response.sha; + const blob = await this.api( + `repos/${repository}/git/blobs/${blobSha}`, + `read ${repository}/${path} blob ${blobSha}`, + ); + if (blob?.sha !== blobSha || blob.size !== response.size) + throw new Error("Large source blob does not match file metadata"); + response = { ...blob, type: "file" }; + } + const content = + typeof response?.content === "string" + ? response.content.replaceAll("\n", "") + : undefined; + if ( + response?.type !== "file" || + response.encoding !== "base64" || + content === undefined || + !Number.isSafeInteger(response.size) || + response.size < 0 || + content.length % 4 !== 0 || + !/^[A-Za-z0-9+/]*={0,2}$/.test(content) + ) { + throw new Error( + `${repository}/${path} at ${ref} is not a complete base64 GitHub file response`, + ); + } + const bytes = Buffer.from(content, "base64"); + if ( + bytes.length !== response.size || + bytes.toString("base64") !== content + ) { + throw new Error( + `${repository}/${path} at ${ref} has incomplete or malformed content`, + ); + } + if ( + blobSha && + createHash("sha1") + .update(`blob ${bytes.length}\0`) + .update(bytes) + .digest("hex") !== blobSha + ) + throw new Error("Large source content does not match its Git blob SHA"); + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } + + async listRestRepositories(organization) { + const repositories = []; + + for (let page = 1; page <= MAX_REPOSITORY_PAGES + 1; page += 1) { + const response = await this.api( + `orgs/${organization}/repos?type=all&per_page=${DEFAULT_PAGE_SIZE}&page=${page}`, + `list ${organization} repositories via REST page ${page}`, + ); + if (!Array.isArray(response)) { + throw new Error(`REST repository page ${page} is not an array`); + } + if (page === MAX_REPOSITORY_PAGES + 1) { + if (response.length > 0) { + throw new Error( + `REST repository inventory exceeded ${MAX_REPOSITORY_COUNT} entries`, + ); + } + return repositories; + } + repositories.push(...response); + if (response.length < DEFAULT_PAGE_SIZE) { + return repositories; + } + } + + throw new Error("REST repository pagination ended unexpectedly"); + } + + async listGraphqlRepositories(organization) { + const description = `list ${organization} repositories via GraphQL`; + const { stdout } = await this.run( + [ + "repo", + "list", + organization, + "--limit", + String(MAX_REPOSITORY_COUNT), + "--json", + "name,isArchived,visibility", + ], + description, + ); + const response = parseJson(stdout, description); + if (!Array.isArray(response)) { + throw new Error("GraphQL repository inventory is not an array"); + } + if (response.length === MAX_REPOSITORY_COUNT) { + throw new Error( + `GraphQL repository inventory reached its ${MAX_REPOSITORY_COUNT}-entry limit and may be truncated`, + ); + } + return response; + } +} diff --git a/tools/rollout/integration.mjs b/tools/rollout/integration.mjs new file mode 100644 index 0000000..4152832 --- /dev/null +++ b/tools/rollout/integration.mjs @@ -0,0 +1,560 @@ +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +import { fingerprint } from "./fingerprint.mjs"; + +// Configuration/executable/startup controls, not application credentials. +const RELEVANT_ENV = + /^(?:(?:npm|pnpm|bun|yarn|corepack)_|HOME$|RUNNER_TEMP$|GITHUB_ENV$|GITHUB_PATH$|SFW_BUN_CONFIG_PATH$|USERPROFILE$|XDG_|APPDATA$|LOCALAPPDATA$|PATH$|NODE_OPTIONS$|NODE_PATH$|BASH_ENV$|ENV$|SHELL$|SHELLOPTS$|BASHOPTS$|CDPATH$|LD_|DYLD_|BASH_FUNC_|HTTP_PROXY$|HTTPS_PROXY$|ALL_PROXY$|NO_PROXY$|NODE_EXTRA_CA_CERTS$|NODE_TLS_REJECT_UNAUTHORIZED$|SSL_CERT_|CURL_CA_BUNDLE$)/i; +// Workflow toolchain-version metadata does not select a registry/config path. +const VERSION_ENV = new Set([ + "NODE_VERSION", + "PNPM_VERSION", + "BUN_VERSION", + "XCODE_VERSION", +]); +export const SUPPORTED_SHELLS = new Set([ + "bash", + "sh", + "bash --noprofile --norc -euo pipefail {0}", +]); +const normalize = (value) => String(value ?? "").replaceAll(/\s+/g, ""); +const expression = (value) => + normalize(value).replace(/^\$\{\{(.*)\}\}$/, "$1"); + +export function environmentUncertain(env) { + if (env === undefined) return false; + if (!env || typeof env !== "object" || Array.isArray(env)) return true; + return Object.entries(env).some( + ([key, value]) => + !/^[A-Za-z_][A-Za-z0-9_]*$/.test(key) || + (RELEVANT_ENV.test(key) && !VERSION_ENV.has(key.toUpperCase())) || + !["string", "number", "boolean"].includes(typeof value), + ); +} + +export function literalWorkingDirectory(value) { + return ( + typeof value === "string" && + /^(?:\.\/)?[\w.-]+(?:\/[\w.-]+)*\/?$/.test(value) && + !value.split("/").includes("..") + ); +} + +export function defaultsUncertain(defaults) { + if (defaults === undefined) return false; + return ( + !defaults || + typeof defaults !== "object" || + Array.isArray(defaults) || + Object.keys(defaults).some((key) => key !== "run") || + !defaults.run || + typeof defaults.run !== "object" || + Array.isArray(defaults.run) || + Object.entries(defaults.run).some(([key, value]) => + key === "working-directory" + ? !literalWorkingDirectory(value) + : key !== "shell" || !SUPPORTED_SHELLS.has(value), + ) + ); +} + +export function certainTeardown(operation, setup) { + if (operation.ref !== APPROVED_RELEASE_SHA) return false; + if ( + !operation.uncertain || + (operation.localRuntimeVerified && + operation.integrationConfigurationUncertain === false) + ) + return true; + if (operation.via || operation.boundaryUncertainWithoutCondition) + return false; + const guard = expression(operation.condition); + if (guard === "always()") return true; + const active = guard.match( + /^always\(\)&&steps\.([A-Za-z_][\w-]*)\.outputs\.active==(['"])true\2$/, + ); + return Boolean( + active && + !setup?.via && + setup?.id === active[1] && + !setup.uncertain && + setup.ref === operation.ref, + ); +} + +// Unknown execution is not evidence of a JS dependency install. Retain review +// candidates only where source actually invokes a JS package tool, including +// unsupported wrappers/options. Do not guess from script names or executables. +export function unresolvedJsInvocation(operation) { + if (operation.explicitJsInvocation) return true; + if ( + operation.corepackControl || + operation.integrationLiteral || + operation.kind !== "unknown-wrapper" + ) + return false; + if (operation.manager) return true; + const command = operation.command ?? ""; + const install = + /(?:^|[\s/("'`])(?:(?:npm|pnpm|bun|yarn)\s+(?:ci|install|i|add|fetch|dlx|pack|update|upgrade|config\s+(?:set|delete|unset))\b|(?:npx|bunx)\s)/; + return ( + install.test(command) && + (/^(?:env|sudo|command|time|exec|bash|sh|zsh|uv|poetry|go)\b/.test( + command, + ) || + /^(?![A-Za-z_][A-Za-z0-9_]*=)[^\s=]+=/.test(command) || + /^(?:\.?\.?\/|\/)[^\s]*\/(?:npm|pnpm|npx|bun|bunx|yarn|corepack)\b/.test( + command, + )) + ); +} + +function contextUncertain(job, context) { + return ( + environmentUncertain(job.env) || + defaultsUncertain(job.defaults) || + (context.integrationContextUncertain ?? context.workflowUncertain) || + job.container !== undefined + ); +} + +// Stable output/event comparisons only. Step outputs must belong to one +// completed earlier step, never a future/duplicate ID or mutable env. +function stableGuard(value, job, step) { + if (typeof value !== "string") return undefined; + const text = value + .trim() + .replace(/^\$\{\{([\s\S]*)\}\}$/, "$1") + .trim(); + if (/\$\{\{|\}\}/.test(text)) return undefined; + const terms = text.split(/\s*&&\s*/); + // ponytail: two conjuncts cover observed workflows; no general expression evaluator. + if (terms.length > 2) return undefined; + if (terms.length === 2) { + const guards = terms.map((term) => stableGuard(term, job, step)); + return guards.every(Boolean) ? guards.sort().join("&&") : undefined; + } + const event = text.match(/^github\.event_name\s*==\s*(['"])([a-z_]+)\1$/); + if (event) return `github.event_name==${event[2]}`; + const match = text.match( + /^(steps|needs)\.([\w-]+)\.outputs\.([\w-]+)\s*==\s*(['"])(true|false)\4$/, + ); + if (!match) return undefined; + if (match[1] === "steps") { + const indexes = job.steps.flatMap((item, index) => + item?.id === match[2] ? [index] : [], + ); + if ( + indexes.length !== 1 || + !Number.isInteger(step) || + indexes[0] >= step - 1 + ) + return undefined; + } + return `${match[1]}.${match[2]}.outputs.${match[3]}==${match[5]}`; +} + +export function observedIntegration(operations, job, context) { + const downloads = []; + const additionalJsPaths = []; + const notes = new Set(); + const expectedToken = `\${{secrets.${context.visibility === "public" ? "PUBLIC_SOCKET_FIREWALL_TOKEN" : "SOCKET_FIREWALL_TOKEN"}}}`; + let setup; + let state; + let bunState; + let corepack = "disabled"; + let environmentGroup; + let persistentEnvironmentUncertain = false; + const uncertainContext = contextUncertain(job, context); + const malformed = + !Array.isArray(job.steps) || + job.steps.length === 0 || + operations.some( + (operation) => + operation.sourceError || + (operation.kind === "unknown" && !operation.uses && !operation.reason), + ); + if (malformed) notes.add("malformed-job-or-step"); + + for (const [index, operation] of operations.entries()) { + const uncertain = + operation.integrationConfigurationUncertain ?? + operation.integrationUncertain ?? + operation.uncertain; + const observedState = + operation.manager === "bun" && bunState ? bunState : state; + const operationGuard = + observedState?.guard && + stableGuard( + job.steps?.[operation.step - 1]?.if, + job, + operation.step, + )?.split("&&"); + const currentState = + observedState?.guard && + !observedState.guard + .split("&&") + .every((term) => operationGuard?.includes(term)) + ? { status: "unresolved", reason: "unmatched-setup-condition" } + : observedState; + const validInterval = ["covered", "fork-exception"].includes( + currentState?.status, + ); + if (operation.stepEnvironmentUncertain) + environmentGroup = operation.commandGroup; + if (operation.environmentPersisting) persistentEnvironmentUncertain = true; + const uncertainStepEnvironment = + persistentEnvironmentUncertain || + (environmentGroup !== undefined && + environmentGroup === operation.commandGroup); + if (operation.corepackControl || operation.integrationCorepackUncertain) + corepack = "unresolved"; + if (["enable", "disable"].includes(operation.corepack)) + corepack = uncertain + ? "unresolved" + : operation.corepack === "enable" + ? "enabled" + : "disabled"; + if (operation.localRuntimeVerified) + notes.add("local-runtime-matches-approved-release"); + if (operation.kind === "sfw-setup") { + setup = operation; + if (operation.configureBun === "true") bunState = undefined; + // The pinned action validates the value and independently restricts + // fallback to public external-fork PRs. An expression cannot widen that. + const publicForkFallback = + context.visibility === "public" && + (operation.fallback === "true" || + /^\$\{\{.+\}\}$/.test(operation.fallback)); + const guard = + !operation.via && !operation.boundaryUncertainWithoutCondition + ? stableGuard(operation.condition, job, operation.step) + : undefined; + if ( + (uncertain && !guard) || + (!publicForkFallback && !["false", "true"].includes(operation.fallback)) + ) { + state = { + status: "unresolved", + reason: "conditional-or-uncertain-setup", + }; + } else if ( + operation.ref !== APPROVED_RELEASE_SHA || + (operation.token !== expectedToken && + (!operation.token.includes("${{") || + /^\$\{\{secrets\.[A-Za-z_][\w]*\}\}$/.test(operation.token))) || + (operation.fallback === "true" && context.visibility !== "public") + ) { + state = { status: "gap", reason: "unsupported-setup-configuration" }; + } else if (operation.token !== expectedToken) { + state = { status: "unresolved", reason: "unresolved-setup-token" }; + } else { + state = { + status: publicForkFallback ? "fork-exception" : "covered", + reason: "approved-setup-interval", + ...(guard ? { guard } : {}), + }; + } + continue; + } + if (operation.kind === "sfw-teardown") { + const certain = certainTeardown(operation, setup); + state = certain + ? undefined + : { status: "unresolved", reason: "conditional-or-uncertain-teardown" }; + setup = undefined; + bunState = undefined; + continue; + } + if ( + operation.kind === "js-public-download" || + operation.kind === "yarn-blocked" + ) { + let result; + if ( + uncertainContext || + uncertainStepEnvironment || + uncertain || + (operation.integrationSyntaxUncertain && !validInterval) + ) { + result = { + status: "unresolved", + reason: + uncertainContext || uncertainStepEnvironment + ? "install-execution-context" + : "uncertain-install-syntax-or-condition", + }; + } else if (operation.registryMutating) { + result = { status: "gap", reason: "registry-override" }; + } else if (operation.kind === "yarn-blocked") { + result = { status: "unresolved", reason: "unsupported-yarn-install" }; + } else if (operation.manager === "pnpm" && corepack === "unresolved") { + result = { status: "unresolved", reason: "uncertain-corepack-shims" }; + } else if ( + operation.corepack || + (operation.manager === "pnpm" && corepack === "enabled") + ) { + result = { + status: "gap", + reason: "corepack-download-bypasses-registry", + }; + } else if (!currentState) { + result = { status: "gap", reason: "no-active-setup" }; + } else if ( + currentState.status === "unresolved" || + currentState.status === "gap" + ) { + result = currentState; + } else if ( + operation.manager === "bun" && + setup?.configureBun !== "true" + ) { + result = { + status: setup?.configureBun === "false" ? "gap" : "unresolved", + reason: "bun-configuration-missing-or-unresolved", + }; + } else { + result = currentState; + } + downloads.push({ + operation: index + 1, + step: operation.step, + manager: operation.manager, + ...result, + ...(operation.registryExclusion + ? { + exclusionId: operation.registryExclusion, + status: + result.status === "covered" + ? "covered-with-exclusion" + : result.status, + } + : {}), + }); + } + if (operation.registryPersisting) { + const changed = uncertain + ? { status: "unresolved", reason: "uncertain-registry-mutation" } + : { status: "gap", reason: "registry-override" }; + if (operation.registryManager === "bun") bunState = changed; + else { + state = changed; + setup = undefined; + } + } + if (operation.offlineValidation) + notes.add("capability-guarded-offline-validation"); + const unknown = operation.kind.startsWith("unknown"); + if (unknown || uncertain) + notes.add( + unknown ? "opaque-execution" : "conditional-or-uncertain-operation", + ); + if (operation.integrationScript) { + notes.add("package-script-code-unverified"); + } + if ( + operation.explicitJsInvocation || + (!operation.integrationScript && + unresolvedJsInvocation(operation) && + !operation.registryPersisting) + ) { + const managers = new Set([ + ...(operation.integrationScript ? [] : [operation.manager]), + ...(operation.integrationManagers ?? []), + ]); + const managerUncertain = + !["npm", "pnpm", "bun", "yarn"].some((manager) => + managers.has(manager), + ) || + operation.integrationCorepackDownload || + managers.has("yarn") || + (managers.has("pnpm") && corepack !== "disabled") || + (managers.has("bun") && + (setup?.configureBun !== "true" || + (bunState && + !["covered", "fork-exception"].includes(bunState.status)))); + additionalJsPaths.push({ + operation: index + 1, + step: operation.step, + status: + validInterval && + !managerUncertain && + !operation.registryMutating && + !uncertainContext && + !uncertainStepEnvironment && + !uncertain + ? "setup-observed" + : "unresolved", + reason: "unparsed-js-invocation", + }); + } + // No opacity latch: arbitrary code may do anything at runtime, but that + // cannot erase or synthesize the configuration observed in this workflow. + if (operation.integrationExecutor) notes.add("executor-code-unverified"); + } + const statuses = new Set(downloads.map((download) => download.status)); + const disposition = statuses.has("gap") + ? "needs-sfw" + : statuses.has("unresolved") || + additionalJsPaths.some((path) => path.status === "unresolved") || + malformed + ? "needs-review" + : downloads.length || + additionalJsPaths.some((path) => path.status === "setup-observed") + ? downloads.some((entry) => entry.exclusionId) + ? "integrated-with-exclusions" + : "integrated" + : "no-js-ci"; + return { + disposition, + downloads, + additionalJsPaths, + notes: [...notes].sort(), + runtimeVerification: "not-performed", + }; +} + +// Local reusable workflows are already read at this repository's captured SHA. +// Reuse their primary result rather than treating the call itself as opaque. +// Expressions inside the callee remain unresolved; this does not evaluate inputs +// or forward credentials. Missing files and cycles stay review. Acquisition +// bounds this graph to 200 workflow files; no new source is fetched here. +export function resolveLocalWorkflowCalls(workflows) { + const byPath = new Map( + workflows.map((workflow) => [workflow.path, workflow]), + ); + const resolved = new Map(); + function visit(path, stack = new Set()) { + if (stack.has(path) || !byPath.has(path)) return undefined; + if (resolved.has(path)) return resolved.get(path); + const workflow = byPath.get(path); + const result = { + ...workflow, + jobs: workflow.jobs.map((job) => { + const call = job.operations.find( + (operation) => operation.kind === "reusable-call", + ); + if (!call?.uses.startsWith("./")) return job; + const target = call.uses.slice(2); + const callee = visit(target, new Set([...stack, path])); + if (!callee) return job; + return { + ...job, + reviewFingerprint: fingerprint([ + job.reviewFingerprint, + callee.reviewFingerprint, + callee.jobs.map((child) => child.reviewFingerprint), + ]), + integration: { + ...job.integration, + disposition: integrationDisposition([callee]), + referencedWorkflow: target, + notes: ["local-workflow-analyzed-at-snapshot"], + }, + }; + }), + }; + const sourceByJob = new Map( + result.jobs.map((job) => [job.job, job.reviewFingerprint]), + ); + result.jobs = result.jobs.map((job) => + job.reviewDependencies?.length + ? { + ...job, + reviewFingerprint: fingerprint([ + job.reviewFingerprint, + Object.fromEntries( + job.reviewDependencies.map((name) => [ + name, + sourceByJob.get(name) ?? null, + ]), + ), + ]), + } + : job, + ); + resolved.set(path, result); + return result; + } + return workflows.map((workflow) => visit(workflow.path)); +} + +// Reuse already-captured remote source only when it has no observed JS install +// paths. A callee with installs needs caller-specific inputs/secrets/visibility; +// its coverage cannot simply be copied across repositories. Never substitute a +// default-branch body for a different pinned ref, tag, owner or unread source. +export function resolveNoInstallWorkflowCalls(repositories, organization) { + const byName = new Map( + repositories.map((repository) => [repository.name, repository]), + ); + return repositories.map((repository) => { + if (["audit-error", "empty", "no-ci"].includes(repository.disposition)) + return repository; + const workflows = repository.workflows.map((workflow) => ({ + ...workflow, + jobs: workflow.jobs.map((job) => { + const call = job.operations.find( + (operation) => operation.kind === "reusable-call", + ); + const match = call?.uses.match( + /^([^/]+)\/([^/]+)\/(\.github\/workflows\/[^/@]+\.ya?ml)@(.+)$/, + ); + if (!match || match[1] !== organization) return job; + const target = byName.get(match[2]); + if ( + !target?.headSha || + target.disposition === "audit-error" || + ![target.headSha, target.defaultBranch].includes(match[4]) + ) + return job; + const callee = target.workflows.find( + (candidate) => candidate.path === match[3], + ); + if (!callee || integrationDisposition([callee]) !== "no-js-ci") + return job; + return { + ...job, + integration: { + ...job.integration, + disposition: "no-js-ci", + referencedRepository: target.name, + referencedWorkflow: callee.path, + referencedHeadSha: target.headSha, + notes: ["referenced-snapshot-has-no-observed-js-install"], + }, + }; + }), + })); + return { + ...repository, + workflows, + disposition: integrationDisposition(workflows, repository.exclusions), + }; + }); +} + +export function integrationDisposition(workflows, exclusions = []) { + const dispositions = new Set( + workflows.flatMap((workflow) => [ + ...(workflow.parseError !== undefined ? ["needs-review"] : []), + ...workflow.jobs.map( + (job) => job.integration?.disposition ?? "needs-review", + ), + ]), + ); + if (exclusions.some((entry) => entry.status === "stale")) + dispositions.add("needs-review"); + if ( + dispositions.has("integrated") && + exclusions.some((entry) => entry.status === "matched") + ) + dispositions.add("integrated-with-exclusions"); + for (const disposition of [ + "needs-sfw", + "needs-review", + "integrated-with-exclusions", + "integrated", + ]) { + if (dispositions.has(disposition)) return disposition; + } + return workflows.length ? "no-js-ci" : "no-ci"; +} diff --git a/tools/rollout/integration.test.mjs b/tools/rollout/integration.test.mjs new file mode 100644 index 0000000..f2756cf --- /dev/null +++ b/tools/rollout/integration.test.mjs @@ -0,0 +1,729 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { classifyJob, classifyWorkflow } from "./classify.mjs"; +import { integrationDisposition } from "./integration.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +import { runAudit } from "./audit.mjs"; + +const setup = { + id: "sfw", + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { + token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}", + "configure-bun": true, + }, +}; +const teardown = { + uses: `workos/setup-socket-firewall/teardown@${APPROVED_RELEASE_SHA}`, +}; +const install = { run: "npm ci" }; +const build = { run: "npm run build" }; +const context = { visibility: "private" }; +function job(steps, properties = {}, overrides = {}) { + return classifyJob( + "fixture", + { steps, ...properties }, + { ...context, ...overrides }, + ["push"], + ); +} +const primary = (...args) => job(...args).integration.disposition; + +test("observed install coverage and opaque execution are separate", () => { + const result = job([setup, install, build]); + assert.equal(result.status, "unknown"); + assert.equal(result.integration.disposition, "integrated"); + assert.equal(result.integration.downloads[0].status, "covered"); + assert.ok(result.integration.notes.includes("opaque-execution")); + assert.equal(primary([install, build]), "needs-sfw"); + assert.equal(primary([setup, install, teardown, build]), "integrated"); + assert.equal(primary([setup, install, build, install]), "integrated"); + assert.equal(primary([build, setup, install]), "integrated"); +}); + +test("an integrated sibling cannot hide a missing or unresolved install job", () => { + assert.equal( + integrationDisposition([ + { jobs: [job([setup, install]), job([install, build])] }, + ]), + "needs-sfw", + ); + assert.equal( + integrationDisposition([{ jobs: [job([setup, install]), job([build])] }]), + "integrated", + ); + assert.equal(primary([{ uses: "./unresolved" }]), "needs-review"); + assert.equal( + primary([ + { run: "go build ./..." }, + { uses: "actions/upload-artifact@v4" }, + ]), + "no-js-ci", + ); + assert.equal(primary([{ run: "git status" }]), "no-js-ci"); +}); + +test("supported toolchain roles preserve intervals without synthesizing setup", () => { + for (const tool of [ + { uses: "pnpm/action-setup@v4" }, + { + uses: "pnpm/action-setup@v4", + with: { version: "10", run_install: false }, + }, + { uses: "oven-sh/setup-bun@v2", with: { "bun-version": "1.3.14" } }, + ]) { + assert.equal(primary([setup, tool, install, build]), "integrated"); + assert.equal(primary([tool, install, build]), "needs-sfw"); + assert.equal(job([setup, tool, install]).status, "unknown"); + } + for (const tool of [ + { uses: "pnpm/action-setup@v4", with: { run_install: true } }, + { uses: "pnpm/action-setup@v4", with: { custom: "anything" } }, + { uses: "pnpm/action-setup/other@v4" }, + { uses: "untrusted/action-setup@v4" }, + { + uses: "oven-sh/setup-bun@v2", + with: { "bun-download-url": "https://example.invalid/bun" }, + }, + { uses: "oven-sh/setup-bun/other@v2" }, + ]) + assert.equal(primary([setup, tool, install]), "integrated"); + assert.equal( + primary([ + setup, + { + uses: "pnpm/action-setup@v4", + with: { run_install: "${{ inputs.install }}" }, + }, + install, + ]), + "needs-review", + ); +}); + +test("job reachability and harmless context do not alter install ordering", () => { + assert.equal( + primary([setup, install], { + if: "github.ref == 'refs/heads/main'", + env: { CI: true }, + defaults: { run: { shell: "bash" } }, + }), + "integrated", + ); + assert.equal(primary([setup, install], { if: false }), "no-js-ci"); + const workflow = classifyWorkflow( + "on: push\ndefaults:\n run:\n shell: bash\nenv:\n CI: true\njobs:\n test:\n steps:\n - uses: " + + setup.uses + + "\n with:\n token: '${{ secrets.SOCKET_FIREWALL_TOKEN }}'\n - run: npm ci\n", + { ...context, path: ".github/workflows/ci.yml" }, + ); + assert.equal(integrationDisposition([workflow]), "integrated"); + assert.equal( + primary([setup, { ...install, env: { NODE_ENV: "production" } }]), + "integrated", + ); + for (const env of [ + { NPM_CONFIG_REGISTRY: "https://example.invalid" }, + { PATH: "/custom" }, + { HOME: "/custom" }, + { BASH_ENV: "startup.sh" }, + { "${{ inputs.key }}": "value" }, + { UNMODELED: {} }, + ]) { + assert.equal(primary([setup, install], { env }), "needs-review"); + assert.equal(primary([setup, { ...install, env }]), "needs-review"); + } + assert.equal( + primary([setup, install], { defaults: { run: { shell: "pwsh" } } }), + "needs-review", + ); +}); + +test("public fallback is recorded as an exception, not missing integration", () => { + const fallback = { + ...setup, + with: { + token: "${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }}", + "allow-external-fork-fallback": true, + }, + }; + const result = job([fallback, install, build], {}, { visibility: "public" }); + assert.equal(result.integration.disposition, "integrated"); + assert.equal(result.integration.downloads[0].status, "fork-exception"); + assert.equal(result.status, "unknown"); + assert.equal( + primary([ + { + ...setup, + with: { ...setup.with, "allow-external-fork-fallback": true }, + }, + install, + ]), + "needs-sfw", + ); + assert.equal( + primary([ + { + ...setup, + with: { + ...setup.with, + "allow-external-fork-fallback": "${{ inputs.fallback }}", + }, + }, + install, + ]), + "needs-review", + ); +}); + +test("per-download negative controls remain actionable or unresolved", () => { + for (const steps of [ + [{ ...setup, if: false }, install, build], + [install, setup, install], + [setup, install, teardown, install, build], + [{ ...setup, uses: "workos/setup-socket-firewall@v1" }, install], + [{ ...setup, with: { token: "wrong" } }, install], + [ + { ...setup, with: { ...setup.with, "configure-bun": false } }, + { run: "bun install" }, + ], + [setup, { run: "npm ci --reg=https://example.invalid" }], + [{ run: "corepack enable" }, setup, { run: "pnpm install" }], + ]) + assert.equal(primary(steps), "needs-sfw"); + for (const steps of [ + [{ ...setup, if: "github.ref == 'refs/heads/main'" }, install], + [{ ...setup, "continue-on-error": true }, install], + [setup, { run: "npm ci --future-option=unknown" }], + [setup, { run: "npm --prefix help ci" }], + [setup, { uses: "./unresolved" }, install], + [setup, { run: "corepack enable pnpm" }, { run: "pnpm install" }], + ]) + assert.equal(primary(steps), "needs-review"); +}); + +test("teardown guards establish only the appropriate success-path boundary", () => { + for (const condition of [ + "always()", + "${{ always() && steps.sfw.outputs.active == 'true' }}", + ]) { + const clean = job([ + setup, + install, + { ...teardown, if: condition }, + { run: "npm publish" }, + ]); + assert.equal(clean.integration.disposition, "integrated"); + assert.equal(clean.status, "unknown"); + assert.ok( + !clean.violations.some((v) => v.includes("without a same-SHA teardown")), + ); + assert.equal( + primary([setup, install, { ...teardown, if: condition }, install]), + "needs-sfw", + ); + assert.notEqual( + job([ + setup, + install, + { ...teardown, if: condition }, + { run: "npm publish", if: "always()" }, + ]).status, + "protected", + ); + } + for (const boundary of [ + { ...teardown, if: "always() && steps.other.outputs.active == 'true'" }, + { ...teardown, if: "always()", "continue-on-error": true }, + { ...teardown, if: "inputs.cleanup" }, + ]) { + assert.equal( + job([setup, install, boundary, { run: "npm publish" }]).status, + "unsafe-publish", + ); + assert.equal(primary([setup, install, boundary, install]), "needs-review"); + } +}); + +test("audit v3 separates primary integration counts from assurance and errors", async () => { + const workflow = `on: push\njobs:\n build:\n steps:\n - uses: ${setup.uses}\n with: { token: '${setup.with.token}' }\n - run: npm ci\n - run: npm test\n`; + const client = { + listRestRepositories: async () => + ["app", "broken"].map((name) => ({ + name, + default_branch: "main", + archived: false, + visibility: "private", + })), + listGraphqlRepositories: async () => + ["app", "broken"].map((name) => ({ + name, + isArchived: false, + visibility: "private", + })), + getRef: async () => ({ object: { sha: "a".repeat(40) } }), + getTree: async (repo) => ({ + truncated: repo.endsWith("broken"), + tree: [ + { path: ".github/workflows/ci.yml", type: "blob", mode: "100644" }, + ], + }), + getText: async () => workflow, + }; + const report = await runAudit(client); + assert.equal(report.schemaVersion, 3); + assert.deepEqual(report.dispositions, { "audit-error": 1, integrated: 1 }); + assert.deepEqual(report.assuranceDispositions, { + "audit-error": 1, + "needs-review": 1, + }); + assert.equal(report.scanStatus, "partial"); + assert.equal(report.scanErrors, 1); + assert.equal(report.runtimeVerification, "not-performed"); + assert.equal( + integrationDisposition([ + classifyWorkflow("on: [", { ...context, path: "fixture" }), + ]), + "needs-review", + ); +}); + +test("malformed steps cannot become no-js-ci or integrated", () => { + for (const steps of [ + undefined, + null, + [], + [null], + [{}], + [setup, install, {}], + ]) { + assert.equal(job(steps).integration.disposition, "needs-review"); + } + assert.equal( + integrationDisposition([ + classifyWorkflow("on: push\njobs:\n bad: false\n", { + visibility: "private", + }), + ]), + "needs-review", + ); +}); + +test("explicit unparsed JS installers remain candidates, generic executors do not", () => { + for (const run of [ + "uv run npm ci", + "poetry run npm ci", + "command npm ci", + "npm --prefix help ci", + ]) { + assert.equal(job([{ run }]).integration.disposition, "needs-review", run); + assert.equal( + integrationDisposition([ + { jobs: [job([setup, install]), job([{ run }])] }, + ]), + "needs-review", + run, + ); + } + for (const run of [ + "go run bootstrap.go", + "python scripts/bootstrap.py", + "CI=1 make bootstrap", + "CI=1 node scripts/bootstrap.mjs", + "CI=1 ./scripts/bootstrap.sh", + "${{ inputs.command }}", + ]) { + assert.equal(primary([{ run }]), "no-js-ci"); + assert.equal( + integrationDisposition([ + { jobs: [job([setup, install]), job([{ run }])] }, + ]), + "integrated", + ); + } + assert.equal( + job([], { + uses: "example/shared/.github/workflows/build.yml@main", + if: false, + }).integration.disposition, + "no-js-ci", + ); +}); + +test("dynamic Bun configuration and uncertain local boundaries stay unresolved", () => { + assert.equal( + job([ + { + ...setup, + with: { ...setup.with, "configure-bun": "${{ inputs.bun }}" }, + }, + { run: "bun install" }, + ]).integration.disposition, + "needs-review", + ); + const localActions = new Map([ + [ + "inner/action.yml", + `runs:\n using: composite\n steps:\n - uses: ${setup.uses}\n with: { token: '${setup.with.token}' }\n - run: npm ci\n shell: bash\n`, + ], + ]); + const nested = classifyJob( + "fixture", + { steps: [{ uses: "./inner", if: "inputs.install" }] }, + { visibility: "private", localActions }, + ["push"], + ); + assert.equal(nested.integration.disposition, "needs-review"); + assert.equal( + job([setup, { uses: "pnpm/action-setup" }, install]).integration + .disposition, + "integrated", + ); +}); + +test("active-output teardown does not confuse composite-local step IDs with job IDs", () => { + const localActions = new Map([ + [ + "inner/action.yml", + `runs:\n using: composite\n steps:\n - uses: ${setup.uses}\n id: sfw\n with: { token: '${setup.with.token}' }\n - run: npm ci\n shell: bash\n`, + ], + ]); + const result = classifyJob( + "fixture", + { + steps: [ + { uses: "./inner" }, + { ...teardown, if: "always() && steps.sfw.outputs.active == 'true'" }, + { run: "npm publish" }, + ], + }, + { visibility: "private", localActions }, + ["push"], + ); + assert.equal(result.status, "unsafe-publish"); + assert.equal(primary([{ run: "git diff file.js" }]), "no-js-ci"); +}); + +test("Corepack uncertainty survives setup and conditional or targeted controls", () => { + for (const steps of [ + [ + { run: "corepack enable" }, + { run: "corepack disable", if: "inputs.disable_corepack" }, + ], + [{ run: "corepack enable", if: "inputs.enable_corepack" }], + [{ run: "corepack enable pnpm", if: "inputs.enable_corepack" }], + [{ run: "corepack enable pnpm" }], + [ + { run: "corepack enable" }, + { run: "corepack disable yarn", if: "inputs.disable_corepack" }, + ], + ]) { + const result = job([...steps, setup, { run: "pnpm install" }]); + assert.equal(result.integration.disposition, "needs-review"); + assert.equal(result.integration.downloads[0].status, "unresolved"); + } + assert.equal( + primary([ + { run: "corepack enable", if: "inputs.enable_corepack" }, + { run: "corepack disable" }, + setup, + { run: "pnpm install" }, + ]), + "integrated", + ); + assert.equal( + primary([ + { run: "corepack enable pnpm", if: "inputs.enable_corepack" }, + { run: "corepack disable" }, + setup, + { run: "pnpm install" }, + ]), + "integrated", + ); + assert.equal( + primary([{ run: "corepack enable" }, setup, { run: "pnpm install" }]), + "needs-sfw", + ); +}); + +test("script diagnostics survive later setup without becoming invented installs", () => { + for (const before of [ + [], + [{ ...setup, if: "inputs.enable_sfw" }], + [{ ...setup, with: { token: "wrong" } }], + [setup, install, teardown], + ]) { + const result = job([ + ...before, + { run: "npm run bootstrap" }, + setup, + install, + ]); + assert.equal(result.integration.disposition, "integrated"); + assert.ok( + result.integration.notes.includes("package-script-code-unverified"), + ); + assert.equal(result.status, "unknown"); + } +}); + +test("malformed local source remains review even after a covered install", () => { + for (const text of [ + "runs: [", + "runs: false", + "runs: { using: composite, steps: false }", + "false", + ]) { + const result = job( + [setup, install, { uses: "./broken" }], + {}, + { + localActions: new Map([["broken/action.yml", text]]), + }, + ); + assert.equal(result.integration.disposition, "needs-review"); + assert.ok(result.operations.some((operation) => operation.sourceError)); + assert.equal(result.integration.downloads[0].status, "covered"); + } +}); + +test("inline environment and unsupported env wrappers retain installer candidates", () => { + for (const run of [ + "HOME=/tmp npm ci", + "PATH=/tmp npm ci", + "NODE_OPTIONS=--require=./bootstrap.cjs npm ci", + "env HOME=/tmp npm ci", + "env -i npm ci", + "env --ignore-environment npm ci", + "env --unset=HOME npm ci", + "1BAD=x npm ci", + "sudo npm ci", + "time npm ci", + "exec npm ci", + ]) { + assert.equal(primary([setup, { run }]), "needs-review", run); + assert.equal(primary([{ run }]), "needs-review", run); + assert.equal( + integrationDisposition([ + { jobs: [job([setup, install]), job([{ run }])] }, + ]), + "needs-review", + run, + ); + } + for (const run of ["CI=1 npm ci", "env CI=1 npm ci"]) { + assert.equal(primary([setup, { run }]), "integrated", run); + assert.equal(primary([{ run }]), "needs-sfw", run); + assert.notEqual(job([setup, { run }]).status, "protected"); + } + assert.equal( + primary([{ run: "git status" }, { uses: "actions/upload-artifact@v4" }]), + "no-js-ci", + ); +}); + +test("only bounded literal logging and safe shell prologues preserve direct installs", () => { + for (const prefix of [ + "set -e", + "set -euo pipefail", + 'echo "Installing dependencies"', + "echo 'Installing dependencies'", + 'echo "source: npm ci; npm ci"', + 'echo "--registry=https://example.invalid"', + ]) { + const run = `${prefix}\nnpm ci`; + assert.equal(primary([setup, { run }]), "integrated", run); + assert.equal(primary([{ run }]), "needs-sfw", run); + assert.notEqual(job([setup, { run }]).status, "protected"); + } + for (const run of [ + 'echo "literal source: npm ci"', + 'echo "source: npm ci; npm ci"', + "echo 'source: npm ci && npm ci'", + ]) { + assert.notEqual(primary([setup, { run }]), "integrated", run); + assert.notEqual(primary([{ run }]), "needs-sfw", run); + } +}); + +test("ordinary package script roles preserve observed setup, never assure script code", () => { + for (const run of [ + "npm run build", + "npm test", + "npm test -- --runInBand", + "pnpm run lint", + "bun run build", + "yarn test", + ]) { + const result = job([ + setup, + { run }, + { run: "npx wrangler deploy --env production" }, + ]); + assert.equal(result.integration.disposition, "integrated", run); + assert.equal(result.status, "unknown", run); + assert.ok( + result.integration.notes.includes("package-script-code-unverified"), + ); + assert.equal(primary([setup, { run }]), "no-js-ci", run); + assert.equal(primary([{ run }, setup, install]), "integrated", run); + } + for (const run of ["npm --prefix scripts run build"]) { + assert.notEqual(primary([setup, { run }, install]), "integrated", run); + } + assert.equal(primary([setup, build, teardown, install]), "needs-sfw"); + assert.equal( + primary([ + setup, + build, + { run: "npm config set registry https://example.invalid" }, + install, + ]), + "needs-sfw", + ); +}); + +test("direct executor target arguments are not installer configuration", () => { + for (const run of [ + "npx wrangler deploy --env production", + "npx -y wrangler deploy --env production", + "npx --package=tool tool", + "bunx biome check --write", + "npx tool --registry=https://example.invalid", + "npx tool --env $ENVIRONMENT", + "npx tool $(node bootstrap.mjs)", + ]) { + const result = job([setup, { run }]); + assert.equal(result.integration.disposition, "integrated", run); + assert.equal(result.status, "unknown", run); + assert.ok(result.integration.notes.includes("executor-code-unverified")); + assert.equal(primary([{ run }]), "needs-sfw", run); + } + for (const run of [ + "npx --registry=https://example.invalid wrangler", + "bunx --reg=https://example.invalid tool", + "npx --@scope:registry=https://example.invalid tool", + ]) { + assert.equal(primary([setup, { run }]), "needs-sfw", run); + } + for (const run of [ + "npx --future-option tool", + "npx --package=$PACKAGE tool", + "npx $TOOL --env production", + "npx", + "npm exec tool -- --env production", + "npm exec tool", + "env -i npx tool", + ]) + assert.equal(primary([setup, { run }]), "needs-review", run); +}); + +test("application environment and sibling services do not change observed host routing", () => { + const env = { + APP_TOKEN: "${{ secrets.APP_TOKEN }}", + CI: "${{ inputs.ci }}", + DEPLOY_ENV: "production", + }; + assert.equal(primary([setup, install], { env }), "integrated"); + assert.equal(primary([setup, { ...install, env }]), "integrated"); + assert.equal(job([setup, install], { env }).status, "unknown"); + const workflow = classifyWorkflow( + `on: push\nenv:\n APP_TOKEN: '${env.APP_TOKEN}'\njobs:\n test:\n steps:\n - uses: ${setup.uses}\n with: { token: '${setup.with.token}' }\n - run: npm ci\n`, + context, + ); + assert.equal(integrationDisposition([workflow]), "integrated"); + for (const key of [ + "npm_config_registry", + "PNPM_HOME", + "BUN_CONFIG", + "YARN_RC_FILENAME", + "HOME", + "USERPROFILE", + "XDG_CONFIG_HOME", + "APPDATA", + "LOCALAPPDATA", + "PATH", + "NODE_OPTIONS", + "NODE_PATH", + "BASH_ENV", + "ENV", + "SHELL", + "LD_PRELOAD", + "DYLD_INSERT_LIBRARIES", + "HTTPS_PROXY", + "NODE_EXTRA_CA_CERTS", + "bad-key", + ]) { + assert.equal( + primary([setup, install], { env: { [key]: "${{ secrets.VALUE }}" } }), + "needs-review", + key, + ); + } + for (const env of ["${{ inputs.environment }}", [], null, { APP_TOKEN: {} }]) + assert.equal(primary([setup, install], { env }), "needs-review"); + const services = { database: { image: "postgres:17" } }; + assert.equal(primary([setup, install], { services }), "integrated"); + assert.equal(primary([install], { services }), "needs-sfw"); + assert.equal(job([setup, install], { services }).status, "unknown"); + assert.equal( + primary([setup, install], { container: "node:22" }), + "needs-review", + ); +}); + +test("unresolved setup token forwarding is not evidence of a wrong token", () => { + for (const token of [ + "${{ inputs.token }}", + "${{ secrets[inputs.token_name] }}", + ]) { + const result = job([{ ...setup, with: { token } }, install]); + assert.equal(result.integration.disposition, "needs-review"); + assert.equal( + result.integration.downloads[0].reason, + "unresolved-setup-token", + ); + assert.notEqual(result.status, "protected"); + } + for (const token of [ + "wrong", + "", + "${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }}", + ]) + assert.equal( + primary([{ ...setup, with: { token } }, install]), + "needs-sfw", + ); + assert.equal(primary([install]), "needs-sfw"); +}); + +test("exact npm lockfile host replacement preserves the configured registry", () => { + assert.equal( + primary([setup, { run: "npm ci --replace-registry-host=always" }]), + "integrated", + ); + assert.equal( + primary([{ run: "npm ci --replace-registry-host=always" }]), + "needs-sfw", + ); + for (const option of [ + "--registry=https://example.invalid", + "--reg=https://example.invalid", + "--@scope:registry=https://example.invalid", + ]) + assert.equal( + primary([ + setup, + { run: `npm ci --replace-registry-host=always ${option}` }, + ]), + "needs-sfw", + ); + for (const run of [ + "npm ci --replace-registry-host=never", + "npm ci --replace-registry-host=npmjs", + "npm ci --replace-registry-host=example.invalid", + "npm ci --replace-registry-host always", + "pnpm install --replace-registry-host=always", + ]) + assert.equal(primary([setup, { run }]), "needs-review", run); +}); diff --git a/tools/rollout/inventory.mjs b/tools/rollout/inventory.mjs new file mode 100644 index 0000000..e9a202d --- /dev/null +++ b/tools/rollout/inventory.mjs @@ -0,0 +1,132 @@ +function assert(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +function compareStrings(left, right) { + return left < right ? -1 : left > right ? 1 : 0; +} + +function sorted(values) { + return [...values].sort(compareStrings); +} + +function indexRepositories(repositories, source) { + assert(Array.isArray(repositories), `${source} inventory is not an array`); + const indexed = new Map(); + + for (const repository of repositories) { + assert( + repository && typeof repository.name === "string" && repository.name, + `${source} inventory contains a repository without a name`, + ); + assert( + !indexed.has(repository.name), + `${source} inventory contains duplicate repository ${repository.name}`, + ); + const archived = + source === "REST" ? repository.archived : repository.isArchived; + assert( + typeof archived === "boolean", + `${source} repository ${repository.name} has invalid archived state`, + ); + indexed.set(repository.name, repository); + } + + return indexed; +} + +function difference(left, right) { + return sorted([...left].filter((name) => !right.has(name))); +} + +function normalizedVisibility(repository, source) { + const visibility = String(repository.visibility ?? "").toLowerCase(); + assert( + ["internal", "private", "public"].includes(visibility), + `${source} repository ${repository.name} has invalid visibility`, + ); + return visibility; +} + +export function reconcileRepositoryInventories( + restRepositories, + graphqlRepositories, +) { + const rest = indexRepositories(restRepositories, "REST"); + const graphql = indexRepositories(graphqlRepositories, "GraphQL"); + const restNames = new Set(rest.keys()); + const graphqlNames = new Set(graphql.keys()); + const restActive = new Set( + [...rest] + .filter(([, repository]) => repository.archived === false) + .map(([name]) => name), + ); + const graphqlActive = new Set( + [...graphql] + .filter(([, repository]) => repository.isArchived === false) + .map(([name]) => name), + ); + + const visibilityMismatches = sorted(restNames) + .filter((name) => graphql.has(name)) + .filter( + (name) => + normalizedVisibility(rest.get(name), "REST") !== + normalizedVisibility(graphql.get(name), "GraphQL"), + ); + const differences = { + activeOnlyGraphql: difference(graphqlActive, restActive), + activeOnlyRest: difference(restActive, graphqlActive), + allOnlyGraphql: difference(graphqlNames, restNames), + allOnlyRest: difference(restNames, graphqlNames), + visibilityMismatches, + }; + const mismatchCount = Object.values(differences).reduce( + (total, names) => total + names.length, + 0, + ); + assert( + mismatchCount === 0, + `REST and GraphQL repository inventories differ: ${JSON.stringify(differences)}`, + ); + + const repositories = sorted(restActive).map((name) => { + const repository = rest.get(name); + assert( + typeof repository.default_branch === "string" && + repository.default_branch, + `active REST repository ${name} has no default branch`, + ); + return { + defaultBranch: repository.default_branch, + ...(repository.id === undefined ? {} : { repositoryId: repository.id }), + name, + visibility: normalizedVisibility(repository, "REST"), + }; + }); + + const visibility = { internal: 0, private: 0, public: 0 }; + for (const repository of repositories) { + visibility[repository.visibility] += 1; + } + + return { + activeCount: repositories.length, + archivedCount: rest.size - repositories.length, + differences, + repositories, + schemaVersion: 1, + totalCount: rest.size, + visibility, + }; +} + +export async function captureRepositoryInventory(client, organization) { + const [restRepositories, graphqlRepositories] = await Promise.all([ + client.listRestRepositories(organization), + client.listGraphqlRepositories(organization), + ]); + return reconcileRepositoryInventories(restRepositories, graphqlRepositories); +} diff --git a/tools/rollout/package-manager.test.mjs b/tools/rollout/package-manager.test.mjs new file mode 100644 index 0000000..896cf22 --- /dev/null +++ b/tools/rollout/package-manager.test.mjs @@ -0,0 +1,140 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { stringify } from "yaml"; +import { auditRepository } from "./audit.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; + +const sha = "a".repeat(40); +const checkout = { uses: `actions/checkout@${"b".repeat(40)}` }; +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}" }, +}; +const run = `set -euo pipefail +pnpm_package="$(node --print 'require("./package.json").packageManager')" +npm install --global "$pnpm_package" --ignore-scripts --no-audit --no-fund`; + +async function inspect({ + packageManager = "pnpm@11.20.0", + steps = [checkout, setup, { run }], + job = {}, + files = {}, + manifestMode = "100644", +} = {}) { + const sources = { + ".github/workflows/ci.yml": stringify({ + on: "push", + jobs: { check: { ...job, steps } }, + }), + "package.json": JSON.stringify({ packageManager }), + ...files, + }; + const reads = []; + const result = await auditRepository( + { + getRef: async () => ({ object: { sha } }), + getTree: async () => ({ + truncated: false, + tree: Object.keys(sources).map((path) => ({ + path, + type: "blob", + mode: path === "package.json" ? manifestMode : "100644", + })), + }), + getText: async (repository, path, ref) => { + assert.equal(ref, sha); + reads.push(path); + if (!Object.hasOwn(sources, path)) throw new Error("missing source"); + return sources[path]; + }, + }, + { name: "fixture", defaultBranch: "main", visibility: "private" }, + ); + return { ...result, reads }; +} + +test("root packageManager bootstrap resolves the captured pin, not an arbitrary variable", async () => { + const result = await inspect(); + assert.equal(result.disposition, "integrated"); + assert.equal(result.assuranceDisposition, "needs-review"); + assert.ok(result.reads.includes("package.json")); + assert.equal( + (await inspect({ steps: [checkout, { run }] })).disposition, + "needs-sfw", + ); + assert.equal( + ( + await inspect({ steps: [checkout, setup, { run: "npm ci" }] }) + ).reads.includes("package.json"), + false, + ); +}); + +test("unresolved or changed packageManager source never clears the finding", async () => { + for (const packageManager of [ + null, + ["pnpm@11.20.0"], + { packageManager: "pnpm@11.20.0" }, + "pnpm@latest", + "pnpm@11.20.0\n", + "pnpm@^11.20.0", + "pnpm@https://example.com/pnpm.tgz", + "--registry=https://example.com", + "pnpm@11.20.0\nnpm install evil", + "npm@11.0.0", + ]) + assert.equal( + (await inspect({ packageManager })).disposition, + "needs-review", + String(packageManager), + ); + for (const changed of [ + { steps: [setup, { run }] }, + { steps: [checkout, setup, { uses: 42 }, { run }] }, + { steps: [{ ...checkout, with: { ref: "other" } }, setup, { run }] }, + { steps: [{ ...checkout, with: { path: "nested" } }, setup, { run }] }, + { steps: [checkout, setup, { run: "node rewrite-package.mjs" }, { run }] }, + { + steps: [checkout, setup, { uses: "./rewrite" }, { run }], + files: { + "rewrite/action.yml": stringify({ + runs: { + using: "composite", + steps: [{ shell: "bash", run: "node rewrite-package.mjs" }], + }, + }), + }, + }, + { steps: [checkout, setup, { run, "working-directory": "nested" }] }, + { job: { defaults: { run: { "working-directory": "nested" } } } }, + { job: { env: { NODE_OPTIONS: "--require ./rewrite.cjs" } } }, + { files: { ".npmrc": "registry=https://other.invalid/" } }, + { + steps: [ + checkout, + setup, + { run: `${run}\nnpm config set registry https://other.invalid/` }, + ], + }, + { + steps: [ + checkout, + setup, + { run: run.replace("--no-fund", "--registry=https://other.invalid/") }, + ], + }, + ]) + assert.notEqual( + (await inspect(changed)).disposition, + "integrated", + JSON.stringify(changed), + ); + assert.equal( + (await inspect({ manifestMode: "120000" })).disposition, + "audit-error", + ); + assert.equal( + (await inspect({ files: { "package.json": "{broken" } })).disposition, + "audit-error", + ); +}); diff --git a/tools/rollout/primary-boundaries.test.mjs b/tools/rollout/primary-boundaries.test.mjs new file mode 100644 index 0000000..23723f6 --- /dev/null +++ b/tools/rollout/primary-boundaries.test.mjs @@ -0,0 +1,165 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { classifyJob } from "./classify.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}" }, +}; +const install = { run: "npm ci" }; +const context = { visibility: "private" }; +const inspect = (steps, extra = {}) => + classifyJob("fixture", { steps }, { ...context, ...extra }, ["push"]); + +test("ordinary run guards preserve observed ordering, not boundary reachability", () => { + for (const guard of [ + "github.ref == 'refs/heads/main'", + "${{ github.event_name == 'push' }}", + ]) { + assert.equal( + inspect([setup, { ...install, if: guard }]).integration.disposition, + "integrated", + ); + assert.equal( + inspect([{ ...install, if: guard }]).integration.disposition, + "needs-sfw", + ); + const row = inspect([ + setup, + install, + { run: "npm run deploy", if: guard }, + { run: "npx wrangler deploy --env production", if: guard }, + ]); + assert.equal(row.integration.disposition, "integrated"); + assert.equal(row.status, "unknown"); + assert.equal( + inspect([{ ...setup, if: guard }, install]).integration.disposition, + "needs-review", + ); + } + for (const guard of [ + "always()", + "Always()", + "failure()", + "!cancelled()", + "success() || true", + ]) { + assert.equal( + inspect([setup, { ...install, if: guard }]).integration.disposition, + "needs-review", + ); + } + const localActions = new Map([ + [ + "configure/action.yml", + `runs:\n using: composite\n steps:\n - uses: ${setup.uses}\n with:\n token: '${setup.with.token}'\n`, + ], + ]); + assert.equal( + inspect( + [{ uses: "./configure", if: "github.ref == 'refs/heads/main'" }, install], + { localActions }, + ).integration.disposition, + "needs-review", + ); +}); + +test("toolchain version metadata is distinct from package-manager configuration", () => { + const steps = [setup, install]; + assert.equal( + classifyJob( + "fixture", + { steps, env: { BUN_VERSION: "1.3.14", PNPM_VERSION: "9.15.9" } }, + context, + ["push"], + ).integration.disposition, + "integrated", + ); + for (const env of [ + { BUN_INSTALL: "/tmp/other" }, + { PNPM_CONFIG_REGISTRY: "https://registry.example.invalid" }, + ]) { + assert.equal( + classifyJob("fixture", { steps, env }, context, ["push"]).integration + .disposition, + "needs-review", + ); + } +}); + +test("GitHub expression execution remains unverified without erasing separate install configuration", () => { + for (const run of [ + "echo '${{ inputs.payload }}'\nnpm ci", + 'echo "${{ inputs.payload }}"\nnpm ci', + ]) { + assert.equal( + inspect([setup, { run }]).integration.disposition, + "integrated", + ); + assert.equal(inspect([setup, { run }]).status, "unknown"); + assert.equal(inspect([{ run }]).integration.disposition, "needs-sfw"); + } + assert.equal( + inspect([setup, { run: "echo '$SHELL_LITERAL'\nnpm ci" }]).integration + .disposition, + "integrated", + ); +}); + +test("exhausted local expansion retains covered evidence but requires review", () => { + const steps = [ + ...Array.from({ length: 997 }, () => ({ run: "echo ok", shell: "bash" })), + { run: "npm ci --registry=https://example.invalid", shell: "bash" }, + ]; + const localActions = new Map([ + [ + "large/action.yml", + JSON.stringify({ runs: { using: "composite", steps } }), + ], + ]); + const result = inspect([setup, install, { uses: "./large" }], { + localActions, + }); + assert.equal( + result.operations.at(-1).sourceError, + "unresolved-local-action-expansion", + ); + assert.equal(result.integration.downloads[0].status, "covered"); + assert.equal(result.integration.disposition, "needs-review"); +}); + +test("missing, cyclic and malformed local/action source stays visible after install", () => { + assert.equal( + inspect([setup, install, { uses: "./missing" }]).integration.disposition, + "needs-review", + ); + const localActions = new Map([ + [ + "cycle/action.yml", + "runs:\n using: composite\n steps:\n - uses: ./cycle\n", + ], + ]); + assert.equal( + inspect([setup, install, { uses: "./cycle" }], { localActions }).integration + .disposition, + "needs-review", + ); + for (const value of [{ nested: true }, ["not", "a", "scalar"]]) { + assert.equal( + inspect([ + setup, + { uses: "pnpm/action-setup@v4", with: { version: value } }, + install, + ]).integration.disposition, + "needs-review", + ); + assert.equal( + inspect([ + setup, + install, + { uses: "actions/upload-artifact@v4", with: { path: value } }, + ]).integration.disposition, + "needs-review", + ); + } +}); diff --git a/tools/rollout/regressions.test.mjs b/tools/rollout/regressions.test.mjs new file mode 100644 index 0000000..7b198de --- /dev/null +++ b/tools/rollout/regressions.test.mjs @@ -0,0 +1,217 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { auditRepository, runAudit } from "./audit.mjs"; +import { + classifyJob, + classifyWorkflow, + repositoryDisposition, +} from "./classify.mjs"; +import { scanExitCode } from "./cli.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +import { parseYamlSource } from "./yaml.mjs"; +import { statusFromText } from "./github.mjs"; + +test("GitHub status parsing handles both CLI diagnostic formats", () => { + assert.equal(statusFromText("gh: API rate limit exceeded (HTTP 429)"), 429); + assert.equal(statusFromText("HTTP 429: rate limit exceeded"), 429); + assert.equal(statusFromText("HTTP 403: rate limit exceeded"), 403); + assert.equal(statusFromText("HTTP 404: Not Found"), 404); + assert.equal(statusFromText("network failure"), undefined); +}); + +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}" }, +}; +const install = { run: "npm ci" }; +const context = { visibility: "private", path: ".github/workflows/ci.yml" }; +function inspect(steps, overrides = {}) { + const job = classifyJob("build", { steps }, { ...context, ...overrides }, [ + "push", + ]); + return { job, disposition: repositoryDisposition([{ jobs: [job] }]) }; +} + +test("option values cannot masquerade as informational or publishing verbs", () => { + for (const run of [ + "npm --prefix help ci", + "npm --prefix publish ci", + "pnpm --filter list install", + "npm --prefix=help ci", + ]) { + for (const steps of [[{ run }], [setup, install, { run }]]) { + assert.equal(inspect(steps).disposition, "needs-review", run); + } + } +}); + +test("scoped and abbreviated registry options cannot earn protected", () => { + for (const run of [ + "npm install @fixture/pkg --@fixture:registry=https://registry.example.invalid", + "npm ci --reg=https://registry.example.invalid", + "npm ci --regis=https://registry.example.invalid", + "pnpm install --config.registry=https://registry.example.invalid", + ]) { + const result = inspect([setup, { run }]); + assert.equal(result.disposition, "needs-sfw", run); + assert.match(result.job.violations.join(" "), /registry-mutating/); + } + for (const run of [ + "npm ci --userc=other.npmrc", + "npm ci --future-config=other", + ]) { + assert.equal(inspect([setup, { run }]).disposition, "needs-review", run); + } + assert.equal( + inspect([setup, { run: "npm ci --ignore-scripts --no-audit --no-fund" }]) + .disposition, + "protected", + ); +}); + +test("ecosystem executors remain candidates rather than being excluded", () => { + for (const run of [ + "uv run npm ci", + "poetry run npm ci", + "conda run npm ci", + "pipenv run npm ci", + "go run bootstrap.go", + "cargo run --bin installer", + ]) { + assert.equal(inspect([{ run }]).disposition, "needs-review", run); + assert.equal( + inspect([setup, install, { run }]).disposition, + "needs-review", + run, + ); + } + assert.equal( + inspect([{ run: "pip install -r requirements.txt" }]).job.status, + "out-of-scope", + ); +}); + +test("targeted Corepack controls cannot clear a different manager's state", () => { + for (const enable of ["corepack enable", "corepack enable pnpm"]) { + assert.equal( + inspect([ + { run: enable }, + { run: "corepack disable yarn" }, + setup, + { run: "pnpm install" }, + ]).disposition, + "needs-review", + ); + } + assert.equal( + inspect([ + { run: "corepack enable" }, + { run: "corepack disable" }, + setup, + { run: "pnpm install" }, + ]).disposition, + "protected", + ); +}); + +test("conditional protection and legitimate fork fallback are review, not known gaps", () => { + for (const step of [ + { ...setup, if: "github.ref == 'refs/heads/main'" }, + { ...setup, "continue-on-error": true }, + ]) + assert.equal(inspect([step, install]).disposition, "needs-review"); + assert.equal( + inspect( + [ + { + ...setup, + with: { + token: "${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }}", + "allow-external-fork-fallback": true, + }, + }, + install, + ], + { visibility: "public" }, + ).disposition, + "needs-review", + ); + assert.equal( + inspect([{ ...setup, if: false }, install]).disposition, + "needs-sfw", + ); +}); + +test("unsupported YAML never emits source-bearing warnings or returns a clean workflow", () => { + const emitted = []; + const emitWarning = process.emitWarning; + process.emitWarning = (...args) => emitted.push(args); + try { + const invalidSources = [ + "name: !private-canary private-workflow-canary\n", + "? [private, canary]\n: value\n", + "---\nname: second-document\n", + "name: [unterminated\n", + ]; + for (const invalid of invalidSources) { + const source = `on: push\njobs:\n build:\n steps:\n - run: echo done\n${invalid}`; + assert.equal( + repositoryDisposition([classifyWorkflow(source, context)]), + "needs-review", + ); + assert.throws(() => parseYamlSource(source)); + } + const localActions = new Map([ + [ + "local/action.yml", + `name: !private-canary private-composite-canary\nruns:\n using: composite\n steps:\n - uses: ${setup.uses}\n with:\n token: '${setup.with.token}'\n - run: npm ci\n shell: bash\n`, + ], + ]); + assert.equal( + inspect([{ uses: "./local" }], { localActions }).disposition, + "needs-review", + ); + assert.deepEqual(emitted, []); + } finally { + process.emitWarning = emitWarning; + } +}); + +test("malformed non-blob modes produce partial audit errors, never no-ci", async () => { + const repository = { + name: "fixture", + defaultBranch: "main", + visibility: "private", + }; + for (const entry of [ + { type: "tree", mode: "100644" }, + { type: "tree" }, + { type: "commit", mode: "100644" }, + { type: "commit" }, + ]) { + const client = { + listRestRepositories: async () => [ + { ...repository, default_branch: "main", archived: false }, + ], + listGraphqlRepositories: async () => [ + { ...repository, isArchived: false }, + ], + getRef: async () => ({ object: { sha: "a".repeat(40) } }), + getTree: async () => ({ + truncated: false, + tree: [{ path: ".github/workflows/ci.yml", ...entry }], + }), + getText: async () => { + throw new Error("must not read malformed tree source"); + }, + }; + assert.equal( + (await auditRepository(client, repository)).disposition, + "audit-error", + ); + const report = await runAudit(client); + assert.equal(report.scanStatus, "partial"); + assert.equal(report.scanErrors, 1); + assert.equal(scanExitCode(report), 1); + } +}); diff --git a/tools/rollout/release-snapshot.test.mjs b/tools/rollout/release-snapshot.test.mjs new file mode 100644 index 0000000..359f300 --- /dev/null +++ b/tools/rollout/release-snapshot.test.mjs @@ -0,0 +1,90 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { GitHubClient } from "./github.mjs"; +import { APPROVED_RUNTIME_BLOBS } from "./constants.mjs"; +import { verifyActionRelease } from "./release.mjs"; + +const snapshot = JSON.parse( + readFileSync(new URL("./fixtures/approved-release.json", import.meta.url)), +); +function client(data, historicalRefs = false) { + const prefix = `repos/${data.provenance.repository}`; + const replies = new Map([ + [`${prefix}/git/commits/${data.commit.sha}`, data.commit], + [`${prefix}/git/trees/${data.commit.tree.sha}?recursive=1`, data.tree], + ...Object.entries(data.contents).map(([path, body]) => [ + `${prefix}/contents/${path}?ref=${data.commit.sha}`, + body, + ]), + ...Object.values(data.discovery).map((body) => [ + `${prefix}/git/ref/${body.ref.slice(5)}`, + historicalRefs + ? { ...body, object: { type: "commit", sha: data.commit.sha } } + : body, + ]), + ]); + return new GitHubClient({ + execute: async (args) => { + assert.deepEqual(args.slice(0, 3), ["api", "--method", "GET"]); + assert.ok(replies.has(args[3]), args[3]); + return { stdout: JSON.stringify(replies.get(args[3])) }; + }, + }); +} +const verify = (data, historicalRefs = false) => + verifyActionRelease({ + client: client(data, historicalRefs), + manifestUrl: new URL("./fixtures/release-manifest.txt", import.meta.url), + }); + +test("captured immutable API shapes verify without manufacturing a tree from production constants", async () => { + // Only discovery refs are synthesized to the historical commit. The capture + // records the newer live refs separately; it does not claim they still match. + const result = await verify(snapshot, true); + assert.equal(result.sha, snapshot.commit.sha); + assert.equal(result.treeSha, snapshot.tree.sha); + for (const [path, sha] of Object.entries(APPROVED_RUNTIME_BLOBS)) + assert.equal( + snapshot.tree.tree.find((entry) => entry.path === path)?.sha, + sha, + ); +}); + +test("captured moved discovery refs fail the historical verifier without entering normal source CI", async () => { + assert.notEqual(snapshot.discovery.branch.object.sha, snapshot.commit.sha); + await assert.rejects(verify(snapshot), /unapproved SHA/); +}); + +test("captured responses retain signature, tree and content integrity failures", async () => { + for (const [mutate, message] of [ + [ + (data) => { + data.commit.verification.verified = false; + }, + /not GitHub-verified/, + ], + [ + (data) => { + data.tree.truncated = true; + }, + /truncated/, + ], + [ + (data) => { + data.tree.tree[0].mode = "120000"; + }, + /tree differs/, + ], + [ + (data) => { + data.contents["action.yml"].type = "symlink"; + }, + /not a complete base64/, + ], + ]) { + const data = structuredClone(snapshot); + mutate(data); + await assert.rejects(verify(data, true), message); + } +}); diff --git a/tools/rollout/release.mjs b/tools/rollout/release.mjs new file mode 100644 index 0000000..8d724f2 --- /dev/null +++ b/tools/rollout/release.mjs @@ -0,0 +1,208 @@ +import { readFile } from "node:fs/promises"; +import { parseYamlSource as parseYaml } from "./yaml.mjs"; + +import { + ACTION_REPOSITORY, + APPROVED_RELEASE_SHA, + EXPECTED_RELEASE_TREE, + RELEASE_BRANCH, + RELEASE_CHANNEL, +} from "./constants.mjs"; + +const DEFAULT_MANIFEST_URL = new URL( + "../../release-manifest.txt", + import.meta.url, +); +const FORBIDDEN_RELEASE_PATHS = [ + ".github/", + "package.json", + "package-lock.json", + "reports/", + "tools/", +]; + +function assert(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +function compareStrings(left, right) { + return left < right ? -1 : left > right ? 1 : 0; +} + +function sortedTreeEntries(entries) { + return entries + .map(({ mode, path, type }) => ({ mode, path, type })) + .sort((left, right) => compareStrings(left.path, right.path)); +} + +function assertExactTree(actual, expected) { + const actualJson = JSON.stringify(sortedTreeEntries(actual)); + const expectedJson = JSON.stringify(sortedTreeEntries(expected)); + assert( + actualJson === expectedJson, + `release tree differs from the reviewed manifest: expected ${expectedJson}, received ${actualJson}`, + ); +} + +function parseAction(text, path) { + let action; + try { + action = parseYaml(text); + } catch (error) { + throw new Error(`${path} is not valid YAML: ${error.message}`); + } + assert(action && typeof action === "object", `${path} is not a YAML object`); + assert( + action.runs?.using === "composite", + `${path} is not a composite action`, + ); + assert(Array.isArray(action.runs.steps), `${path} has no composite steps`); + return action; +} + +function assertRuntimeReference(action, actionPath, expectedCommand) { + const commands = action.runs.steps + .map((step) => step.run) + .filter((run) => typeof run === "string"); + assert( + commands.length === 1 && commands[0] === expectedCommand, + `${actionPath} must execute only ${expectedCommand}`, + ); + + const localUses = action.runs.steps + .map((step) => step.uses) + .filter((uses) => typeof uses === "string" && uses.startsWith(".")); + assert( + localUses.length === 0, + `${actionPath} contains an unexpected local action reference`, + ); +} + +export function parseReleaseManifest(text) { + const paths = []; + const seen = new Set(); + + for (const [index, rawLine] of text.split("\n").entries()) { + if (rawLine === "" || rawLine.startsWith("#")) { + continue; + } + const lineNumber = index + 1; + assert( + rawLine === rawLine.trim(), + `manifest line ${lineNumber} has whitespace`, + ); + assert(!rawLine.startsWith("/"), `manifest line ${lineNumber} is absolute`); + assert( + !rawLine.split("/").includes(".."), + `manifest line ${lineNumber} traverses outside the release`, + ); + assert(!seen.has(rawLine), `manifest contains duplicate path ${rawLine}`); + seen.add(rawLine); + paths.push(rawLine); + } + + assert(paths.length > 0, "release manifest is empty"); + return paths.sort(compareStrings); +} + +export async function verifyActionRelease(options) { + const client = options.client; + const manifestUrl = options.manifestUrl ?? DEFAULT_MANIFEST_URL; + const manifest = parseReleaseManifest(await readFile(manifestUrl, "utf8")); + + for (const forbidden of FORBIDDEN_RELEASE_PATHS) { + assert( + !manifest.some( + (path) => path === forbidden || path.startsWith(forbidden), + ), + `release manifest includes forbidden source path ${forbidden}`, + ); + } + + const expectedBlobPaths = EXPECTED_RELEASE_TREE.filter( + ({ type }) => type === "blob", + ) + .map(({ path }) => path) + .sort(compareStrings); + assert( + JSON.stringify(manifest) === JSON.stringify(expectedBlobPaths), + "local release manifest differs from the reviewed runtime allowlist", + ); + + const [branch, tag] = await Promise.all([ + client.getRef(ACTION_REPOSITORY, `heads/${RELEASE_BRANCH}`), + client.getRef(ACTION_REPOSITORY, `tags/${RELEASE_CHANNEL}`), + ]); + const branchSha = branch.object?.sha; + const tagSha = tag.object?.sha; + assert( + branch.object?.type === "commit" && tag.object?.type === "commit", + "release discovery refs must point directly to commits", + ); + assert(branchSha === tagSha, "release discovery refs do not match"); + assert( + branchSha === APPROVED_RELEASE_SHA, + `release discovery refs point to unapproved SHA ${branchSha ?? "missing"}`, + ); + + const commit = await client.getCommit( + ACTION_REPOSITORY, + APPROVED_RELEASE_SHA, + ); + assert( + commit.sha === APPROVED_RELEASE_SHA, + "release commit SHA does not match", + ); + assert( + commit.verification?.verified === true && + commit.verification?.reason === "valid", + "release commit is not GitHub-verified", + ); + assert( + typeof commit.tree?.sha === "string" && + /^[0-9a-f]{40}$/.test(commit.tree.sha), + "release commit has an invalid tree SHA", + ); + + const [tree, rootActionText, teardownActionText] = await Promise.all([ + client.getTree(ACTION_REPOSITORY, commit.tree.sha, true), + client.getText(ACTION_REPOSITORY, "action.yml", APPROVED_RELEASE_SHA), + client.getText( + ACTION_REPOSITORY, + "teardown/action.yml", + APPROVED_RELEASE_SHA, + ), + ]); + assert( + tree.sha === commit.tree.sha, + "release commit and recursive tree SHAs do not match", + ); + assert(tree.truncated === false, "release tree response is truncated"); + assert(Array.isArray(tree.tree), "release tree response has no entries"); + assertExactTree(tree.tree, EXPECTED_RELEASE_TREE); + + const rootAction = parseAction(rootActionText, "action.yml"); + const teardownAction = parseAction(teardownActionText, "teardown/action.yml"); + assertRuntimeReference( + rootAction, + "action.yml", + 'bash "$GITHUB_ACTION_PATH/scripts/configure.sh"', + ); + assertRuntimeReference( + teardownAction, + "teardown/action.yml", + 'bash "$GITHUB_ACTION_PATH/../scripts/teardown.sh"', + ); + + return { + branch: RELEASE_BRANCH, + channel: RELEASE_CHANNEL, + commitVerified: true, + manifest, + repository: ACTION_REPOSITORY, + sha: APPROVED_RELEASE_SHA, + treeSha: commit.tree?.sha, + }; +} diff --git a/tools/rollout/review-cli.mjs b/tools/rollout/review-cli.mjs new file mode 100644 index 0000000..af084a5 --- /dev/null +++ b/tools/rollout/review-cli.mjs @@ -0,0 +1,179 @@ +#!/usr/bin/env node +import { readFile, lstat, mkdir, open, realpath, rm } from "node:fs/promises"; +import { basename, dirname, join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { pathToFileURL } from "node:url"; +import { GitHubClient } from "./github.mjs"; +import { runAudit, writeReportAtomically } from "./audit.mjs"; +import { canonicalJson } from "./fingerprint.mjs"; +import { + advanceReview, + newReviewState, + recordDecision, + reviewView, + validateState, +} from "./review.mjs"; + +async function load(path) { + const stat = await lstat(path); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error("The ledger must be a regular file"); + return validateState(JSON.parse(await readFile(path, "utf8"))); +} + +// One writer across run/record/forget/init. A crashed writer leaves an explicit +// lock error, never an empty ledger or a lost acknowledgement. No TTL expiry. +export async function withLedgerLock(path, action) { + const lock = `${path}.lock`; + const handle = await open(lock, "wx", 0o600); + try { + await handle.writeFile(`${process.pid}\n`); + return await action(); + } finally { + await handle.close(); + await rm(lock); + } +} + +export async function reviewMain(argv, options = {}) { + const { values, positionals } = parseArgs({ + args: argv, + allowPositionals: true, + strict: true, + options: Object.fromEntries( + [ + "state", + "report", + "case", + "repository", + "kind", + "reason", + "evidence", + "by", + "expected", + ].map((name) => [name, { type: "string" }]), + ), + }); + const [command] = positionals; + const allowed = { + init: ["state"], + run: ["state", "report"], + show: ["state"], + record: [ + "state", + "case", + "repository", + "kind", + "reason", + "evidence", + "by", + "expected", + ], + forget: ["state", "case", "expected"], + }; + if ( + positionals.length !== 1 || + !allowed[command] || + !values.state || + Object.keys(values).some((key) => !allowed[command].includes(key)) + ) + throw new Error( + "usage: review --state [options]", + ); + const statePath = resolve(values.state); + const reportPath = resolve(values.report ?? "reports/review-audit.json"); + if ( + command === "run" && + [statePath, `${statePath}.lock`].includes(reportPath) + ) + throw new Error("Audit report and ledger/lock paths must differ"); + if (command === "init") + await mkdir(dirname(statePath), { recursive: true, mode: 0o700 }); + const output = options.output ?? process.stdout; + if (command === "show") { + const view = reviewView(await load(statePath)); + output.write(`${canonicalJson(view)}\n`); + return view; + } + return withLedgerLock(statePath, async () => { + let state; + if (command === "init") { + try { + await lstat(statePath); + throw new Error("Ledger already exists; init never replaces decisions"); + } catch (error) { + if (error.code !== "ENOENT") throw error; + } + state = newReviewState(); + } else { + state = await load(statePath); // Missing/corrupt state is an error, never []/auto-init. + if (command === "run") { + await mkdir(dirname(reportPath), { recursive: true, mode: 0o700 }); + const target = join( + await realpath(dirname(reportPath)), + basename(reportPath), + ); + const ledger = join( + await realpath(dirname(statePath)), + basename(statePath), + ); + if ([ledger, `${ledger}.lock`].includes(target)) + throw new Error("Audit report and ledger/lock paths must differ"); + const report = await runAudit(options.client ?? new GitHubClient(), { + progress: (done, total) => { + if (done % 25 === 0 || done === total) + (options.progress ?? process.stderr).write( + `audited ${done}/${total} repositories\n`, + ); + }, + }); + await writeReportAtomically(reportPath, report); + state = advanceReview(state, report); // Partial results never replace last-good ledger. + } else if (command === "record") { + state = recordDecision(state, { + id: values.case, + repository: values.repository, + kind: values.kind, + reason: values.reason, + evidence: values.evidence, + recordedBy: values.by, + expected: values.expected, + }); + } else { + if ( + reviewView(state).snapshot !== values.expected || + !Object.hasOwn(state.decisions, values.case ?? "") + ) + throw new Error( + "Current snapshot and an existing exact decision ID are required to forget", + ); + delete state.decisions[values.case]; + } + } + await writeReportAtomically(statePath, JSON.parse(canonicalJson(state))); + const view = reviewView(state); + output.write(`${canonicalJson(view)}\n`); + return view; + }); +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + reviewMain(process.argv.slice(2)).catch((error) => { + // No source/API body in logs, and never a success-shaped empty finding list. + console.error( + JSON.stringify({ + status: "failed", + error: + error.code === "EEXIST" + ? "Ledger locked; verify no writer is active before removing its .lock file." + : error.code === "ERR_ASSERTION" + ? error.message + : "Review failed; check arguments, ledger, permissions, API availability, and private audit scanStatus. Existing ledger was not reset.", + }), + ); + process.exitCode = 1; + }); +} diff --git a/tools/rollout/review.mjs b/tools/rollout/review.mjs new file mode 100644 index 0000000..62cf1c1 --- /dev/null +++ b/tools/rollout/review.mjs @@ -0,0 +1,376 @@ +import assert from "node:assert/strict"; +import { fingerprint } from "./fingerprint.mjs"; +import { ORGANIZATION } from "./constants.mjs"; + +const HASH = /^[0-9a-f]{64}$/; +const statuses = new Set(["needs-sfw", "needs-review"]); +const kinds = new Set(["known-review", "tracked-gap", "exception"]); +const mapping = (value) => + value && typeof value === "object" && !Array.isArray(value); +const sorted = (rows) => + [...rows].sort((a, b) => { + const left = `${a.repository}/${a.path ?? ""}/${a.job ?? ""}/${a.id ?? ""}`; + const right = `${b.repository}/${b.path ?? ""}/${b.job ?? ""}/${b.id ?? ""}`; + return left < right ? -1 : left > right ? 1 : 0; + }); + +export function newReviewState() { + return { + schemaVersion: 1, + organization: ORGANIZATION, + decisions: {}, + findings: [], + unobserved: [], + }; +} + +function validRepository(value) { + return ( + Number.isSafeInteger(value.repositoryId) && + value.repositoryId > 0 && + typeof value.repository === "string" && + /^workos\/[\w.-]+$/.test(value.repository) + ); +} + +export function validateState(state) { + assert( + mapping(state) && + state.schemaVersion === 1 && + state.organization === ORGANIZATION, + "Unsupported review ledger", + ); + assert( + Object.keys(state).every((key) => + [ + "schemaVersion", + "organization", + "decisions", + "findings", + "unobserved", + ].includes(key), + ), + "Unknown ledger fields", + ); + assert( + mapping(state.decisions) && + Array.isArray(state.findings) && + Array.isArray(state.unobserved), + "Malformed review ledger", + ); + assert( + new Set(state.findings.map((item) => item.id)).size === + state.findings.length, + "Duplicate review finding", + ); + for (const item of state.findings) { + assert( + validRepository(item) && + HASH.test(item.id) && + HASH.test(item.fingerprint) && + statuses.has(item.status), + "Malformed review finding", + ); + assert( + ["job", "workflow", "exclusion"].includes(item.kind) && + typeof item.path === "string" && + (item.job === null || typeof item.job === "string"), + "Malformed finding scope", + ); + assert( + item.id === + fingerprint([item.repositoryId, item.kind, item.path, item.job]), + "Finding identity does not match its scope", + ); + assert( + Array.isArray(item.reasons) && + item.reasons.every((reason) => typeof reason === "string"), + "Malformed finding reasons", + ); + } + for (const [id, decision] of Object.entries(state.decisions)) { + assert( + HASH.test(id) && + validRepository(decision) && + HASH.test(decision.fingerprint) && + statuses.has(decision.status) && + kinds.has(decision.kind), + "Malformed review decision", + ); + assert( + mapping(decision.scope) && + ["job", "workflow", "exclusion"].includes(decision.scope.kind) && + typeof decision.scope.path === "string" && + (decision.scope.job === null || typeof decision.scope.job === "string"), + "Missing exact decision scope", + ); + assert( + id === + fingerprint([ + decision.repositoryId, + decision.scope.kind, + decision.scope.path, + decision.scope.job, + ]), + "Decision identity does not match its scope", + ); + assert( + decision.kind === "exception" || + decision.status === + (decision.kind === "known-review" ? "needs-review" : "needs-sfw"), + "Decision kind conflicts with classification", + ); + assert(typeof decision.active === "boolean", "Missing decision validity"); + validateRationale(decision); + } + assert( + state.unobserved.every(validRepository), + "Malformed visibility history", + ); + return state; +} + +function validateRationale({ reason, evidence, recordedBy }) { + assert( + typeof reason === "string" && + reason.trim().length >= 10 && + reason.length <= 4000, + "A specific reason is required", + ); + assert( + typeof recordedBy === "string" && + recordedBy.trim() && + recordedBy.length <= 200, + "Recorded-by identity is required", + ); + const url = new URL(evidence); + assert( + url.protocol === "https:" && + !url.username && + !url.password && + evidence.length <= 2048, + "Evidence must be a credential-free HTTPS URL", + ); +} + +function collect(report) { + assert( + report.schemaVersion === 3 && + report.organization === ORGANIZATION && + report.scanStatus === "complete" && + report.scanErrors === 0, + "Incomplete audit: ledger and last-good review must not advance", + ); + assert( + Array.isArray(report.repositories) && + report.repositories.length === report.inventory.activeCount, + "Incomplete repository inventory", + ); + assert( + Object.values(report.inventory.differences).every( + (values) => Array.isArray(values) && values.length === 0, + ), + "Inventory sources disagree", + ); + const seen = new Map(); + const facts = new Map(); + const findings = []; + const add = (repo, kind, path, job, status, source, reasons) => { + assert( + HASH.test(source), + "Audit lacks review fingerprints; run the current detector", + ); + const item = { + id: fingerprint([repo.repositoryId, kind, path, job]), + repositoryId: repo.repositoryId, + repository: `${ORGANIZATION}/${repo.name}`, + kind, + path, + job, + status, + fingerprint: source, + reasons: [...new Set(reasons)].sort(), + }; + assert(!facts.has(item.id), "Duplicate finding scope"); + facts.set(item.id, item); + if (statuses.has(status)) findings.push(item); + }; + for (const repo of report.repositories) { + assert( + validRepository({ + repositoryId: repo.repositoryId, + repository: `${ORGANIZATION}/${repo.name}`, + }) && !seen.has(repo.repositoryId), + "Missing or duplicate immutable repository ID", + ); + assert(repo.disposition !== "audit-error", "Partial repository scan"); + seen.set(repo.repositoryId, repo.name); + for (const workflow of repo.workflows) { + if (workflow.parseError !== undefined) + add( + repo, + "workflow", + workflow.path, + null, + "needs-review", + workflow.reviewFingerprint, + ["workflow-source-unparsed"], + ); + for (const job of workflow.jobs) { + const integration = job.integration; + assert( + integration && typeof integration.disposition === "string", + "Missing job classification", + ); + add( + repo, + "job", + workflow.path, + job.job, + integration.disposition, + job.reviewFingerprint, + [ + ...(integration.downloads ?? []) + .filter((item) => ["gap", "unresolved"].includes(item.status)) + .map((item) => item.reason), + ...(integration.additionalJsPaths ?? []) + .filter((item) => item.status === "unresolved") + .map((item) => item.reason), + ...job.operations + .filter((item) => item.sourceError) + .map((item) => item.sourceError), + ...(job.operations.some((item) => item.kind === "reusable-call") + ? ["reusable-workflow-context"] + : []), + ], + ); + } + } + for (const exclusion of repo.exclusions ?? []) { + add( + repo, + "exclusion", + exclusion.id, + null, + exclusion.status === "stale" + ? "needs-review" + : "integrated-with-exclusions", + exclusion.reviewFingerprint, + ["approved-exclusion-drift"], + ); + } + } + return { seen, facts, findings: sorted(findings) }; +} + +// Acknowledgements bind to source inputs, never a repository-wide waiver or a +// last-seen timestamp. Missing visibility is not evidence of resolution. +export function advanceReview(previous, report) { + validateState(previous); + const { seen, facts, findings } = collect(report); + const state = structuredClone(previous); + for (const [id, decision] of Object.entries(state.decisions)) { + if (!seen.has(decision.repositoryId)) continue; + const fact = facts.get(id); + if ( + !fact || + fact.fingerprint !== decision.fingerprint || + (statuses.has(fact.status) && fact.status !== decision.status) + ) + decision.active = false; + decision.repository = `${ORGANIZATION}/${seen.get(decision.repositoryId)}`; + } + const prior = [ + ...previous.findings, + ...Object.values(previous.decisions), + ...previous.unobserved, + ]; + state.unobserved = sorted([ + ...new Map( + prior + .filter((item) => !seen.has(item.repositoryId)) + .map((item) => [ + item.repositoryId, + { repositoryId: item.repositoryId, repository: item.repository }, + ]), + ).values(), + ]); + state.findings = findings; + return validateState(state); +} + +export function reviewView(state) { + validateState(state); + const pending = []; + const known = []; + for (const item of sorted(state.findings)) { + const decision = state.decisions[item.id]; + if ( + decision?.active && + decision.fingerprint === item.fingerprint && + decision.status === item.status + ) { + known.push({ + ...item, + decision: { + kind: decision.kind, + reason: decision.reason, + evidence: decision.evidence, + recordedBy: decision.recordedBy, + }, + }); + } else pending.push(item); + } + return { + schemaVersion: 1, + snapshot: fingerprint(sorted(state.findings)), + needsProtection: pending.filter((item) => item.status === "needs-sfw"), + needsReview: pending.filter((item) => item.status === "needs-review"), + known, + unobserved: sorted(state.unobserved), + }; +} + +export function recordDecision( + previous, + { expected, id, repository, kind, reason, evidence, recordedBy }, +) { + validateState(previous); + assert( + HASH.test(expected) && reviewView(previous).snapshot === expected, + "Review snapshot changed; inspect it before recording a decision", + ); + assert( + Boolean(id) !== Boolean(repository) && kinds.has(kind), + "Select one exact case or current repository findings and a valid decision kind", + ); + validateRationale({ reason, evidence, recordedBy }); + const selected = previous.findings.filter((item) => + id ? item.id === id : item.repository === repository, + ); + assert(selected.length > 0, "No current finding matches this selection"); + assert( + selected.every( + (item) => + kind === "exception" || + item.status === + (kind === "known-review" ? "needs-review" : "needs-sfw"), + ), + "Use known-review for uncertainty, tracked-gap for unfinished protection, or an explicitly justified exception", + ); + const state = structuredClone(previous); + for (const item of selected) + state.decisions[item.id] = { + repositoryId: item.repositoryId, + repository: item.repository, + fingerprint: item.fingerprint, + scope: { kind: item.kind, path: item.path, job: item.job }, + status: item.status, + kind, + reason, + evidence, + recordedBy, + active: true, + }; + return validateState(state); +} diff --git a/tools/rollout/review.test.mjs b/tools/rollout/review.test.mjs new file mode 100644 index 0000000..5bacbb3 --- /dev/null +++ b/tools/rollout/review.test.mjs @@ -0,0 +1,642 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + mkdtemp, + readFile, + rm, + stat, + writeFile, + symlink, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { stringify } from "yaml"; +import { runAudit } from "./audit.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +import { REGISTRY_EXCLUSIONS } from "./exclusions.mjs"; +import { GitHubClient, GhCommandError } from "./github.mjs"; +import { canonicalJson, fingerprint } from "./fingerprint.mjs"; +import { + advanceReview, + newReviewState, + recordDecision, + reviewView, +} from "./review.mjs"; +import { reviewMain, withLedgerLock } from "./review-cli.mjs"; + +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}" }, +}; +const job = { + "runs-on": "ubuntu-latest", + steps: [{ uses: "actions/checkout@v4" }, { run: "npm ci" }], +}; +const workflow = (jobs = { install: job }) => stringify({ on: "push", jobs }); +function client({ + files = { ".github/workflows/ci.yml": workflow() }, + head = "a".repeat(40), + id = 42, + name = "demo", + partial = false, + visible = true, + reverse = false, +} = {}) { + return { + async listRestRepositories() { + return visible + ? [ + { + id, + name, + default_branch: "main", + visibility: "private", + archived: false, + }, + ] + : []; + }, + async listGraphqlRepositories() { + return visible + ? [{ name, visibility: "PRIVATE", isArchived: false }] + : []; + }, + async getRef() { + return { object: { sha: head } }; + }, + async getTree() { + const tree = Object.entries(files).map(([path, text]) => ({ + path, + mode: "100644", + type: "blob", + sha: fingerprint(text).slice(0, 40), + })); + return { truncated: partial, tree: reverse ? tree.reverse() : tree }; + }, + async getText(_repo, path, ref) { + assert.equal(ref, head); + assert.ok(Object.hasOwn(files, path)); + return files[path]; + }, + }; +} +const scan = (options) => runAudit(client(options)); +const reason = { + reason: "Tracked fixture gap with a separate remediation PR.", + evidence: "https://github.com/workos/example/pull/1", + recordedBy: "test reviewer", +}; +function acknowledge(state, extra = {}) { + return recordDecision(state, { + expected: reviewView(state).snapshot, + id: state.findings[0].id, + kind: "tracked-gap", + ...reason, + ...extra, + }); +} + +test("bare transport EOF is retried within the existing bound, not mistaken for changed repository state", async () => { + let calls = 0; + const delays = []; + const api = new GitHubClient({ + execute: async () => { + calls++; + if (calls === 1) + throw new GhCommandError( + 'Get "https://api.github.com/repos/owner/example": EOF', + ); + return { stdout: "{}" }; + }, + sleep: async (delay) => { + delays.push(delay); + }, + }); + assert.deepEqual(await api.api("repos/owner/example"), {}); + assert.equal(calls, 2); + assert.deepEqual(delays, [5000]); + calls = 0; + delays.length = 0; + api.execute = async () => { + calls++; + throw new GhCommandError("EOF"); + }; + await assert.rejects(api.api("repos/owner/example"), /failed/); + assert.equal(calls, 4); + assert.deepEqual(delays, [5000, 10000, 20000]); + calls = 0; + api.execute = async () => { + calls++; + throw new GhCommandError("denied", { status: 403 }); + }; + await assert.rejects(api.api("repos/owner/example"), /denied/); + assert.equal(calls, 1); +}); + +test("weekly findings and decisions are stable across order, timestamps, unrelated commits, YAML comments and other jobs", async () => { + const first = advanceReview(newReviewState(), await scan()); + assert.equal(reviewView(first).needsProtection.length, 1); + const known = acknowledge(first); + const exception = acknowledge(first, { kind: "exception" }); + assert.equal( + reviewView(advanceReview(exception, await scan())).known[0].decision.kind, + "exception", + ); + assert.equal(reviewView(exception).known[0].status, "needs-sfw"); + const second = await scan({ + head: "b".repeat(40), + reverse: true, + files: { + "README.md": "unrelated documentation", + ".github/workflows/ci.yml": + "# a YAML comment\n" + + workflow({ + unrelated: { steps: [{ run: "echo okay" }] }, + install: job, + }), + }, + }); + second.generatedAt = "another week"; + const repeated = advanceReview(known, second); + assert.equal(canonicalJson(repeated), canonicalJson(known)); + assert.equal( + canonicalJson(reviewView(repeated)), + canonicalJson(reviewView(known)), + ); + assert.equal(reviewView(repeated).known[0].status, "needs-sfw"); // Not falsely relabelled integrated. +}); + +test("new jobs, new repos and changed existing installs cannot inherit an acknowledgement", async () => { + const known = acknowledge(advanceReview(newReviewState(), await scan())); + const added = advanceReview( + known, + await scan({ + files: { + ".github/workflows/ci.yml": workflow({ install: job, another: job }), + }, + }), + ); + assert.equal(reviewView(added).known.length, 1); + assert.equal(reviewView(added).needsProtection.length, 1); + const changed = advanceReview( + known, + await scan({ + files: { + ".github/workflows/ci.yml": workflow({ + install: { + ...job, + steps: [...job.steps, { run: "npm install another" }], + }, + }), + }, + }), + ); + assert.equal(reviewView(changed).needsProtection.length, 1); + assert.equal(Object.values(changed.decisions)[0].active, false); + assert.equal( + reviewView(advanceReview(changed, await scan())).needsProtection.length, + 1, + "reverting inputs does not silently reactivate a stale decision", + ); + const recreated = advanceReview(known, await scan({ id: 43 })); + assert.equal(reviewView(recreated).needsProtection.length, 1); + assert.equal(recreated.unobserved[0].repositoryId, 42); + const renamed = advanceReview(known, await scan({ name: "renamed" })); + assert.equal(reviewView(renamed).needsProtection.length, 0); + assert.equal(reviewView(renamed).known[0].repository, "workos/renamed"); +}); + +test("resolution followed by regression reopens; absent visibility does not resolve, expire or discard decisions", async () => { + const known = acknowledge(advanceReview(newReviewState(), await scan())); + const protectedReport = await scan({ + files: { + ".github/workflows/ci.yml": workflow({ + install: { ...job, steps: [setup, ...job.steps] }, + }), + }, + }); + const resolved = advanceReview(known, protectedReport); + assert.equal(resolved.findings.length, 0); + assert.equal( + reviewView(advanceReview(resolved, await scan())).needsProtection.length, + 1, + ); + const missing = advanceReview(known, await scan({ visible: false })); + assert.equal(missing.unobserved.length, 1); + assert.equal(Object.values(missing.decisions)[0].active, true); + assert.deepEqual( + advanceReview(missing, await scan({ visible: false })), + missing, + ); + assert.deepEqual(advanceReview(missing, await scan()), known); +}); + +test("only referenced helpers, configuration and upstream producers invalidate their reviewed consumers", async () => { + const files = { + ".github/workflows/ci.yml": workflow({ + producer: { steps: [{ run: "echo value=one" }] }, + install: { + ...job, + needs: "producer", + steps: [job.steps[0], { uses: "./.github/actions/install" }], + }, + }), + ".github/actions/install/action.yml": stringify({ + runs: { using: "composite", steps: [{ shell: "bash", run: "npm ci" }] }, + }), + ".github/actions/unused/action.yml": "unused", + ".npmrc": "registry=https://registry.npmjs.org/", + }; + const known = acknowledge( + advanceReview(newReviewState(), await scan({ files })), + ); + assert.deepEqual( + advanceReview( + known, + await scan({ + files: { ...files, ".github/actions/unused/action.yml": "changed" }, + reverse: true, + }), + ), + known, + ); + for (const [path, value] of [ + [ + ".github/actions/install/action.yml", + files[".github/actions/install/action.yml"].replace( + "npm ci", + "npm install", + ), + ], + [".npmrc", "registry=https://other.invalid/"], + [ + ".github/workflows/ci.yml", + files[".github/workflows/ci.yml"].replace("value=one", "value=two"), + ], + ]) + assert.equal( + Object.values( + advanceReview(known, await scan({ files: { ...files, [path]: value } })) + .decisions, + )[0].active, + false, + path, + ); +}); + +test("review uncertainty is separately acknowledged, not a blanket exception for a later proven gap", async () => { + const files = { + ".github/workflows/ci.yml": workflow({ + install: { ...job, steps: [{ uses: "./missing" }] }, + }), + }; + const initial = advanceReview(newReviewState(), await scan({ files })); + assert.equal(reviewView(initial).needsReview.length, 1); + assert.throws(() => acknowledge(initial), /known-review/); + const known = acknowledge(initial, { kind: "known-review" }); + assert.equal( + reviewView(advanceReview(known, await scan())).needsProtection.length, + 1, + ); + assert.throws( + () => + acknowledge(initial, { kind: "known-review", expected: "0".repeat(64) }), + /snapshot changed/, + ); + assert.throws( + () => acknowledge(initial, { kind: "known-review", reason: "" }), + /reason/, + ); + assert.throws( + () => + acknowledge(initial, { + kind: "known-review", + evidence: "https://user:secret@example.com", + }), + /credential-free/, + ); + assert.throws( + () => acknowledge(initial, { kind: "known-review", id: "*" }), + /No current finding/, + ); +}); + +test("partial reports, inventory disagreements, missing IDs and legacy reports cannot reset good state", async () => { + const state = acknowledge(advanceReview(newReviewState(), await scan())); + const bytes = canonicalJson(state); + assert.throws( + () => + advanceReview(state, { + schemaVersion: 3, + organization: "workos", + scanStatus: "partial", + scanErrors: 1, + }), + /Incomplete audit/, + ); + for (const change of [ + (r) => { + delete r.repositories[0].repositoryId; + }, + (r) => { + delete r.repositories[0].workflows[0].jobs[0].reviewFingerprint; + }, + (r) => { + r.inventory.differences.activeOnlyRest = ["demo"]; + }, + (r) => { + r.repositories.push(r.repositories[0]); + r.inventory.activeCount++; + }, + ]) { + const report = await scan(); + change(report); + assert.throws(() => advanceReview(state, report)); + } + assert.equal(canonicalJson(state), bytes); +}); + +test("upstream helper/reusable changes and stale archive inputs reopen their recorded scopes", async () => { + const files = { + ".github/workflows/ci.yml": workflow({ + producer: { uses: "./.github/workflows/producer.yml" }, + install: { ...job, needs: "producer" }, + }), + ".github/workflows/producer.yml": stringify({ + on: "workflow_call", + jobs: { + value: { + steps: [ + { + uses: "actions/checkout@v4", + with: { repository: "workos/demo" }, + }, + { uses: "./.github/actions/producer" }, + ], + }, + }, + }), + ".github/actions/producer/action.yml": stringify({ + runs: { + using: "composite", + steps: [{ shell: "bash", run: "echo value=one" }], + }, + }), + }; + const first = advanceReview(newReviewState(), await scan({ files })); + const gap = first.findings.find((item) => item.status === "needs-sfw"); + const known = acknowledge(first, { id: gap.id }); + const changed = advanceReview( + known, + await scan({ + files: { + ...files, + ".github/actions/producer/action.yml": files[ + ".github/actions/producer/action.yml" + ].replace("value=one", "value=two"), + }, + }), + ); + assert.equal(changed.decisions[gap.id].active, false); + + const archiveFiles = { + ".github/workflows/ci.yml": workflow(), + "package.json": "{}", + ".npmrc": "registry=https://registry.npmjs.org/", + "package-lock.json": JSON.stringify({ + lockfileVersion: 3, + packages: { + "": {}, + "node_modules/example": { + version: "1.0.0", + resolved: "https://example.invalid/archive.tgz", + }, + }, + }), + }; + const stale = advanceReview( + newReviewState(), + await scan({ files: archiveFiles, name: "openapi-spec" }), + ); + const archive = stale.findings.find((item) => item.kind === "exclusion"); + const recorded = acknowledge(stale, { id: archive.id, kind: "known-review" }); + assert.deepEqual( + advanceReview( + recorded, + await scan({ + files: archiveFiles, + name: "openapi-spec", + head: "b".repeat(40), + }), + ), + recorded, + ); + const drift = advanceReview( + recorded, + await scan({ + name: "openapi-spec", + files: { + ...archiveFiles, + "package-lock.json": archiveFiles["package-lock.json"].replace( + "1.0.0", + "2.0.0", + ), + }, + }), + ); + assert.equal(drift.decisions[archive.id].active, false); +}); + +test("accepted WorkOS cases ignore registry override versions, not installation policy changes", async () => { + const rule = REGISTRY_EXCLUSIONS.find((item) => item.repository === "workos"); + const manifest = { + globalOverrides: { + "tree-sitter-kotlin": rule.specifier, + "body-parser@1": "1.20.6", + alias: "npm:one@^1.0.0", + }, + }; + const lock = { + lockfileVersion: "9.0", + overrides: manifest.globalOverrides, + packages: { + [rule.packagePath]: { + version: rule.version, + resolution: { + gitHosted: true, + integrity: rule.integrity, + tarball: rule.resolved, + }, + }, + }, + }; + const run = () => + scan({ + name: "workos", + files: { + ".github/workflows/ci.yml": workflow({ + install: { steps: [job.steps[0], { uses: "./missing" }] }, + }), + [rule.manifest]: JSON.stringify(manifest), + [rule.lockfile]: stringify(lock), + }, + }); + const accepted = acknowledge(advanceReview(newReviewState(), await run()), { + kind: "exception", + }); + manifest.globalOverrides["body-parser@1"] = "1.20.8"; + manifest.globalOverrides.alias = "npm:one@^2.0.0"; + const repeated = advanceReview(accepted, await run()); + assert.equal(canonicalJson(repeated), canonicalJson(accepted)); + manifest.globalOverrides.alias = "npm:two@^2.0.0"; + assert.equal( + reviewView(advanceReview(repeated, await run())).needsReview.length, + 1, + ); + manifest.globalOverrides.alias = "npm:one@^2.0.0"; + manifest.globalOnlyBuiltDependencies = ["new-install-script"]; + assert.equal( + reviewView(advanceReview(repeated, await run())).needsReview.length, + 1, + ); + delete manifest.globalOnlyBuiltDependencies; + manifest.globalOverrides["tree-sitter-kotlin"] = + "github:other/repo#unapproved"; + assert.ok( + reviewView(advanceReview(repeated, await run())).needsReview.some( + (item) => item.kind === "exclusion", + ), + ); +}); + +test("accepted OpenAPI cases survive registry bumps but not new source exceptions or routing changes", async () => { + const rule = REGISTRY_EXCLUSIONS.find( + (item) => item.repository === "openapi-spec", + ); + const files = { + ".github/workflows/ci.yml": workflow({ + install: { + steps: [job.steps[0], setup, { run: "npm ci" }, { uses: "./missing" }], + }, + }), + ".npmrc": rule.projectNpmrc.join("\n"), + }; + const dependencies = { + ordinary: "1.0.0", + "tree-sitter-kotlin": rule.resolved, + }; + const manifest = { dependencies, overrides: { "js-yaml": "^5.4.2" } }; + const lock = { + lockfileVersion: 3, + packages: { + "": { dependencies }, + "node_modules/ordinary": { + version: "1.0.0", + resolved: "https://registry.npmjs.org/ordinary/-/ordinary-1.0.0.tgz", + }, + [rule.packagePath]: { + version: rule.version, + resolved: rule.resolved, + integrity: rule.integrity, + }, + }, + }; + const run = (config = files[".npmrc"]) => + scan({ + name: "openapi-spec", + files: { + ...files, + ".npmrc": config, + "package.json": JSON.stringify(manifest), + "package-lock.json": JSON.stringify(lock), + }, + }); + const accepted = acknowledge(advanceReview(newReviewState(), await run()), { + kind: "exception", + }); + dependencies.ordinary = "2.0.0"; + manifest.overrides["js-yaml"] = "^5.4.3"; + lock.packages["node_modules/ordinary"] = { + version: "2.0.0", + resolved: "https://registry.npmjs.org/ordinary/-/ordinary-2.0.0.tgz", + }; + const repeated = advanceReview(accepted, await run()); + assert.equal(canonicalJson(repeated), canonicalJson(accepted)); + assert.equal(reviewView(repeated).known[0].decision.kind, "exception"); + lock.packages[rule.packagePath].integrity = "sha512-unapproved"; + const changedSource = reviewView(advanceReview(repeated, await run())); + assert.equal(changedSource.needsReview.length, 1); + assert.equal(changedSource.needsReview[0].kind, "exclusion"); + assert.ok( + reviewView( + advanceReview(repeated, await run("registry=https://other.invalid/")), + ).needsReview.some((item) => item.kind === "job"), + ); +}); + +test("CLI persistence survives restart; missing/corrupt state, races and partial scans fail without replacing decisions", async () => { + const dir = await mkdtemp(join(tmpdir(), "sfw-weekly-")); + const path = join(dir, "state.json"); + const report = join(dir, "audit.json"); + const options = { + client: client(), + output: { write() {} }, + progress: { write() {} }, + }; + const run = (command, args = [], extra = {}) => + reviewMain([command, "--state", path, ...args], { ...options, ...extra }); + try { + await assert.rejects(run("run", ["--report", report]), /ENOENT/); + await run("init"); + const first = await run("run", ["--report", report]); + await run("record", [ + "--expected", + first.snapshot, + "--case", + first.needsProtection[0].id, + "--kind", + "tracked-gap", + "--reason", + reason.reason, + "--evidence", + reason.evidence, + "--by", + reason.recordedBy, + ]); + const bytes = await readFile(path, "utf8"); + assert.equal((await stat(path)).mode & 0o777, 0o600); + assert.equal((await stat(report)).mode & 0o777, 0o600); + assert.equal((await run("show")).known.length, 1); + await run("run", ["--report", report], { + client: client({ head: "b".repeat(40) }), + }); + assert.equal(await readFile(path, "utf8"), bytes); + await assert.rejects(run("init"), /already exists/); + await assert.rejects(run("run", ["--report", path]), /must differ/); + const alias = join(dir, "alias"); + await symlink(dir, alias); + await assert.rejects( + run("run", ["--report", join(alias, "state.json")]), + /must differ/, + ); + await assert.rejects( + run("run", ["--report", report], { client: client({ partial: true }) }), + /Incomplete audit/, + ); + assert.equal(await readFile(path, "utf8"), bytes); + await withLedgerLock(path, async () => { + await assert.rejects(run("run", ["--report", report]), { + code: "EEXIST", + }); + }); + assert.equal(await readFile(path, "utf8"), bytes); + await writeFile(path, "{bad json"); + await assert.rejects(run("run", ["--report", report])); + assert.equal(await readFile(path, "utf8"), "{bad json"); + await rm(path); + await symlink(report, path); + await assert.rejects(run("show"), /regular file/); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); diff --git a/tools/rollout/rollout.test.mjs b/tools/rollout/rollout.test.mjs new file mode 100644 index 0000000..80fcf20 --- /dev/null +++ b/tools/rollout/rollout.test.mjs @@ -0,0 +1,439 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { describe, test } from "node:test"; + +import { + ACTION_REPOSITORY, + APPROVED_RELEASE_SHA, + EXPECTED_RELEASE_TREE, + RELEASE_BRANCH, + RELEASE_CHANNEL, +} from "./constants.mjs"; +import { GhCommandError, GitHubClient } from "./github.mjs"; +import { + captureRepositoryInventory, + reconcileRepositoryInventories, +} from "./inventory.mjs"; +import { main } from "./cli.mjs"; +import { parseReleaseManifest, verifyActionRelease } from "./release.mjs"; + +// Unit tests intentionally use a historical snapshot, not the evolving source +// manifest or live discovery refs. The manual verifier remains strict. +function verifyFixtureRelease(options) { + return verifyActionRelease({ + ...options, + manifestUrl: new URL("./fixtures/release-manifest.txt", import.meta.url), + }); +} + +const TREE_SHA = "5cdbe39b0edafee9457767134320d95c61d91a60"; + +const ROOT_ACTION = ` +name: Setup Socket Firewall +runs: + using: composite + steps: + - shell: bash + run: bash "$GITHUB_ACTION_PATH/scripts/configure.sh" +`; +const TEARDOWN_ACTION = ` +name: Teardown Socket Firewall +runs: + using: composite + steps: + - shell: bash + run: bash "$GITHUB_ACTION_PATH/../scripts/teardown.sh" +`; + +function releaseClient(overrides = {}) { + return { + async getCommit() { + return ( + overrides.commit ?? { + sha: APPROVED_RELEASE_SHA, + tree: { sha: TREE_SHA }, + verification: { reason: "valid", verified: true }, + } + ); + }, + async getRef(_repository, ref) { + const defaultResponse = { + object: { sha: APPROVED_RELEASE_SHA, type: "commit" }, + }; + return overrides.refs?.[ref] ?? defaultResponse; + }, + async getText(_repository, path) { + if (path === "action.yml") { + return overrides.rootAction ?? ROOT_ACTION; + } + if (path === "teardown/action.yml") { + return overrides.teardownAction ?? TEARDOWN_ACTION; + } + throw new Error(`unexpected path ${path}`); + }, + async getTree() { + return ( + overrides.tree ?? { + sha: TREE_SHA, + tree: EXPECTED_RELEASE_TREE, + truncated: false, + } + ); + }, + }; +} + +function restRepository(name, options = {}) { + return { + archived: options.archived ?? false, + default_branch: options.defaultBranch ?? "main", + name, + visibility: options.visibility ?? "private", + }; +} + +function graphqlRepository(name, options = {}) { + return { + isArchived: options.archived ?? false, + name, + visibility: (options.visibility ?? "private").toUpperCase(), + }; +} + +describe("source dependency lockfile", () => { + test("contains no Socket Firewall resolution URL", async () => { + const lockfile = await readFile( + new URL("../../package-lock.json", import.meta.url), + "utf8", + ); + assert.doesNotMatch( + lockfile, + /https?:\/\/[^/]*(?:socket-firewall|socket\.dev)/i, + ); + }); +}); + +describe("release verification", () => { + test("accepts the exact signed action-only release", async () => { + const result = await verifyFixtureRelease({ client: releaseClient() }); + + assert.deepEqual(result, { + branch: RELEASE_BRANCH, + channel: RELEASE_CHANNEL, + commitVerified: true, + manifest: [ + "LICENSE", + "action.yml", + "scripts/configure.sh", + "scripts/teardown.sh", + "teardown/action.yml", + ], + repository: ACTION_REPOSITORY, + sha: APPROVED_RELEASE_SHA, + treeSha: TREE_SHA, + }); + }); + + test("rejects mismatched discovery refs", async () => { + const client = releaseClient({ + refs: { + [`tags/${RELEASE_CHANNEL}`]: { + object: { sha: "f".repeat(40), type: "commit" }, + }, + }, + }); + + await assert.rejects( + verifyFixtureRelease({ client }), + /release discovery refs do not match/, + ); + }); + + test("rejects an unverified release commit", async () => { + const client = releaseClient({ + commit: { + sha: APPROVED_RELEASE_SHA, + tree: { sha: TREE_SHA }, + verification: { reason: "unsigned", verified: false }, + }, + }); + + await assert.rejects( + verifyFixtureRelease({ client }), + /release commit is not GitHub-verified/, + ); + }); + + test("rejects extra release-tree content", async () => { + const client = releaseClient({ + tree: { + sha: TREE_SHA, + tree: [ + ...EXPECTED_RELEASE_TREE, + { mode: "100644", path: "tools/audit.mjs", type: "blob" }, + ], + truncated: false, + }, + }); + + await assert.rejects( + verifyFixtureRelease({ client }), + /release tree differs from the reviewed manifest/, + ); + }); + + test("rejects action metadata that executes another local path", async () => { + const client = releaseClient({ + rootAction: ` +name: Unsafe +runs: + using: composite + steps: + - uses: ./tools +`, + }); + + await assert.rejects(verifyFixtureRelease({ client }), /must execute only/); + }); + + test("rejects unsafe manifest paths", () => { + assert.throws( + () => parseReleaseManifest("action.yml\n../secret\n"), + /traverses outside/, + ); + assert.throws( + () => parseReleaseManifest("action.yml\naction.yml\n"), + /duplicate path/, + ); + assert.throws( + () => parseReleaseManifest(" action.yml\n"), + /has whitespace/, + ); + }); +}); + +describe("GitHub adapter", () => { + test("paginates REST repository inventory without omission", async () => { + const repositories = Array.from({ length: 301 }, (_, index) => + restRepository(`repo-${String(index).padStart(3, "0")}`), + ); + const pages = []; + const client = new GitHubClient({ + async execute(args) { + const endpoint = args.at(-1); + const page = Number( + new URL(`https://api.github.test/${endpoint}`).searchParams.get( + "page", + ), + ); + pages.push(page); + const start = (page - 1) * 100; + return { + stderr: "", + stdout: JSON.stringify(repositories.slice(start, start + 100)), + }; + }, + }); + + const result = await client.listRestRepositories("workos"); + + assert.equal(result.length, 301); + assert.deepEqual(pages, [1, 2, 3, 4]); + }); + + test("runs the independent GraphQL-backed repository command", async () => { + let received; + const response = [graphqlRepository("one")]; + const client = new GitHubClient({ + async execute(args) { + received = args; + return { stderr: "", stdout: JSON.stringify(response) }; + }, + }); + + assert.deepEqual(await client.listGraphqlRepositories("workos"), response); + assert.deepEqual(received, [ + "repo", + "list", + "workos", + "--limit", + "10000", + "--json", + "name,isArchived,visibility", + ]); + }); + + test("honors retry-after for a bounded rate-limit retry", async () => { + let attempts = 0; + const delays = []; + const client = new GitHubClient({ + async execute() { + attempts += 1; + if (attempts === 1) { + throw new GhCommandError("rate limited", { + retryAfterMs: 7_000, + status: 429, + }); + } + return { stderr: "", stdout: "[]" }; + }, + async sleep(delay) { + delays.push(delay); + }, + }); + + assert.deepEqual(await client.api("example"), []); + assert.equal(attempts, 2); + assert.deepEqual(delays, [7_000]); + }); + + test("fails fast for a non-rate-limit 403", async () => { + let attempts = 0; + const client = new GitHubClient({ + defaultRetryDelayMs: 1, + async execute() { + attempts += 1; + throw new GhCommandError("forbidden", { status: 403 }); + }, + async sleep() {}, + }); + + await assert.rejects(client.api("example"), /forbidden/); + assert.equal(attempts, 1); + }); +}); + +describe("repository inventory", () => { + test("reconciles exact REST and GraphQL active sets", () => { + const rest = [ + restRepository("public-repo", { visibility: "public" }), + restRepository("internal-repo", { visibility: "internal" }), + restRepository("old-repo", { archived: true }), + ]; + const graphql = [ + graphqlRepository("internal-repo", { visibility: "internal" }), + graphqlRepository("old-repo", { archived: true }), + graphqlRepository("public-repo", { visibility: "public" }), + ]; + + assert.deepEqual(reconcileRepositoryInventories(rest, graphql), { + activeCount: 2, + archivedCount: 1, + differences: { + activeOnlyGraphql: [], + activeOnlyRest: [], + allOnlyGraphql: [], + allOnlyRest: [], + visibilityMismatches: [], + }, + repositories: [ + { + defaultBranch: "main", + name: "internal-repo", + visibility: "internal", + }, + { + defaultBranch: "main", + name: "public-repo", + visibility: "public", + }, + ], + schemaVersion: 1, + totalCount: 3, + visibility: { internal: 1, private: 0, public: 1 }, + }); + }); + + test("fails closed when repository visibility differs", () => { + assert.throws( + () => + reconcileRepositoryInventories( + [restRepository("one", { visibility: "private" })], + [graphqlRepository("one", { visibility: "public" })], + ), + /inventories differ/, + ); + }); + + test("rejects a malformed archived state", () => { + assert.throws( + () => + reconcileRepositoryInventories( + [{ ...restRepository("one"), archived: "false" }], + [graphqlRepository("one")], + ), + /invalid archived state/, + ); + }); + + test("fails closed when either inventory omits a repository", () => { + assert.throws( + () => + reconcileRepositoryInventories( + [restRepository("one"), restRepository("two")], + [graphqlRepository("one")], + ), + /inventories differ/, + ); + }); + + test("captures both sources before returning inventory", async () => { + const calls = []; + const client = { + async listGraphqlRepositories() { + calls.push("graphql"); + return [graphqlRepository("one")]; + }, + async listRestRepositories() { + calls.push("rest"); + return [restRepository("one")]; + }, + }; + + const result = await captureRepositoryInventory(client, "workos"); + + assert.equal(result.activeCount, 1); + assert.deepEqual(calls.sort(), ["graphql", "rest"]); + }); +}); + +describe("CLI", () => { + test("rejects unknown commands and extra arguments", async () => { + await assert.rejects(main([], {}), /usage:/); + await assert.rejects(main(["inventory", "--org", "other"], {}), /usage:/); + }); + + test("prints sanitized inventory JSON and keeps full inventory private", async (t) => { + const directory = await mkdtemp(join(tmpdir(), "sfw-inventory-")); + t.after(() => rm(directory, { recursive: true, force: true })); + const reportPath = join(directory, "inventory.json"); + let output = ""; + const client = { + async listGraphqlRepositories() { + return [graphqlRepository("one")]; + }, + async listRestRepositories() { + return [restRepository("one")]; + }, + }; + + await main(["inventory"], { + client, + reportPath, + output: { + write(value) { + output += value; + }, + }, + }); + + assert.equal(JSON.parse(output).activeCount, 1); + assert.equal(JSON.parse(output).repositories, undefined); + assert.equal( + JSON.parse(await readFile(reportPath, "utf8")).repositories[0].name, + "one", + ); + }); +}); diff --git a/tools/rollout/source-approvals.test.mjs b/tools/rollout/source-approvals.test.mjs new file mode 100644 index 0000000..d63b3cf --- /dev/null +++ b/tools/rollout/source-approvals.test.mjs @@ -0,0 +1,191 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import test from "node:test"; +import { stringify } from "yaml"; +import { GitHubClient } from "./github.mjs"; +import { readRegistryExclusions, REGISTRY_EXCLUSIONS } from "./exclusions.mjs"; + +const approval = "61c52b1b-8081-48fa-9c8d-e2462e931ef7"; + +test("large file reads use the metadata's immutable blob and reject incomplete or substituted content", async () => { + const bytes = Buffer.alloc(1024 * 1024 + 1, "a"); + const sha = createHash("sha1") + .update(`blob ${bytes.length}\0`) + .update(bytes) + .digest("hex"); + const metadata = { + type: "file", + encoding: "none", + content: "", + size: bytes.length, + sha, + }; + const blob = { + sha, + size: bytes.length, + encoding: "base64", + content: bytes.toString("base64"), + }; + const calls = []; + let response = blob; + const api = new GitHubClient(); + api.api = async (endpoint) => { + calls.push(endpoint); + return endpoint.includes("/contents/") ? metadata : response; + }; + assert.equal( + await api.getText("workos/example", "large.lock", "a".repeat(40)), + bytes.toString(), + ); + assert.equal(calls[1], `repos/workos/example/git/blobs/${sha}`); + for (const change of [ + { sha: "b".repeat(40) }, + { size: bytes.length - 1 }, + { content: blob.content.slice(4) }, + { content: Buffer.alloc(bytes.length, "b").toString("base64") }, + { encoding: "none" }, + ]) { + response = { ...blob, ...change }; + await assert.rejects( + api.getText("workos/example", "large.lock", "a".repeat(40)), + ); + } + metadata.size = 11 * 1024 * 1024; + calls.length = 0; + await assert.rejects( + api.getText("workos/example", "large.lock", "a".repeat(40)), + ); + assert.equal( + calls.length, + 1, + "unsupported size is rejected before fetching the blob", + ); +}); + +test("WorkOS approval binds the exact Rush Git override, pnpm source and integrity, not unrelated registry versions", async () => { + const rule = REGISTRY_EXCLUSIONS.find((item) => item.repository === "workos"); + assert.ok(rule, "the requesting employee approved the WorkOS Kotlin source"); + assert.equal(rule.approvalRequestId, approval); + assert.deepEqual( + rule.workflows, + [], + "source approval never waives a CI installation", + ); + // Independently transcribed approved source, not generated from the rule. + const specifier = + "github:fwcd/tree-sitter-kotlin#f66d2908542e93c0204c6c241f794afe4e9cd5d1"; + const tarball = + "https://codeload.github.com/fwcd/tree-sitter-kotlin/tar.gz/f66d2908542e93c0204c6c241f794afe4e9cd5d1"; + const integrity = + "sha512-7pk1Tg/gXh+6hM4E0F2rPKeLyA/bNlYyqVu6T1Md/AV/vloakbvEZG0R4CYwUfVtgFAMZRbOtA8JBzX1SFatBA=="; + assert.equal(rule.specifier, specifier); + assert.equal(rule.resolved, tarball); + assert.equal(rule.integrity, integrity); + assert.equal(rule.version, "0.4.0"); + const config = { + $schema: "https://example.invalid/schema.json", + note: "/* quoted, not a comment */", + globalOverrides: { "tree-sitter-kotlin": specifier, ordinary: "^1.0.0" }, + }; + const entry = { + version: "0.4.0", + resolution: { gitHosted: true, tarball, integrity }, + }; + const lock = { + lockfileVersion: "9.0", + overrides: { ...config.globalOverrides }, + packages: { + [rule.packagePath]: entry, + "ordinary@1.0.0": { resolution: { integrity: "sha512-fixture" } }, + "vendored@file:../../third_party/npm/example.tgz": { + resolution: { + integrity: "sha512-local", + tarball: "file:../../third_party/npm/example.tgz", + }, + }, + }, + }; + const read = async (c = config, l = lock) => { + const files = { + [rule.manifest]: `/** Rush configuration */\n// preserve URLs inside strings\n${JSON.stringify(c)}`, + [rule.lockfile]: stringify(l), + }; + const [result] = await readRegistryExclusions("workos", async (path) => { + assert.ok(Object.hasOwn(files, path), path); + return files[path]; + }); + return result.status; + }; + assert.equal(await read(), "matched"); + assert.equal( + await read( + { + ...config, + globalOverrides: { ...config.globalOverrides, ordinary: "^2.0.0" }, + }, + { ...lock, overrides: { ...lock.overrides, ordinary: "^2.0.0" } }, + ), + "matched", + ); + assert.equal( + await read({ + ...config, + globalOverrides: { "tree-sitter-kotlin": "github:other/repo#unapproved" }, + }), + "stale", + ); + assert.equal(await read(config, { ...lock, overrides: {} }), "stale"); + assert.equal( + await read(config, { ...lock, lockfileVersion: "10.0" }), + "stale", + ); + for (const change of [ + { version: "0.5.0" }, + { resolution: { ...entry.resolution, integrity: "sha512-other" } }, + { + resolution: { + ...entry.resolution, + tarball: "https://example.invalid/other.tgz", + }, + }, + { resolution: { ...entry.resolution, repo: "https://example.invalid" } }, + ]) + assert.equal( + await read(config, { + ...lock, + packages: { + ...lock.packages, + [rule.packagePath]: { ...entry, ...change }, + }, + }), + "stale", + ); + for (const resolution of [ + { + tarball: "https://example.invalid/unapproved.tgz", + integrity: "sha512-other", + }, + { + type: "git", + repo: "https://example.invalid/repo", + commit: "b".repeat(40), + }, + { tarball: "file://remote.example/archive.tgz", integrity: "sha512-other" }, + ]) + assert.equal( + await read(config, { + ...lock, + packages: { ...lock.packages, extra: { resolution } }, + }), + "stale", + ); + await assert.rejects( + readRegistryExclusions("workos", async () => { + throw new Error("unavailable"); + }), + /unavailable/, + ); + await assert.rejects( + readRegistryExclusions("workos", async () => "malformed"), + ); +}); diff --git a/tools/rollout/source-resolution.test.mjs b/tools/rollout/source-resolution.test.mjs new file mode 100644 index 0000000..86231c2 --- /dev/null +++ b/tools/rollout/source-resolution.test.mjs @@ -0,0 +1,239 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { stringify } from "yaml"; +import { auditRepository } from "./audit.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; + +const sha = "c".repeat(40); +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}" }, +}; +const action = stringify({ + runs: { using: "composite", steps: [{ shell: "bash", run: "npm ci" }] }, +}); +const captured = JSON.parse( + readFileSync(new URL("./fixtures/approved-release.json", import.meta.url)), +); +async function audit( + steps, + files = { "actions/install/action.yml": action }, + entries = [], + on = "push", +) { + files = { + ...files, + ".github/workflows/ci.yml": stringify({ + on, + jobs: { test: { "runs-on": "ubuntu-latest", steps } }, + }), + }; + const reads = []; + const result = await auditRepository( + { + async getRef() { + return { object: { sha } }; + }, + async getTree() { + return { + truncated: false, + tree: [ + ...Object.keys(files) + .filter((path) => !entries.some((e) => e.path === path)) + .map((path) => ({ path, type: "blob", mode: "100644" })), + ...entries, + ], + }; + }, + async getText(_repo, path, ref) { + assert.equal(ref, sha); + reads.push(path); + assert.ok(Object.hasOwn(files, path), path); + return files[path]; + }, + }, + { name: "fixture", defaultBranch: "main", visibility: "private" }, + ); + return { result, reads }; +} + +test("a same-snapshot checkout mount resolves its local action, preserving uncovered installs", async () => { + const checkout = { uses: "actions/checkout@v4", with: { path: "source" } }; + const local = { uses: "./source/actions/install" }; + const { result, reads } = await audit([checkout, setup, local]); + assert.equal(result.disposition, "integrated"); + assert.ok(reads.includes("actions/install/action.yml")); + assert.equal( + (await audit([checkout, local])).result.disposition, + "needs-sfw", + ); + assert.equal( + (await audit([local, checkout])).result.disposition, + "needs-review", + ); + for (const withInput of [ + { path: "source", repository: "other/repo" }, + { path: "source", ref: "other-branch" }, + { path: "source", ref: "${{ inputs.checker-ref }}" }, + { path: "source", "sparse-checkout": "something-else" }, + { path: "../source" }, + ]) + assert.equal( + (await audit([{ ...checkout, with: withInput }, setup, local])).result + .disposition, + "needs-review", + ); + for (const extra of [ + { if: "inputs.enabled" }, + { "continue-on-error": true }, + { env: { PATH: "/other" } }, + ]) + assert.equal( + (await audit([{ ...checkout, ...extra }, setup, local])).result + .disposition, + "needs-review", + ); + assert.equal( + ( + await audit([ + checkout, + { + uses: "actions/checkout@v4", + with: { repository: "other/repo", path: "${{ env.DESTINATION }}" }, + }, + setup, + local, + ]) + ).result.disposition, + "needs-review", + ); +}); + +test("a caller-selected checker revision is unresolved provenance, not a nonexistent local action", async () => { + const { result, reads } = await audit([ + { uses: "actions/checkout@v4" }, + { + uses: "actions/checkout@v4", + with: { + repository: "workos/fixture", + ref: "${{ inputs.checker-ref }}", + path: ".checker", + }, + }, + { uses: "./.checker/actions/install" }, + ]); + assert.equal(result.disposition, "needs-review"); + const operation = result.workflows[0].jobs[0].operations.find( + (o) => o.kind === "unknown-local-action", + ); + assert.equal(operation.sourceError, "unresolved-checkout-source"); + assert.equal(operation.checkoutRef, "${{ inputs.checker-ref }}"); + assert.equal(reads.includes("actions/install/action.yml"), false); +}); + +test("local SFW entrypoints require all immutable reviewed runtime blobs", async () => { + const files = Object.fromEntries( + Object.entries(captured.contents).map(([path, entry]) => [ + path, + Buffer.from(entry.content, "base64").toString(), + ]), + ); + const entries = captured.tree.tree.filter((e) => e.type === "blob"); + const steps = [ + { uses: "actions/checkout@v4" }, + { uses: "./", with: setup.with }, + { run: "npm ci" }, + ]; + const { result } = await audit(steps, files, entries); + assert.equal(result.disposition, "integrated"); + assert.equal(result.assuranceDisposition, "needs-review"); + assert.ok( + result.workflows[0].jobs[0].integration.notes.includes( + "local-runtime-matches-approved-release", + ), + ); + for (const path of [ + "action.yml", + "scripts/configure.sh", + "scripts/teardown.sh", + "teardown/action.yml", + ]) + assert.notEqual( + ( + await audit( + steps, + files, + entries.map((e) => + e.path === path ? { ...e, sha: "f".repeat(40) } : e, + ), + ) + ).result.disposition, + "integrated", + ); + assert.equal( + ( + await audit( + [ + { uses: "actions/checkout@v4", with: { ref: "other" } }, + ...steps.slice(1), + ], + files, + entries, + ) + ).result.disposition, + "needs-sfw", + ); + assert.notEqual( + ( + await audit( + [steps[0], { ...steps[1], with: {} }, steps[2]], + files, + entries, + ) + ).result.disposition, + "integrated", + ); + assert.notEqual( + (await audit(steps.slice(1), files, entries)).result.disposition, + "integrated", + ); + assert.notEqual( + ( + await audit( + [{ ...steps[0], if: false }, ...steps.slice(1)], + files, + entries, + ) + ).result.disposition, + "integrated", + ); + assert.notEqual( + (await audit(steps, files, entries, "workflow_call")).result.disposition, + "integrated", + ); + assert.equal( + ( + await audit( + [ + { ...steps[0], with: { repository: "workos/fixture", ref: sha } }, + ...steps.slice(1), + ], + files, + entries, + "workflow_call", + ) + ).result.disposition, + "integrated", + ); + assert.equal( + ( + await audit( + [...steps, { uses: "./teardown" }, { run: "npm ci" }], + files, + entries, + ) + ).result.disposition, + "needs-sfw", + ); +}); diff --git a/tools/rollout/yaml.mjs b/tools/rollout/yaml.mjs new file mode 100644 index 0000000..794db3a --- /dev/null +++ b/tools/rollout/yaml.mjs @@ -0,0 +1,18 @@ +import { parseDocument } from "yaml"; + +// yaml.parse() emits source-bearing warnings directly to stderr. Keep parsing +// non-emitting and reject unsupported constructs instead of guessing at them. +export function parseYamlSource(text) { + const document = parseDocument(text, { + logLevel: "error", + stringKeys: true, + prettyErrors: false, + }); + const diagnostics = [...document.errors, ...document.warnings]; + if (diagnostics.length > 0) { + throw new Error( + `YAML requires review: ${[...new Set(diagnostics.map((item) => item.code))].join(", ")}`, + ); + } + return document.toJS({ maxAliasCount: 100 }); +}