From e60c0d3acc049b89c88abfcf617f302921437a5e Mon Sep 17 00:00:00 2001 From: Matt Peake Date: Thu, 3 Sep 2026 10:26:02 -0400 Subject: [PATCH 01/19] feat: add rollout verifier foundation --- .github/workflows/ci.yml | 25 ++ .gitignore | 5 + CONTRIBUTING.md | 5 +- README.md | 25 +- package-lock.json | 52 ++++ package.json | 25 ++ renovate.json | 10 +- tools/rollout/cli.mjs | 36 +++ tools/rollout/constants.mjs | 27 +++ tools/rollout/github.mjs | 200 ++++++++++++++++ tools/rollout/inventory.mjs | 131 +++++++++++ tools/rollout/release.mjs | 208 ++++++++++++++++ tools/rollout/rollout.test.mjs | 419 +++++++++++++++++++++++++++++++++ 13 files changed, 1160 insertions(+), 8 deletions(-) create mode 100644 .gitignore create mode 100644 package-lock.json create mode 100644 package.json create mode 100644 tools/rollout/cli.mjs create mode 100644 tools/rollout/constants.mjs create mode 100644 tools/rollout/github.mjs create mode 100644 tools/rollout/inventory.mjs create mode 100644 tools/rollout/release.mjs create mode 100644 tools/rollout/rollout.test.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index f27b108..39e727b 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -15,6 +15,25 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 + with: + node-version: 22 + cache: npm + - name: Configure Socket Firewall for verifier dependencies + uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 + with: + token: ${{ secrets.SOCKET_FIREWALL_TOKEN }} + allow-external-fork-fallback: true + - name: Install verifier dependencies + run: npm ci --ignore-scripts --no-audit --no-fund + - name: Remove Socket Firewall credentials before executing verifier source + uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 + - name: Run verifier foundation tests + env: + GH_TOKEN: ${{ github.token }} + run: npm run check - name: Check shell style and test coverage policy run: | set -euo pipefail @@ -67,6 +86,8 @@ jobs: manager: [npm, pnpm, bun] steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - name: Setup Node.js uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 @@ -210,6 +231,8 @@ jobs: timeout-minutes: 10 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 @@ -247,6 +270,8 @@ jobs: timeout-minutes: 5 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0 with: node-version: 22 diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..a5f7024 --- /dev/null +++ b/.gitignore @@ -0,0 +1,5 @@ +node_modules/ +*.inventory.json +reports/* +!reports/.gitkeep +!reports/socket-firewall-ci-rollout-final.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 2b24203..899ef9d 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,6 +6,7 @@ Thanks for helping improve the WorkOS Socket Firewall GitHub Action. - Bash on Linux or macOS - Go, for the pinned `shfmt` check +- Node.js 22 and npm, for the source-only rollout verifier - ShellCheck - Passwordless `sudo` and a disposable Linux runner for integration testing that modifies `/etc/hosts` @@ -23,11 +24,13 @@ bash scripts/configure.test.sh bash scripts/teardown.test.sh bash scripts/build-release.test.sh bash scripts/publish-release.test.sh +npm ci --ignore-scripts --no-audit --no-fund +npm run check ``` CI runs the same static and unit checks on every pull request. Token-backed GitHub-hosted smoke jobs additionally exercise every supported package manager. -The minimum test-coverage policy is one shell test suite for every executable shell source file. Changes to supported package-manager behavior must also include a token-backed frozen-lockfile smoke test. +The minimum test-coverage policy is one shell test suite for every executable shell source file. Verifier changes require Node fixture tests and must preserve exact REST/GraphQL inventory reconciliation. Changes to supported package-manager behavior must also include a token-backed frozen-lockfile smoke test. ## Pull request guidelines diff --git a/README.md b/README.md index 4aca5d6..465ffad 100644 --- a/README.md +++ b/README.md @@ -16,7 +16,7 @@ It does not route package publication or Python, Java, Go, Ruby, Rust, .NET, pri Consumers must pin the full 40-character SHA of the reviewed action-only `v1` commit. Do not execute a mutable tag, branch, abbreviated SHA, or normal source commit. ```yaml -uses: workos/setup-socket-firewall@ # v1 +uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 ``` The moving `v1` tag and `action-release/v1` branch are for human discovery and Renovate lookup. The repository’s normal source history contains tests and rollout tooling; each action-only release commit contains only the files in `release-manifest.txt`. @@ -36,7 +36,7 @@ Run package-manager setup first, then configure SFW before the first dependency registry-url: https://registry.npmjs.org/ - name: Configure Socket Firewall - uses: workos/setup-socket-firewall@ # v1 + uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 with: token: ${{ secrets.SOCKET_FIREWALL_TOKEN }} @@ -67,7 +67,7 @@ steps: node-version: 22 - name: Configure Socket Firewall - uses: workos/setup-socket-firewall@ # v1 + uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 with: token: ${{ secrets.SOCKET_FIREWALL_TOKEN }} allow-external-fork-fallback: true @@ -87,7 +87,7 @@ When an existing job must both install and publish, run the teardown entrypoint ```yaml - name: Configure Socket Firewall - uses: workos/setup-socket-firewall@ # v1 + uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 with: token: ${{ secrets.SOCKET_FIREWALL_TOKEN }} @@ -95,7 +95,7 @@ When an existing job must both install and publish, run the teardown entrypoint - run: pnpm build - name: Restore public package registry - uses: workos/setup-socket-firewall/teardown@ # v1 + uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 - name: Publish package run: pnpm publish --access public --provenance --no-git-checks @@ -148,7 +148,20 @@ No maintainer runs local release commands. The workflow uses only the repository-scoped `GITHUB_TOKEN` with `contents: write`, serializes releases, skips stale successful commits when `main` has advanced, and can be retried through `workflow_dispatch`. A future breaking release must change the reviewed channel to `v2`; it must not repurpose `v1`. -Never publish the normal source commit as an action release: it contains tests and, after HELP-724 Phase 2, the one-time rollout verifier and report source. +Never publish the normal source commit as an action release: it contains tests and the one-time HELP-724 rollout verifier source. + +## Rollout verifier + +The source branch contains operator-only verification tooling that never enters the action release. Its foundation validates the signed action tree and independently reconciles REST and GraphQL WorkOS repository inventories: + +```bash +npm ci --ignore-scripts +npm test +npm run verify-action +npm run inventory +``` + +`inventory` requires an existing `gh` session with `repo` and `read:org`. It is read-only, accepts no configuration flags, and fails unless both inventory sources return exactly the same active repository set. Its output can contain internal repository names; do not attach it to a public issue or workflow log. ## Contributing diff --git a/package-lock.json b/package-lock.json new file mode 100644 index 0000000..e070f21 --- /dev/null +++ b/package-lock.json @@ -0,0 +1,52 @@ +{ + "name": "@workos/setup-socket-firewall-rollout", + "version": "0.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "@workos/setup-socket-firewall-rollout", + "version": "0.0.0", + "dependencies": { + "yaml": "2.9.0" + }, + "devDependencies": { + "prettier": "3.9.6" + }, + "engines": { + "node": ">=22" + } + }, + "node_modules/prettier": { + "version": "3.9.6", + "resolved": "https://registry.npmjs.org/prettier/-/prettier-3.9.6.tgz", + "integrity": "sha512-OpN0zzVdiaiAhxpuuj5efpIS4sY9j7bY6uR5mnj5yPzGkdkjNKSJeUThPb60Jw29QuAZgA4o+/iB49kFiaBX6g==", + "dev": true, + "license": "MIT", + "bin": { + "prettier": "bin/prettier.cjs" + }, + "engines": { + "node": ">=14" + }, + "funding": { + "url": "https://github.com/prettier/prettier?sponsor=1" + } + }, + "node_modules/yaml": { + "version": "2.9.0", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.9.0.tgz", + "integrity": "sha512-2AvhNX3mb8zd6Zy7INTtSpl1F15HW6Wnqj0srWlkKLcpYl/gMIMJiyuGq2KeI2YFxUPjdlB+3Lc10seMLtL4cA==", + "license": "ISC", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } + } + } +} diff --git a/package.json b/package.json new file mode 100644 index 0000000..db1f94b --- /dev/null +++ b/package.json @@ -0,0 +1,25 @@ +{ + "name": "@workos/setup-socket-firewall-rollout", + "version": "0.0.0", + "private": true, + "description": "One-time HELP-724 rollout verification tooling", + "type": "module", + "engines": { + "node": ">=22" + }, + "scripts": { + "check": "npm run format:check && npm test && npm run verify-action", + "format": "prettier --write package.json tools/rollout/*.mjs", + "format:check": "prettier --check package.json tools/rollout/*.mjs", + "inventory": "node tools/rollout/cli.mjs inventory", + "test": "node --test tools/rollout/*.test.mjs", + "test:unit": "node --test tools/rollout/rollout.test.mjs", + "verify-action": "node tools/rollout/cli.mjs verify-action" + }, + "dependencies": { + "yaml": "2.9.0" + }, + "devDependencies": { + "prettier": "3.9.6" + } +} diff --git a/renovate.json b/renovate.json index 2ce3453..a6f8814 100644 --- a/renovate.json +++ b/renovate.json @@ -1,5 +1,13 @@ { "$schema": "https://docs.renovatebot.com/renovate-schema.json", "extends": ["github>workos/renovate-config:public"], - "enabledManagers": ["github-actions"] + "enabledManagers": ["github-actions", "npm"], + "packageRules": [ + { + "description": "Update the approved Socket Firewall SHA only with its verifier constant", + "matchManagers": ["github-actions"], + "matchDepNames": ["workos/setup-socket-firewall"], + "enabled": false + } + ] } diff --git a/tools/rollout/cli.mjs b/tools/rollout/cli.mjs new file mode 100644 index 0000000..f9870f9 --- /dev/null +++ b/tools/rollout/cli.mjs @@ -0,0 +1,36 @@ +#!/usr/bin/env node + +import { pathToFileURL } from "node:url"; + +import { GitHubClient } from "./github.mjs"; +import { captureRepositoryInventory } from "./inventory.mjs"; +import { ORGANIZATION } from "./constants.mjs"; +import { verifyActionRelease } from "./release.mjs"; + +const COMMANDS = new Set(["inventory", "verify-action"]); + +export async function main(argv, options = {}) { + if (argv.length !== 1 || !COMMANDS.has(argv[0])) { + throw new Error("usage: cli.mjs "); + } + + const client = options.client ?? new GitHubClient(); + const output = options.output ?? process.stdout; + const command = argv[0]; + const result = + command === "verify-action" + ? await verifyActionRelease({ client }) + : await captureRepositoryInventory(client, ORGANIZATION); + output.write(`${JSON.stringify(result, null, 2)}\n`); + return result; +} + +const invokedPath = process.argv[1] + ? pathToFileURL(process.argv[1]).href + : undefined; +if (import.meta.url === invokedPath) { + main(process.argv.slice(2)).catch((error) => { + process.stderr.write(`rollout verifier failed: ${error.message}\n`); + process.exitCode = 1; + }); +} diff --git a/tools/rollout/constants.mjs b/tools/rollout/constants.mjs new file mode 100644 index 0000000..11a9a1c --- /dev/null +++ b/tools/rollout/constants.mjs @@ -0,0 +1,27 @@ +export const ORGANIZATION = "workos"; +export const ACTION_REPOSITORY = "workos/setup-socket-firewall"; +export const RELEASE_CHANNEL = "v1"; +export const RELEASE_BRANCH = `action-release/${RELEASE_CHANNEL}`; +export const APPROVED_RELEASE_SHA = "ca93dd8aa351f54f4729fe3377a9be23c631c25d"; + +export const EXPECTED_RELEASE_TREE = Object.freeze([ + Object.freeze({ mode: "100644", path: "LICENSE", type: "blob" }), + Object.freeze({ mode: "100644", path: "action.yml", type: "blob" }), + Object.freeze({ mode: "040000", path: "scripts", type: "tree" }), + Object.freeze({ + mode: "100755", + path: "scripts/configure.sh", + type: "blob", + }), + Object.freeze({ + mode: "100755", + path: "scripts/teardown.sh", + type: "blob", + }), + Object.freeze({ mode: "040000", path: "teardown", type: "tree" }), + Object.freeze({ + mode: "100644", + path: "teardown/action.yml", + type: "blob", + }), +]); diff --git a/tools/rollout/github.mjs b/tools/rollout/github.mjs new file mode 100644 index 0000000..e75a8da --- /dev/null +++ b/tools/rollout/github.mjs @@ -0,0 +1,200 @@ +import { execFile as execFileCallback } from "node:child_process"; +import { promisify } from "node:util"; +import { setTimeout as sleep } from "node:timers/promises"; + +const execFile = promisify(execFileCallback); +const DEFAULT_MAX_ATTEMPTS = 3; +const DEFAULT_PAGE_SIZE = 100; +const MAX_REPOSITORY_COUNT = 10_000; +const MAX_REPOSITORY_PAGES = MAX_REPOSITORY_COUNT / DEFAULT_PAGE_SIZE; + +function statusFromText(text) { + const match = String(text).match(/\(HTTP ([0-9]{3})\)/); + return match ? Number(match[1]) : undefined; +} + +function retryAfterFromText(text) { + const match = String(text).match(/retry-after:\s*([0-9]+)/i); + return match ? Number(match[1]) * 1_000 : undefined; +} + +function parseJson(stdout, description) { + try { + return JSON.parse(stdout); + } catch (error) { + throw new Error(`${description} returned invalid JSON: ${error.message}`); + } +} + +export class GhCommandError extends Error { + constructor(message, options = {}) { + super(message, options); + this.name = "GhCommandError"; + this.status = options.status; + this.retryAfterMs = options.retryAfterMs; + this.exitCode = options.exitCode; + } +} + +export async function executeGh(args) { + try { + return await execFile("gh", args, { + encoding: "utf8", + env: process.env, + maxBuffer: 50 * 1024 * 1024, + }); + } catch (error) { + const stderr = String(error.stderr ?? ""); + const stdout = String(error.stdout ?? ""); + const status = statusFromText(stderr) ?? statusFromText(stdout); + const retryAfterMs = + retryAfterFromText(stderr) ?? retryAfterFromText(stdout); + const detail = + stderr.trim() || `gh exited with code ${error.code ?? "unknown"}`; + throw new GhCommandError(detail, { + cause: error, + exitCode: error.code, + retryAfterMs, + status, + }); + } +} + +export class GitHubClient { + constructor(options = {}) { + this.execute = options.execute ?? executeGh; + this.sleep = options.sleep ?? sleep; + this.maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS; + this.defaultRetryDelayMs = options.defaultRetryDelayMs ?? 1_000; + + if (!Number.isInteger(this.maxAttempts) || this.maxAttempts < 1) { + throw new Error("maxAttempts must be a positive integer"); + } + } + + async run(args, description) { + for (let attempt = 1; attempt <= this.maxAttempts; attempt += 1) { + try { + return await this.execute(args); + } catch (error) { + const rateLimited403 = + error.status === 403 && + (error.retryAfterMs !== undefined || + /rate.?limit/i.test(error.message)); + const retryable = error.status === 429 || rateLimited403; + if (!retryable || attempt === this.maxAttempts) { + throw new Error(`${description} failed: ${error.message}`, { + cause: error, + }); + } + + const delay = + error.retryAfterMs ?? + this.defaultRetryDelayMs * 2 ** Math.max(0, attempt - 1); + await this.sleep(delay); + } + } + + throw new Error(`${description} failed without an attempt`); + } + + async api(endpoint, description = `GitHub API ${endpoint}`) { + const { stdout } = await this.run( + ["api", "--method", "GET", endpoint], + description, + ); + return parseJson(stdout, description); + } + + async getRef(repository, ref) { + return this.api( + `repos/${repository}/git/ref/${ref}`, + `read ${repository} ref ${ref}`, + ); + } + + async getCommit(repository, sha) { + return this.api( + `repos/${repository}/git/commits/${sha}`, + `read ${repository} commit ${sha}`, + ); + } + + async getTree(repository, sha, recursive = false) { + const suffix = recursive ? "?recursive=1" : ""; + return this.api( + `repos/${repository}/git/trees/${sha}${suffix}`, + `read ${repository} tree ${sha}`, + ); + } + + async getText(repository, path, ref) { + const response = await this.api( + `repos/${repository}/contents/${path}?ref=${encodeURIComponent(ref)}`, + `read ${repository}/${path} at ${ref}`, + ); + if (response.type !== "file" || response.encoding !== "base64") { + throw new Error( + `${repository}/${path} at ${ref} is not a base64 GitHub file response`, + ); + } + return Buffer.from( + response.content.replaceAll("\n", ""), + "base64", + ).toString("utf8"); + } + + async listRestRepositories(organization) { + const repositories = []; + + for (let page = 1; page <= MAX_REPOSITORY_PAGES + 1; page += 1) { + const response = await this.api( + `orgs/${organization}/repos?type=all&per_page=${DEFAULT_PAGE_SIZE}&page=${page}`, + `list ${organization} repositories via REST page ${page}`, + ); + if (!Array.isArray(response)) { + throw new Error(`REST repository page ${page} is not an array`); + } + if (page === MAX_REPOSITORY_PAGES + 1) { + if (response.length > 0) { + throw new Error( + `REST repository inventory exceeded ${MAX_REPOSITORY_COUNT} entries`, + ); + } + return repositories; + } + repositories.push(...response); + if (response.length < DEFAULT_PAGE_SIZE) { + return repositories; + } + } + + throw new Error("REST repository pagination ended unexpectedly"); + } + + async listGraphqlRepositories(organization) { + const description = `list ${organization} repositories via GraphQL`; + const { stdout } = await this.run( + [ + "repo", + "list", + organization, + "--limit", + String(MAX_REPOSITORY_COUNT), + "--json", + "name,isArchived,visibility", + ], + description, + ); + const response = parseJson(stdout, description); + if (!Array.isArray(response)) { + throw new Error("GraphQL repository inventory is not an array"); + } + if (response.length === MAX_REPOSITORY_COUNT) { + throw new Error( + `GraphQL repository inventory reached its ${MAX_REPOSITORY_COUNT}-entry limit and may be truncated`, + ); + } + return response; + } +} diff --git a/tools/rollout/inventory.mjs b/tools/rollout/inventory.mjs new file mode 100644 index 0000000..12fa778 --- /dev/null +++ b/tools/rollout/inventory.mjs @@ -0,0 +1,131 @@ +function assert(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +function compareStrings(left, right) { + return left < right ? -1 : left > right ? 1 : 0; +} + +function sorted(values) { + return [...values].sort(compareStrings); +} + +function indexRepositories(repositories, source) { + assert(Array.isArray(repositories), `${source} inventory is not an array`); + const indexed = new Map(); + + for (const repository of repositories) { + assert( + repository && typeof repository.name === "string" && repository.name, + `${source} inventory contains a repository without a name`, + ); + assert( + !indexed.has(repository.name), + `${source} inventory contains duplicate repository ${repository.name}`, + ); + const archived = + source === "REST" ? repository.archived : repository.isArchived; + assert( + typeof archived === "boolean", + `${source} repository ${repository.name} has invalid archived state`, + ); + indexed.set(repository.name, repository); + } + + return indexed; +} + +function difference(left, right) { + return sorted([...left].filter((name) => !right.has(name))); +} + +function normalizedVisibility(repository, source) { + const visibility = String(repository.visibility ?? "").toLowerCase(); + assert( + ["internal", "private", "public"].includes(visibility), + `${source} repository ${repository.name} has invalid visibility`, + ); + return visibility; +} + +export function reconcileRepositoryInventories( + restRepositories, + graphqlRepositories, +) { + const rest = indexRepositories(restRepositories, "REST"); + const graphql = indexRepositories(graphqlRepositories, "GraphQL"); + const restNames = new Set(rest.keys()); + const graphqlNames = new Set(graphql.keys()); + const restActive = new Set( + [...rest] + .filter(([, repository]) => repository.archived === false) + .map(([name]) => name), + ); + const graphqlActive = new Set( + [...graphql] + .filter(([, repository]) => repository.isArchived === false) + .map(([name]) => name), + ); + + const visibilityMismatches = sorted(restNames) + .filter((name) => graphql.has(name)) + .filter( + (name) => + normalizedVisibility(rest.get(name), "REST") !== + normalizedVisibility(graphql.get(name), "GraphQL"), + ); + const differences = { + activeOnlyGraphql: difference(graphqlActive, restActive), + activeOnlyRest: difference(restActive, graphqlActive), + allOnlyGraphql: difference(graphqlNames, restNames), + allOnlyRest: difference(restNames, graphqlNames), + visibilityMismatches, + }; + const mismatchCount = Object.values(differences).reduce( + (total, names) => total + names.length, + 0, + ); + assert( + mismatchCount === 0, + `REST and GraphQL repository inventories differ: ${JSON.stringify(differences)}`, + ); + + const repositories = sorted(restActive).map((name) => { + const repository = rest.get(name); + assert( + typeof repository.default_branch === "string" && + repository.default_branch, + `active REST repository ${name} has no default branch`, + ); + return { + defaultBranch: repository.default_branch, + name, + visibility: normalizedVisibility(repository, "REST"), + }; + }); + + const visibility = { internal: 0, private: 0, public: 0 }; + for (const repository of repositories) { + visibility[repository.visibility] += 1; + } + + return { + activeCount: repositories.length, + archivedCount: rest.size - repositories.length, + differences, + repositories, + schemaVersion: 1, + totalCount: rest.size, + visibility, + }; +} + +export async function captureRepositoryInventory(client, organization) { + const [restRepositories, graphqlRepositories] = await Promise.all([ + client.listRestRepositories(organization), + client.listGraphqlRepositories(organization), + ]); + return reconcileRepositoryInventories(restRepositories, graphqlRepositories); +} diff --git a/tools/rollout/release.mjs b/tools/rollout/release.mjs new file mode 100644 index 0000000..906b7d2 --- /dev/null +++ b/tools/rollout/release.mjs @@ -0,0 +1,208 @@ +import { readFile } from "node:fs/promises"; +import { parse as parseYaml } from "yaml"; + +import { + ACTION_REPOSITORY, + APPROVED_RELEASE_SHA, + EXPECTED_RELEASE_TREE, + RELEASE_BRANCH, + RELEASE_CHANNEL, +} from "./constants.mjs"; + +const DEFAULT_MANIFEST_URL = new URL( + "../../release-manifest.txt", + import.meta.url, +); +const FORBIDDEN_RELEASE_PATHS = [ + ".github/", + "package.json", + "package-lock.json", + "reports/", + "tools/", +]; + +function assert(condition, message) { + if (!condition) { + throw new Error(message); + } +} + +function compareStrings(left, right) { + return left < right ? -1 : left > right ? 1 : 0; +} + +function sortedTreeEntries(entries) { + return entries + .map(({ mode, path, type }) => ({ mode, path, type })) + .sort((left, right) => compareStrings(left.path, right.path)); +} + +function assertExactTree(actual, expected) { + const actualJson = JSON.stringify(sortedTreeEntries(actual)); + const expectedJson = JSON.stringify(sortedTreeEntries(expected)); + assert( + actualJson === expectedJson, + `release tree differs from the reviewed manifest: expected ${expectedJson}, received ${actualJson}`, + ); +} + +function parseAction(text, path) { + let action; + try { + action = parseYaml(text); + } catch (error) { + throw new Error(`${path} is not valid YAML: ${error.message}`); + } + assert(action && typeof action === "object", `${path} is not a YAML object`); + assert( + action.runs?.using === "composite", + `${path} is not a composite action`, + ); + assert(Array.isArray(action.runs.steps), `${path} has no composite steps`); + return action; +} + +function assertRuntimeReference(action, actionPath, expectedCommand) { + const commands = action.runs.steps + .map((step) => step.run) + .filter((run) => typeof run === "string"); + assert( + commands.length === 1 && commands[0] === expectedCommand, + `${actionPath} must execute only ${expectedCommand}`, + ); + + const localUses = action.runs.steps + .map((step) => step.uses) + .filter((uses) => typeof uses === "string" && uses.startsWith(".")); + assert( + localUses.length === 0, + `${actionPath} contains an unexpected local action reference`, + ); +} + +export function parseReleaseManifest(text) { + const paths = []; + const seen = new Set(); + + for (const [index, rawLine] of text.split("\n").entries()) { + if (rawLine === "" || rawLine.startsWith("#")) { + continue; + } + const lineNumber = index + 1; + assert( + rawLine === rawLine.trim(), + `manifest line ${lineNumber} has whitespace`, + ); + assert(!rawLine.startsWith("/"), `manifest line ${lineNumber} is absolute`); + assert( + !rawLine.split("/").includes(".."), + `manifest line ${lineNumber} traverses outside the release`, + ); + assert(!seen.has(rawLine), `manifest contains duplicate path ${rawLine}`); + seen.add(rawLine); + paths.push(rawLine); + } + + assert(paths.length > 0, "release manifest is empty"); + return paths.sort(compareStrings); +} + +export async function verifyActionRelease(options) { + const client = options.client; + const manifestUrl = options.manifestUrl ?? DEFAULT_MANIFEST_URL; + const manifest = parseReleaseManifest(await readFile(manifestUrl, "utf8")); + + for (const forbidden of FORBIDDEN_RELEASE_PATHS) { + assert( + !manifest.some( + (path) => path === forbidden || path.startsWith(forbidden), + ), + `release manifest includes forbidden source path ${forbidden}`, + ); + } + + const expectedBlobPaths = EXPECTED_RELEASE_TREE.filter( + ({ type }) => type === "blob", + ) + .map(({ path }) => path) + .sort(compareStrings); + assert( + JSON.stringify(manifest) === JSON.stringify(expectedBlobPaths), + "local release manifest differs from the reviewed runtime allowlist", + ); + + const [branch, tag] = await Promise.all([ + client.getRef(ACTION_REPOSITORY, `heads/${RELEASE_BRANCH}`), + client.getRef(ACTION_REPOSITORY, `tags/${RELEASE_CHANNEL}`), + ]); + const branchSha = branch.object?.sha; + const tagSha = tag.object?.sha; + assert( + branch.object?.type === "commit" && tag.object?.type === "commit", + "release discovery refs must point directly to commits", + ); + assert(branchSha === tagSha, "release discovery refs do not match"); + assert( + branchSha === APPROVED_RELEASE_SHA, + `release discovery refs point to unapproved SHA ${branchSha ?? "missing"}`, + ); + + const commit = await client.getCommit( + ACTION_REPOSITORY, + APPROVED_RELEASE_SHA, + ); + assert( + commit.sha === APPROVED_RELEASE_SHA, + "release commit SHA does not match", + ); + assert( + commit.verification?.verified === true && + commit.verification?.reason === "valid", + "release commit is not GitHub-verified", + ); + assert( + typeof commit.tree?.sha === "string" && + /^[0-9a-f]{40}$/.test(commit.tree.sha), + "release commit has an invalid tree SHA", + ); + + const [tree, rootActionText, teardownActionText] = await Promise.all([ + client.getTree(ACTION_REPOSITORY, commit.tree.sha, true), + client.getText(ACTION_REPOSITORY, "action.yml", APPROVED_RELEASE_SHA), + client.getText( + ACTION_REPOSITORY, + "teardown/action.yml", + APPROVED_RELEASE_SHA, + ), + ]); + assert( + tree.sha === commit.tree.sha, + "release commit and recursive tree SHAs do not match", + ); + assert(tree.truncated === false, "release tree response is truncated"); + assert(Array.isArray(tree.tree), "release tree response has no entries"); + assertExactTree(tree.tree, EXPECTED_RELEASE_TREE); + + const rootAction = parseAction(rootActionText, "action.yml"); + const teardownAction = parseAction(teardownActionText, "teardown/action.yml"); + assertRuntimeReference( + rootAction, + "action.yml", + 'bash "$GITHUB_ACTION_PATH/scripts/configure.sh"', + ); + assertRuntimeReference( + teardownAction, + "teardown/action.yml", + 'bash "$GITHUB_ACTION_PATH/../scripts/teardown.sh"', + ); + + return { + branch: RELEASE_BRANCH, + channel: RELEASE_CHANNEL, + commitVerified: true, + manifest, + repository: ACTION_REPOSITORY, + sha: APPROVED_RELEASE_SHA, + treeSha: commit.tree?.sha, + }; +} diff --git a/tools/rollout/rollout.test.mjs b/tools/rollout/rollout.test.mjs new file mode 100644 index 0000000..d64fddf --- /dev/null +++ b/tools/rollout/rollout.test.mjs @@ -0,0 +1,419 @@ +import assert from "node:assert/strict"; +import { readFile } from "node:fs/promises"; +import { describe, test } from "node:test"; + +import { + ACTION_REPOSITORY, + APPROVED_RELEASE_SHA, + EXPECTED_RELEASE_TREE, + RELEASE_BRANCH, + RELEASE_CHANNEL, +} from "./constants.mjs"; +import { GhCommandError, GitHubClient } from "./github.mjs"; +import { + captureRepositoryInventory, + reconcileRepositoryInventories, +} from "./inventory.mjs"; +import { main } from "./cli.mjs"; +import { parseReleaseManifest, verifyActionRelease } from "./release.mjs"; + +const TREE_SHA = "5cdbe39b0edafee9457767134320d95c61d91a60"; + +const ROOT_ACTION = ` +name: Setup Socket Firewall +runs: + using: composite + steps: + - shell: bash + run: bash "$GITHUB_ACTION_PATH/scripts/configure.sh" +`; +const TEARDOWN_ACTION = ` +name: Teardown Socket Firewall +runs: + using: composite + steps: + - shell: bash + run: bash "$GITHUB_ACTION_PATH/../scripts/teardown.sh" +`; + +function releaseClient(overrides = {}) { + return { + async getCommit() { + return ( + overrides.commit ?? { + sha: APPROVED_RELEASE_SHA, + tree: { sha: TREE_SHA }, + verification: { reason: "valid", verified: true }, + } + ); + }, + async getRef(_repository, ref) { + const defaultResponse = { + object: { sha: APPROVED_RELEASE_SHA, type: "commit" }, + }; + return overrides.refs?.[ref] ?? defaultResponse; + }, + async getText(_repository, path) { + if (path === "action.yml") { + return overrides.rootAction ?? ROOT_ACTION; + } + if (path === "teardown/action.yml") { + return overrides.teardownAction ?? TEARDOWN_ACTION; + } + throw new Error(`unexpected path ${path}`); + }, + async getTree() { + return ( + overrides.tree ?? { + sha: TREE_SHA, + tree: EXPECTED_RELEASE_TREE, + truncated: false, + } + ); + }, + }; +} + +function restRepository(name, options = {}) { + return { + archived: options.archived ?? false, + default_branch: options.defaultBranch ?? "main", + name, + visibility: options.visibility ?? "private", + }; +} + +function graphqlRepository(name, options = {}) { + return { + isArchived: options.archived ?? false, + name, + visibility: (options.visibility ?? "private").toUpperCase(), + }; +} + +describe("source dependency lockfile", () => { + test("contains no Socket Firewall resolution URL", async () => { + const lockfile = await readFile( + new URL("../../package-lock.json", import.meta.url), + "utf8", + ); + assert.doesNotMatch( + lockfile, + /https?:\/\/[^/]*(?:socket-firewall|socket\.dev)/i, + ); + }); +}); + +describe("release verification", () => { + test("accepts the exact signed action-only release", async () => { + const result = await verifyActionRelease({ client: releaseClient() }); + + assert.deepEqual(result, { + branch: RELEASE_BRANCH, + channel: RELEASE_CHANNEL, + commitVerified: true, + manifest: [ + "LICENSE", + "action.yml", + "scripts/configure.sh", + "scripts/teardown.sh", + "teardown/action.yml", + ], + repository: ACTION_REPOSITORY, + sha: APPROVED_RELEASE_SHA, + treeSha: TREE_SHA, + }); + }); + + test("rejects mismatched discovery refs", async () => { + const client = releaseClient({ + refs: { + [`tags/${RELEASE_CHANNEL}`]: { + object: { sha: "f".repeat(40), type: "commit" }, + }, + }, + }); + + await assert.rejects( + verifyActionRelease({ client }), + /release discovery refs do not match/, + ); + }); + + test("rejects an unverified release commit", async () => { + const client = releaseClient({ + commit: { + sha: APPROVED_RELEASE_SHA, + tree: { sha: TREE_SHA }, + verification: { reason: "unsigned", verified: false }, + }, + }); + + await assert.rejects( + verifyActionRelease({ client }), + /release commit is not GitHub-verified/, + ); + }); + + test("rejects extra release-tree content", async () => { + const client = releaseClient({ + tree: { + sha: TREE_SHA, + tree: [ + ...EXPECTED_RELEASE_TREE, + { mode: "100644", path: "tools/audit.mjs", type: "blob" }, + ], + truncated: false, + }, + }); + + await assert.rejects( + verifyActionRelease({ client }), + /release tree differs from the reviewed manifest/, + ); + }); + + test("rejects action metadata that executes another local path", async () => { + const client = releaseClient({ + rootAction: ` +name: Unsafe +runs: + using: composite + steps: + - uses: ./tools +`, + }); + + await assert.rejects(verifyActionRelease({ client }), /must execute only/); + }); + + test("rejects unsafe manifest paths", () => { + assert.throws( + () => parseReleaseManifest("action.yml\n../secret\n"), + /traverses outside/, + ); + assert.throws( + () => parseReleaseManifest("action.yml\naction.yml\n"), + /duplicate path/, + ); + assert.throws( + () => parseReleaseManifest(" action.yml\n"), + /has whitespace/, + ); + }); +}); + +describe("GitHub adapter", () => { + test("paginates REST repository inventory without omission", async () => { + const repositories = Array.from({ length: 301 }, (_, index) => + restRepository(`repo-${String(index).padStart(3, "0")}`), + ); + const pages = []; + const client = new GitHubClient({ + async execute(args) { + const endpoint = args.at(-1); + const page = Number( + new URL(`https://api.github.test/${endpoint}`).searchParams.get( + "page", + ), + ); + pages.push(page); + const start = (page - 1) * 100; + return { + stderr: "", + stdout: JSON.stringify(repositories.slice(start, start + 100)), + }; + }, + }); + + const result = await client.listRestRepositories("workos"); + + assert.equal(result.length, 301); + assert.deepEqual(pages, [1, 2, 3, 4]); + }); + + test("runs the independent GraphQL-backed repository command", async () => { + let received; + const response = [graphqlRepository("one")]; + const client = new GitHubClient({ + async execute(args) { + received = args; + return { stderr: "", stdout: JSON.stringify(response) }; + }, + }); + + assert.deepEqual(await client.listGraphqlRepositories("workos"), response); + assert.deepEqual(received, [ + "repo", + "list", + "workos", + "--limit", + "10000", + "--json", + "name,isArchived,visibility", + ]); + }); + + test("honors retry-after for a bounded rate-limit retry", async () => { + let attempts = 0; + const delays = []; + const client = new GitHubClient({ + async execute() { + attempts += 1; + if (attempts === 1) { + throw new GhCommandError("rate limited", { + retryAfterMs: 7_000, + status: 429, + }); + } + return { stderr: "", stdout: "[]" }; + }, + async sleep(delay) { + delays.push(delay); + }, + }); + + assert.deepEqual(await client.api("example"), []); + assert.equal(attempts, 2); + assert.deepEqual(delays, [7_000]); + }); + + test("fails fast for a non-rate-limit 403", async () => { + let attempts = 0; + const client = new GitHubClient({ + defaultRetryDelayMs: 1, + async execute() { + attempts += 1; + throw new GhCommandError("forbidden", { status: 403 }); + }, + async sleep() {}, + }); + + await assert.rejects(client.api("example"), /forbidden/); + assert.equal(attempts, 1); + }); +}); + +describe("repository inventory", () => { + test("reconciles exact REST and GraphQL active sets", () => { + const rest = [ + restRepository("public-repo", { visibility: "public" }), + restRepository("internal-repo", { visibility: "internal" }), + restRepository("old-repo", { archived: true }), + ]; + const graphql = [ + graphqlRepository("internal-repo", { visibility: "internal" }), + graphqlRepository("old-repo", { archived: true }), + graphqlRepository("public-repo", { visibility: "public" }), + ]; + + assert.deepEqual(reconcileRepositoryInventories(rest, graphql), { + activeCount: 2, + archivedCount: 1, + differences: { + activeOnlyGraphql: [], + activeOnlyRest: [], + allOnlyGraphql: [], + allOnlyRest: [], + visibilityMismatches: [], + }, + repositories: [ + { + defaultBranch: "main", + name: "internal-repo", + visibility: "internal", + }, + { + defaultBranch: "main", + name: "public-repo", + visibility: "public", + }, + ], + schemaVersion: 1, + totalCount: 3, + visibility: { internal: 1, private: 0, public: 1 }, + }); + }); + + test("fails closed when repository visibility differs", () => { + assert.throws( + () => + reconcileRepositoryInventories( + [restRepository("one", { visibility: "private" })], + [graphqlRepository("one", { visibility: "public" })], + ), + /inventories differ/, + ); + }); + + test("rejects a malformed archived state", () => { + assert.throws( + () => + reconcileRepositoryInventories( + [{ ...restRepository("one"), archived: "false" }], + [graphqlRepository("one")], + ), + /invalid archived state/, + ); + }); + + test("fails closed when either inventory omits a repository", () => { + assert.throws( + () => + reconcileRepositoryInventories( + [restRepository("one"), restRepository("two")], + [graphqlRepository("one")], + ), + /inventories differ/, + ); + }); + + test("captures both sources before returning inventory", async () => { + const calls = []; + const client = { + async listGraphqlRepositories() { + calls.push("graphql"); + return [graphqlRepository("one")]; + }, + async listRestRepositories() { + calls.push("rest"); + return [restRepository("one")]; + }, + }; + + const result = await captureRepositoryInventory(client, "workos"); + + assert.equal(result.activeCount, 1); + assert.deepEqual(calls.sort(), ["graphql", "rest"]); + }); +}); + +describe("CLI", () => { + test("rejects unknown commands and extra arguments", async () => { + await assert.rejects(main([], {}), /usage:/); + await assert.rejects(main(["inventory", "--org", "other"], {}), /usage:/); + }); + + test("prints deterministic inventory JSON", async () => { + let output = ""; + const client = { + async listGraphqlRepositories() { + return [graphqlRepository("one")]; + }, + async listRestRepositories() { + return [restRepository("one")]; + }, + }; + + await main(["inventory"], { + client, + output: { + write(value) { + output += value; + }, + }, + }); + + assert.equal(JSON.parse(output).activeCount, 1); + }); +}); From be16bb4459bd7ee1a5c072780e5aba61f60cffb1 Mon Sep 17 00:00:00 2001 From: Matt Peake Date: Thu, 3 Sep 2026 11:05:18 -0400 Subject: [PATCH 02/19] feat: use PUBLIC_SOCKET_FIREWALL_TOKEN for this public repository --- .github/workflows/ci.yml | 14 +++++++------- README.md | 4 ++-- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 39e727b..fb0a426 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -24,7 +24,7 @@ jobs: - name: Configure Socket Firewall for verifier dependencies uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 with: - token: ${{ secrets.SOCKET_FIREWALL_TOKEN }} + token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} allow-external-fork-fallback: true - name: Install verifier dependencies run: npm ci --ignore-scripts --no-audit --no-fund @@ -59,18 +59,18 @@ jobs: outputs: has-token: ${{ steps.probe.outputs.has-token }} steps: - - name: Probe SOCKET_FIREWALL_TOKEN + - name: Probe PUBLIC_SOCKET_FIREWALL_TOKEN id: probe - run: echo "has-token=${{ secrets.SOCKET_FIREWALL_TOKEN != '' }}" >> "$GITHUB_OUTPUT" + run: echo "has-token=${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN != '' }}" >> "$GITHUB_OUTPUT" - name: Require token for trusted runs env: - HAS_TOKEN: ${{ secrets.SOCKET_FIREWALL_TOKEN != '' }} + HAS_TOKEN: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN != '' }} EVENT_NAME: ${{ github.event_name }} HEAD_IS_FORK: ${{ github.event.pull_request.head.repo.fork }} run: | set -euo pipefail if [[ "$HAS_TOKEN" == false && !( "$EVENT_NAME" == pull_request && "$HEAD_IS_FORK" == true ) ]]; then - echo '::error::SOCKET_FIREWALL_TOKEN is required for trusted pull requests and default-branch runs.' + echo '::error::PUBLIC_SOCKET_FIREWALL_TOKEN is required for trusted pull requests and default-branch runs.' exit 1 fi @@ -105,7 +105,7 @@ jobs: id: sfw uses: ./ with: - token: ${{ secrets.SOCKET_FIREWALL_TOKEN }} + token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} configure-bun: ${{ matrix.manager == 'bun' }} - name: Assert protected configuration @@ -241,7 +241,7 @@ jobs: - name: Configure Socket Firewall uses: ./ with: - token: ${{ secrets.SOCKET_FIREWALL_TOKEN }} + token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} - name: Prove public-resolved Yarn lockfile fails closed run: | diff --git a/README.md b/README.md index 465ffad..3328205 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ For a Bun dependency-install job, also set `configure-bun: true`. This writes a The token is fail-closed by default. Private, internal, trusted/default-branch, and Dependabot jobs stop before dependency download when the token is absent. -`SOCKET_FIREWALL_TOKEN` is an organization secret. Dependabot uses a separate secret store: provision the same secret there for dependency-update runs that must pass, or accept the intentional fail-closed result. Ask in `#ask-foundation` about repository selection or token delivery. +`SOCKET_FIREWALL_TOKEN` is an organization secret available to private and internal repositories. Public repositories never receive it; an approved public repository is instead individually selected into the separate `PUBLIC_SOCKET_FIREWALL_TOKEN` organization secret and passes that secret to the same `token` input. Dependabot uses a separate secret store: provision the same secret there for dependency-update runs that must pass, or accept the intentional fail-closed result. Ask in `#ask-foundation` about repository selection or token delivery. ### Public external-fork usage @@ -69,7 +69,7 @@ steps: - name: Configure Socket Firewall uses: workos/setup-socket-firewall@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 with: - token: ${{ secrets.SOCKET_FIREWALL_TOKEN }} + token: ${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }} allow-external-fork-fallback: true - run: npm ci From 36357b5582fddd4b667ee26e84bc73c596a80e34 Mon Sep 17 00:00:00 2001 From: Matt Peake Date: Thu, 3 Sep 2026 11:38:31 -0400 Subject: [PATCH 03/19] feat: add workflow classifier and live audit command --- package.json | 1 + reports/.gitkeep | 0 tools/rollout/audit.mjs | 280 ++++++++++++++ tools/rollout/classify.mjs | 661 ++++++++++++++++++++++++++++++++ tools/rollout/classify.test.mjs | 397 +++++++++++++++++++ tools/rollout/cli.mjs | 26 +- tools/rollout/github.mjs | 17 +- 7 files changed, 1376 insertions(+), 6 deletions(-) create mode 100644 reports/.gitkeep create mode 100644 tools/rollout/audit.mjs create mode 100644 tools/rollout/classify.mjs create mode 100644 tools/rollout/classify.test.mjs diff --git a/package.json b/package.json index db1f94b..93fc9fb 100644 --- a/package.json +++ b/package.json @@ -8,6 +8,7 @@ "node": ">=22" }, "scripts": { + "audit:live": "node tools/rollout/cli.mjs audit", "check": "npm run format:check && npm test && npm run verify-action", "format": "prettier --write package.json tools/rollout/*.mjs", "format:check": "prettier --check package.json tools/rollout/*.mjs", diff --git a/reports/.gitkeep b/reports/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/tools/rollout/audit.mjs b/tools/rollout/audit.mjs new file mode 100644 index 0000000..29f06a1 --- /dev/null +++ b/tools/rollout/audit.mjs @@ -0,0 +1,280 @@ +import { mkdtemp, rename, rm, writeFile } from "node:fs/promises"; +import { dirname, join } from "node:path"; + +import { classifyWorkflow, repositoryDisposition } from "./classify.mjs"; +import { captureRepositoryInventory } from "./inventory.mjs"; +import { ORGANIZATION } from "./constants.mjs"; + +const WORKFLOW_PATH_PATTERN = /^\.(?:github|depot)\/workflows\/[^/]+\.ya?ml$/; +const LOCKFILE_PATTERN = + /(?:^|\/)(?:package-lock\.json|npm-shrinkwrap\.json|pnpm-lock\.yaml|yarn\.lock|bun\.lockb?)$/; +const LOCAL_ACTION_PATTERN = /uses:\s*['"]?\.\/([^\s'"#]+)/g; +const AUDIT_CONCURRENCY = 5; +const MAX_WORKFLOWS_PER_REPOSITORY = 200; +const ADVISORY_WORKSPACE_PATTERN = /-ghsa(?:-[a-z0-9]{4}){3}$/; + +function sortedByName(rows) { + return [...rows].sort((left, right) => + left.name < right.name ? -1 : left.name > right.name ? 1 : 0, + ); +} + +async function mapWithConcurrency(items, limit, mapper) { + const results = new Array(items.length); + let nextIndex = 0; + + async function worker() { + while (nextIndex < items.length) { + const index = nextIndex; + nextIndex += 1; + results[index] = await mapper(items[index], index); + } + } + + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, () => worker()), + ); + return results; +} + +async function isEmptyRepository(client, repository) { + try { + const metadata = await client.api( + `repos/${repository}`, + `read ${repository} metadata`, + ); + return metadata.size === 0; + } catch { + return false; + } +} + +export async function auditRepository(client, repository) { + const fullName = `${ORGANIZATION}/${repository.name}`; + + let headSha; + let tree; + try { + const ref = await client.getRef( + fullName, + `heads/${repository.defaultBranch}`, + ); + headSha = ref?.object?.sha; + if (typeof headSha !== "string" || headSha.length !== 40) { + throw new Error(`unexpected head ref shape for ${fullName}`); + } + tree = await client.getTree( + fullName, + encodeURIComponent(repository.defaultBranch), + true, + ); + } catch (error) { + if ( + error.message?.includes("HTTP 404") && + ADVISORY_WORKSPACE_PATTERN.test(repository.name) + ) { + return { + defaultBranch: repository.defaultBranch, + disposition: "advisory-workspace", + lockfiles: [], + name: repository.name, + visibility: repository.visibility, + workflows: [], + }; + } + if (await isEmptyRepository(client, fullName)) { + return { + defaultBranch: repository.defaultBranch, + disposition: "empty", + jobs: { total: 0 }, + lockfiles: [], + name: repository.name, + visibility: repository.visibility, + workflows: [], + }; + } + return { + defaultBranch: repository.defaultBranch, + disposition: "audit-error", + error: error.message, + lockfiles: [], + name: repository.name, + visibility: repository.visibility, + workflows: [], + }; + } + + if (tree.truncated !== false) { + return { + defaultBranch: repository.defaultBranch, + disposition: "audit-error", + error: "default-branch tree listing is truncated", + headSha, + lockfiles: [], + name: repository.name, + visibility: repository.visibility, + workflows: [], + }; + } + + const blobPaths = new Set( + (tree.tree ?? []) + .filter((entry) => entry.type === "blob") + .map((entry) => entry.path), + ); + const workflowPaths = [...blobPaths] + .filter((path) => WORKFLOW_PATH_PATTERN.test(path)) + .sort(); + const lockfiles = [...blobPaths] + .filter((path) => LOCKFILE_PATTERN.test(path)) + .sort() + .slice(0, 50); + + if (workflowPaths.length > MAX_WORKFLOWS_PER_REPOSITORY) { + return { + defaultBranch: repository.defaultBranch, + disposition: "audit-error", + error: `repository has ${workflowPaths.length} workflow files; refusing unbounded scan`, + headSha, + lockfiles, + name: repository.name, + visibility: repository.visibility, + workflows: [], + }; + } + + try { + const workflowTexts = await mapWithConcurrency( + workflowPaths, + AUDIT_CONCURRENCY, + (path) => client.getText(fullName, path, headSha), + ); + + const localActionPaths = new Set(); + for (const text of workflowTexts) { + for (const match of text.matchAll(LOCAL_ACTION_PATTERN)) { + const base = match[1].replace(/\/+$/, ""); + for (const candidate of [`${base}/action.yml`, `${base}/action.yaml`]) { + if (blobPaths.has(candidate)) { + localActionPaths.add(candidate); + } + } + } + } + const localActions = new Map(); + await mapWithConcurrency( + [...localActionPaths].sort(), + AUDIT_CONCURRENCY, + async (path) => { + localActions.set(path, await client.getText(fullName, path, headSha)); + }, + ); + + const workflows = workflowPaths.map((path, index) => + classifyWorkflow(workflowTexts[index], { + localActions, + path, + visibility: repository.visibility, + }), + ); + + const managers = [ + ...new Set( + workflows.flatMap((workflow) => + workflow.jobs.flatMap((job) => job.managers), + ), + ), + ].sort(); + const statusCounts = {}; + for (const workflow of workflows) { + for (const job of workflow.jobs) { + statusCounts[job.status] = (statusCounts[job.status] ?? 0) + 1; + } + } + + return { + defaultBranch: repository.defaultBranch, + disposition: repositoryDisposition(workflows), + headSha, + jobs: { + byStatus: Object.fromEntries( + Object.entries(statusCounts).sort(([a], [b]) => (a < b ? -1 : 1)), + ), + total: Object.values(statusCounts).reduce( + (total, count) => total + count, + 0, + ), + }, + lockfiles, + managers, + name: repository.name, + visibility: repository.visibility, + workflows, + }; + } catch (error) { + return { + defaultBranch: repository.defaultBranch, + disposition: "audit-error", + error: error.message, + headSha, + lockfiles, + name: repository.name, + visibility: repository.visibility, + workflows: [], + }; + } +} + +export async function runAudit(client, options = {}) { + const progress = options.progress ?? (() => {}); + const inventory = await captureRepositoryInventory(client, ORGANIZATION); + + let completed = 0; + const repositories = await mapWithConcurrency( + inventory.repositories, + AUDIT_CONCURRENCY, + async (repository) => { + const row = await auditRepository(client, repository); + completed += 1; + progress(completed, inventory.repositories.length, repository.name); + return row; + }, + ); + + const rows = sortedByName(repositories); + const dispositions = {}; + for (const row of rows) { + dispositions[row.disposition] = (dispositions[row.disposition] ?? 0) + 1; + } + + return { + dispositions: Object.fromEntries( + Object.entries(dispositions).sort(([a], [b]) => (a < b ? -1 : 1)), + ), + generatedAt: new Date().toISOString(), + inventory: { + activeCount: inventory.activeCount, + archivedCount: inventory.archivedCount, + differences: inventory.differences, + totalCount: inventory.totalCount, + visibility: inventory.visibility, + }, + organization: ORGANIZATION, + repositories: rows, + schemaVersion: 1, + }; +} + +export async function writeReportAtomically(reportPath, report) { + const directory = dirname(reportPath); + const temporaryDirectory = await mkdtemp(join(directory, ".audit-")); + const temporaryPath = join(temporaryDirectory, "report.json"); + try { + await writeFile(temporaryPath, `${JSON.stringify(report, null, 2)}\n`, { + mode: 0o600, + }); + await rename(temporaryPath, reportPath); + } finally { + await rm(temporaryDirectory, { force: true, recursive: true }); + } +} diff --git a/tools/rollout/classify.mjs b/tools/rollout/classify.mjs new file mode 100644 index 0000000..57c0e09 --- /dev/null +++ b/tools/rollout/classify.mjs @@ -0,0 +1,661 @@ +import { parse } from "yaml"; + +import { ACTION_REPOSITORY, APPROVED_RELEASE_SHA } from "./constants.mjs"; + +const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/; +const UNSAFE_PUBLIC_TRIGGERS = new Set([ + "issue_comment", + "pull_request_target", + "repository_dispatch", + "workflow_run", +]); +const CONTRIBUTOR_HEAD_PATTERN = + /github\.event\.pull_request\.head|github\.head_ref|\bhead\.sha\b|\bhead\.ref\b/; + +const NPM_DOWNLOAD = new Set([ + "add", + "audit", + "ci", + "clean-install", + "dedupe", + "exec", + "i", + "install", + "install-ci-test", + "install-test", + "update", + "upgrade", + "x", +]); +const NPM_NO_NETWORK = new Set([ + "build", + "cache", + "config", + "get", + "help", + "link", + "login", + "logout", + "ls", + "outdated", + "pack", + "ping", + "prune", + "rebuild", + "restart", + "run", + "run-script", + "set", + "start", + "stop", + "t", + "test", + "version", + "view", + "whoami", +]); +const PNPM_DOWNLOAD = new Set([ + "add", + "dlx", + "fetch", + "i", + "import", + "install", + "install-test", + "up", + "update", +]); +const PNPM_NO_NETWORK = new Set([ + "build", + "config", + "exec", + "lint", + "list", + "ls", + "pack", + "prune", + "rebuild", + "run", + "start", + "t", + "test", + "version", + "why", +]); +const BUN_DOWNLOAD = new Set(["add", "i", "install", "update", "x"]); +const BUN_NO_NETWORK = new Set(["build", "run", "test"]); +const YARN_DOWNLOAD = new Set(["add", "dlx", "i", "import", "install", "up"]); +const OTHER_ECOSYSTEM_COMMANDS = new Set([ + "apk", + "apt", + "apt-get", + "brew", + "bundle", + "cargo", + "composer", + "conda", + "docker", + "dotnet", + "gem", + "go", + "gradle", + "helm", + "mvn", + "nuget", + "pip", + "pip3", + "pipenv", + "poetry", + "terraform", + "uv", +]); +const WRAPPER_COMMANDS = new Set(["just", "make", "mise", "rake", "task"]); +const NO_NETWORK_ORCHESTRATORS = new Set(["lerna", "nx", "rush", "turbo"]); + +function normalizeExpression(value) { + return String(value ?? "").replaceAll(/\s+/g, ""); +} + +function expectedTokenExpression(visibility) { + const secret = + visibility === "public" + ? "PUBLIC_SOCKET_FIREWALL_TOKEN" + : "SOCKET_FIREWALL_TOKEN"; + return `\${{secrets.${secret}}}`; +} + +function splitCommands(script) { + return String(script) + .split(/\r?\n|&&|\|\||;|(? command.trim()) + .filter((command) => command.length > 0 && !command.startsWith("#")); +} + +function commandWords(command) { + const words = command.split(/\s+/).filter((word) => word.length > 0); + let start = 0; + while ( + start < words.length && + (/^[A-Za-z_][A-Za-z0-9_]*=/.test(words[start]) || + ["sudo", "time", "env", "exec"].includes(words[start])) + ) { + start += 1; + } + return words.slice(start); +} + +function firstSubcommand(words) { + return words.slice(1).find((word) => !word.startsWith("-")); +} + +export function classifyCommand(command) { + const words = commandWords(command); + const program = words[0]; + if (!program) { + return { command, kind: "no-network" }; + } + const name = program.replace(/^.*\//, ""); + + if (name === "npx" || name === "bunx") { + return { + command, + kind: "js-public-download", + manager: name === "npx" ? "npm" : "bun", + }; + } + if (name === "corepack") { + const subcommand = firstSubcommand(words); + if (subcommand === "enable" || subcommand === "disable") { + return { command, kind: "no-network" }; + } + return { command, kind: "js-public-download", manager: "npm" }; + } + if (name === "npm" || name === "pnpm" || name === "bun" || name === "yarn") { + const subcommand = firstSubcommand(words); + if (name === "yarn") { + if ( + subcommand === "publish" || + (subcommand === "npm" && words.includes("publish")) + ) { + return { command, kind: "js-publish", manager: "yarn" }; + } + if (subcommand === undefined || YARN_DOWNLOAD.has(subcommand)) { + return { command, kind: "yarn-blocked", manager: "yarn" }; + } + return { command, kind: "no-network" }; + } + if (subcommand === "publish") { + return { command, kind: "js-publish", manager: name }; + } + const downloads = + name === "npm" + ? NPM_DOWNLOAD + : name === "pnpm" + ? PNPM_DOWNLOAD + : BUN_DOWNLOAD; + const quiet = + name === "npm" + ? NPM_NO_NETWORK + : name === "pnpm" + ? PNPM_NO_NETWORK + : BUN_NO_NETWORK; + if (subcommand !== undefined && downloads.has(subcommand)) { + if (name === "npm" && subcommand === "audit" && !words.includes("fix")) { + return { command, kind: "no-network" }; + } + return { command, kind: "js-public-download", manager: name }; + } + if ( + subcommand === undefined || + quiet.has(subcommand) || + (name === "bun" && /[./]/.test(subcommand)) + ) { + return { command, kind: "no-network" }; + } + return { command, kind: "unknown", manager: name }; + } + if (OTHER_ECOSYSTEM_COMMANDS.has(name)) { + return { command, kind: "other-ecosystem" }; + } + if (NO_NETWORK_ORCHESTRATORS.has(name)) { + if (name === "lerna" && words.includes("bootstrap")) { + return { command, kind: "js-public-download", manager: "npm" }; + } + return { command, kind: "no-network" }; + } + if ( + WRAPPER_COMMANDS.has(name) || + /\.sh$/.test(name) || + ((name === "bash" || name === "sh" || name === "zsh") && + words.slice(1).some((word) => /\.sh$/.test(word))) + ) { + return { command, kind: "unknown-wrapper" }; + } + return { command, kind: "no-network" }; +} + +function parseUses(uses) { + const value = String(uses); + if (value.startsWith("./")) { + return { + kind: "local", + path: value.replace(/^\.\//, "").replace(/\/+$/, ""), + }; + } + if (value.startsWith("docker://")) { + return { image: value, kind: "docker" }; + } + const atIndex = value.lastIndexOf("@"); + const ref = atIndex === -1 ? "" : value.slice(atIndex + 1); + const target = atIndex === -1 ? value : value.slice(0, atIndex); + const segments = target.split("/"); + return { + kind: "remote", + ref, + repository: segments.slice(0, 2).join("/"), + subpath: segments.slice(2).join("/"), + target, + }; +} + +function classifyUsesStep(step, context) { + const uses = parseUses(step.uses); + const withInput = step.with ?? {}; + + if (uses.kind === "local") { + const actionText = + context.localActions?.get(`${uses.path}/action.yml`) ?? + context.localActions?.get(`${uses.path}/action.yaml`); + if (actionText === undefined) { + return [{ kind: "unknown-local-action", uses: step.uses }]; + } + let action; + try { + action = parse(actionText); + } catch { + return [{ kind: "unknown-local-action", uses: step.uses }]; + } + const steps = action?.runs?.steps; + if (action?.runs?.using !== "composite" || !Array.isArray(steps)) { + return [{ kind: "local-action", uses: step.uses }]; + } + return steps.flatMap((inner) => + classifyStep(inner, context).map((operation) => ({ + ...operation, + via: step.uses, + })), + ); + } + if (uses.kind === "docker") { + return [{ kind: "other-ecosystem", uses: step.uses }]; + } + + if (uses.repository === ACTION_REPOSITORY) { + const kind = uses.subpath === "teardown" ? "sfw-teardown" : "sfw-setup"; + if (uses.subpath !== "" && uses.subpath !== "teardown") { + return [{ kind: "unknown", uses: step.uses }]; + } + return [ + { + fallback: normalizeExpression( + withInput["allow-external-fork-fallback"] ?? "false", + ), + kind, + ref: uses.ref, + token: normalizeExpression(withInput.token ?? ""), + uses: step.uses, + }, + ]; + } + if (uses.repository === "actions/setup-node") { + return [ + { + kind: "setup-node", + registryMutating: withInput["registry-url"] !== undefined, + uses: step.uses, + }, + ]; + } + if (uses.repository === "actions/checkout") { + return [ + { + kind: "checkout", + persistCredentials: normalizeExpression( + withInput["persist-credentials"] ?? "true", + ), + ref: String(withInput.ref ?? ""), + repository: String(withInput.repository ?? ""), + uses: step.uses, + }, + ]; + } + if (uses.repository === "pnpm/action-setup") { + const runInstall = withInput.run_install; + if (runInstall !== undefined && String(runInstall) !== "false") { + return [{ kind: "js-public-download", manager: "pnpm", uses: step.uses }]; + } + return [{ kind: "toolchain", uses: step.uses }]; + } + if ( + uses.repository === "changesets/action" || + uses.repository === "JS-DevTools/npm-publish" + ) { + return [{ kind: "js-publish", manager: "npm", uses: step.uses }]; + } + return [{ kind: "remote-action", uses: step.uses }]; +} + +export function classifyStep(step, context) { + if (step === null || typeof step !== "object") { + return [{ kind: "unknown" }]; + } + if (step.uses !== undefined) { + return classifyUsesStep(step, context); + } + if (step.run !== undefined) { + return splitCommands(step.run).map((command) => classifyCommand(command)); + } + return [{ kind: "no-network" }]; +} + +function normalizeTriggers(workflow) { + const triggers = workflow?.on ?? workflow?.[true]; + if (typeof triggers === "string") { + return [triggers]; + } + if (Array.isArray(triggers)) { + return triggers.map(String).sort(); + } + if (triggers && typeof triggers === "object") { + return Object.keys(triggers).sort(); + } + return []; +} + +function collectViolations(operations, context, triggers, job) { + const violations = []; + const downloads = []; + let lastRegistryMutation = -1; + const setups = []; + const teardowns = []; + const publishes = []; + const expectedToken = expectedTokenExpression(context.visibility); + + operations.forEach((operation, index) => { + if (operation.kind === "js-public-download") { + downloads.push(index); + } + if ( + operation.kind === "setup-node" && + operation.registryMutating === true + ) { + lastRegistryMutation = index; + } + if (operation.kind === "js-publish") { + publishes.push(index); + } + if (operation.kind === "sfw-setup") { + setups.push(index); + if (operation.ref !== APPROVED_RELEASE_SHA) { + violations.push( + FULL_SHA_PATTERN.test(operation.ref) + ? `sfw-setup pins unapproved SHA ${operation.ref}` + : `sfw-setup uses mutable or short ref "${operation.ref}"`, + ); + } + if (operation.token !== expectedToken) { + violations.push( + `sfw-setup token must be ${expectedToken} for ${context.visibility} repositories`, + ); + } + if (context.visibility !== "public" && operation.fallback === "true") { + violations.push( + "allow-external-fork-fallback is enabled outside a public repository", + ); + } + } + if (operation.kind === "sfw-teardown") { + teardowns.push(index); + if (operation.ref !== APPROVED_RELEASE_SHA) { + violations.push( + `sfw-teardown ref "${operation.ref}" does not match the approved release SHA`, + ); + } + } + }); + + const firstDownload = downloads[0]; + const validSetup = setups.find( + (index) => + operations[index].ref === APPROVED_RELEASE_SHA && + operations[index].token === expectedToken, + ); + if (firstDownload !== undefined && setups.length > 0) { + if (validSetup === undefined) { + violations.push("no valid Socket Firewall setup precedes downloads"); + } else if (validSetup > firstDownload) { + violations.push("Socket Firewall setup runs after the first download"); + } else if ( + lastRegistryMutation > validSetup && + lastRegistryMutation < firstDownload + ) { + violations.push( + "registry-mutating setup-node runs between Socket Firewall setup and the first download", + ); + } + } + + if (publishes.length > 0 && setups.length > 0) { + const lastDownload = downloads.at(-1) ?? -1; + const validTeardown = teardowns.find( + (index) => + operations[index].ref === APPROVED_RELEASE_SHA && + index > lastDownload && + index < publishes[0], + ); + if (validTeardown === undefined) { + violations.push( + "publish follows Socket Firewall setup without a same-SHA teardown boundary", + ); + } + } + + if (context.visibility === "public") { + const unsafeTriggers = triggers.filter((trigger) => + UNSAFE_PUBLIC_TRIGGERS.has(trigger), + ); + const installBearing = + downloads.length > 0 || setups.length > 0 || publishes.length > 0; + if (unsafeTriggers.length > 0 && installBearing) { + violations.push( + `install-bearing job is reachable from privileged trigger(s): ${unsafeTriggers.join(", ")}`, + ); + } + for (const operation of operations) { + if ( + operation.kind === "checkout" && + installBearing && + (CONTRIBUTOR_HEAD_PATTERN.test(operation.ref) || + CONTRIBUTOR_HEAD_PATTERN.test(operation.repository)) && + unsafeTriggers.length > 0 + ) { + violations.push( + "privileged trigger checks out contributor-controlled head in an install-bearing job", + ); + } + } + } + + if (job?.secrets === "inherit") { + violations.push("reusable workflow call inherits all secrets"); + } + + return violations; +} + +export function classifyJob(jobName, job, context, triggers) { + if (job === null || typeof job !== "object") { + return { + job: jobName, + managers: [], + operations: [], + status: "unknown", + violations: ["job definition is not a mapping"], + }; + } + + if (job.uses !== undefined) { + const violations = + job.secrets === "inherit" + ? ["reusable workflow call inherits all secrets"] + : []; + return { + job: jobName, + managers: [], + operations: [{ kind: "reusable-call", uses: String(job.uses) }], + status: "reusable-call", + violations, + }; + } + + const steps = Array.isArray(job.steps) ? job.steps : []; + const operations = steps.flatMap((step) => classifyStep(step, context)); + const violations = collectViolations(operations, context, triggers, job); + + const kinds = new Set(operations.map((operation) => operation.kind)); + const managers = [ + ...new Set( + operations + .filter((operation) => operation.kind === "js-public-download") + .map((operation) => operation.manager) + .filter(Boolean), + ), + ].sort(); + + const hasDownload = kinds.has("js-public-download"); + const hasPublish = kinds.has("js-publish"); + const hasSetup = kinds.has("sfw-setup"); + const hasUnknown = + kinds.has("unknown") || + kinds.has("unknown-wrapper") || + kinds.has("unknown-local-action"); + const publishUnsafe = violations.some((violation) => + violation.includes("without a same-SHA teardown"), + ); + const trustUnsafe = violations.some( + (violation) => + violation.includes("privileged trigger") || + violation.includes("inherits all secrets"), + ); + + let status; + if (trustUnsafe) { + status = "unsafe-trust"; + } else if (publishUnsafe) { + status = "unsafe-publish"; + } else if (kinds.has("yarn-blocked")) { + status = "blocked-yarn"; + } else if (hasDownload && hasSetup && violations.length === 0) { + status = "protected"; + } else if (hasDownload) { + status = "unprotected"; + } else if (hasUnknown) { + status = "unknown"; + } else if (hasPublish) { + status = hasSetup ? "unsafe-publish" : "safe-publish"; + } else if (kinds.has("other-ecosystem")) { + status = "out-of-scope"; + } else { + status = "no-in-scope-download"; + } + + return { job: jobName, managers, operations, status, violations }; +} + +export function classifyWorkflow(text, context) { + let workflow; + try { + workflow = parse(text); + } catch (error) { + return { + jobs: [], + parseError: error.message, + path: context.path, + status: "unknown", + triggers: [], + }; + } + if (workflow === null || typeof workflow !== "object") { + return { + jobs: [], + parseError: "workflow is not a mapping", + path: context.path, + status: "unknown", + triggers: [], + }; + } + + const triggers = normalizeTriggers(workflow); + const jobEntries = Object.entries(workflow.jobs ?? {}).sort(([a], [b]) => + a < b ? -1 : a > b ? 1 : 0, + ); + const jobs = jobEntries.map(([jobName, job]) => + classifyJob(jobName, job, context, triggers), + ); + + return { jobs, path: context.path, triggers }; +} + +const STATUS_SEVERITY = [ + "unsafe-trust", + "unsafe-publish", + "blocked-yarn", + "unprotected", + "unknown", + "reusable-call", + "protected", + "safe-publish", + "out-of-scope", + "no-in-scope-download", +]; + +export function repositoryDisposition(workflowResults, evidence = {}) { + const statuses = new Set(); + for (const workflow of workflowResults) { + if (workflow.parseError !== undefined) { + statuses.add("unknown"); + } + for (const job of workflow.jobs) { + statuses.add(job.status); + } + } + + if (evidence.error !== undefined) { + return "audit-error"; + } + for (const status of STATUS_SEVERITY) { + if (statuses.has(status)) { + switch (status) { + case "unsafe-trust": + return "blocked-trust"; + case "unsafe-publish": + return "unsafe-publish"; + case "blocked-yarn": + return "blocked-yarn"; + case "unprotected": + return "needs-sfw"; + case "unknown": + case "reusable-call": + return "needs-review"; + case "protected": + return "protected"; + case "safe-publish": + case "out-of-scope": + return "out-of-scope"; + default: + return "no-js-ci"; + } + } + } + return workflowResults.length === 0 ? "no-ci" : "no-js-ci"; +} diff --git a/tools/rollout/classify.test.mjs b/tools/rollout/classify.test.mjs new file mode 100644 index 0000000..26b659a --- /dev/null +++ b/tools/rollout/classify.test.mjs @@ -0,0 +1,397 @@ +import assert from "node:assert/strict"; +import { mkdtemp, readFile, readdir } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import test from "node:test"; + +import { auditRepository, runAudit, writeReportAtomically } from "./audit.mjs"; +import { + classifyCommand, + classifyWorkflow, + repositoryDisposition, +} from "./classify.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; + +const SETUP = `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`; +const TEARDOWN = `workos/setup-socket-firewall/teardown@${APPROVED_RELEASE_SHA}`; +const PRIVATE_TOKEN = "${{ secrets.SOCKET_FIREWALL_TOKEN }}"; +const PUBLIC_TOKEN = "${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }}"; + +function workflow(jobsYaml, triggers = "push") { + return `on: ${triggers}\njobs:\n${jobsYaml}`; +} + +function classify(text, visibility = "private", localActions = new Map()) { + return classifyWorkflow(text, { + localActions, + path: ".github/workflows/ci.yml", + visibility, + }); +} + +test("command grammar", () => { + const cases = [ + ["npm ci --ignore-scripts", "js-public-download"], + ["npm install --no-audit", "js-public-download"], + ["npm audit", "no-network"], + ["npm audit fix", "js-public-download"], + ["npm publish --access public", "js-publish"], + ["npm run build", "no-network"], + ["npm test", "no-network"], + ["npx changeset publish", "js-public-download"], + ["pnpm install --frozen-lockfile", "js-public-download"], + ["pnpm dlx create-thing", "js-public-download"], + ["pnpm exec vitest run", "no-network"], + ["pnpm publish --no-git-checks", "js-publish"], + ["bun install --frozen-lockfile", "js-public-download"], + ["bunx biome check", "js-public-download"], + ["bun run test", "no-network"], + ["bun publish", "js-publish"], + ["yarn install --frozen-lockfile", "yarn-blocked"], + ["yarn", "yarn-blocked"], + ["yarn npm publish", "js-publish"], + ["yarn build", "no-network"], + ["corepack enable", "no-network"], + ["corepack prepare pnpm@10 --activate", "js-public-download"], + ["pip install -r requirements.txt", "other-ecosystem"], + ["go build ./...", "other-ecosystem"], + ["docker build .", "other-ecosystem"], + ["make bootstrap", "unknown-wrapper"], + ["./scripts/setup.sh", "unknown-wrapper"], + ["bash scripts/install.sh", "unknown-wrapper"], + ["turbo run lint", "no-network"], + ["lerna bootstrap", "js-public-download"], + ["CI=1 npm ci", "js-public-download"], + ["echo done", "no-network"], + ]; + for (const [command, kind] of cases) { + assert.equal(classifyCommand(command).kind, kind, command); + } +}); + +test("classifier: unprotected npm install needs SFW", () => { + const result = classify( + workflow( + ` build:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@sha\n - uses: actions/setup-node@sha\n with: { registry-url: "https://registry.npmjs.org/" }\n - run: npm ci\n`, + ), + ); + assert.equal(result.jobs[0].status, "unprotected"); + assert.deepEqual(result.jobs[0].managers, ["npm"]); + assert.equal(repositoryDisposition([result]), "needs-sfw"); +}); + +test("classifier: correctly ordered pinned setup is protected", () => { + const result = classify( + workflow( + ` build:\n runs-on: ubuntu-latest\n steps:\n - uses: actions/checkout@sha\n - uses: actions/setup-node@sha\n with: { registry-url: "https://registry.npmjs.org/" }\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: npm ci\n`, + ), + ); + assert.deepEqual(result.jobs[0].violations, []); + assert.equal(result.jobs[0].status, "protected"); + assert.equal(repositoryDisposition([result]), "protected"); +}); + +test("classifier: registry mutation between setup and download is a violation", () => { + const result = classify( + workflow( + ` build:\n steps:\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - uses: actions/setup-node@sha\n with: { registry-url: "https://registry.npmjs.org/" }\n - run: npm ci\n`, + ), + ); + assert.equal(result.jobs[0].status, "unprotected"); + assert.match(result.jobs[0].violations.join(" "), /registry-mutating/); +}); + +test("classifier: setup after the first download is a violation", () => { + const result = classify( + workflow( + ` build:\n steps:\n - run: npm ci\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n`, + ), + ); + assert.equal(result.jobs[0].status, "unprotected"); + assert.match(result.jobs[0].violations.join(" "), /after the first download/); +}); + +test("classifier: mutable or mismatched refs are violations", () => { + const mutable = classify( + workflow( + ` build:\n steps:\n - uses: workos/setup-socket-firewall@v1\n with: { token: "${PRIVATE_TOKEN}" }\n - run: npm ci\n`, + ), + ); + assert.match(mutable.jobs[0].violations.join(" "), /mutable or short ref/); + + const stale = classify( + workflow( + ` build:\n steps:\n - uses: workos/setup-socket-firewall@${"a".repeat(40)}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: npm ci\n`, + ), + ); + assert.match(stale.jobs[0].violations.join(" "), /unapproved SHA/); +}); + +test("classifier: token must match repository visibility", () => { + const publicWrongSecret = classify( + workflow( + ` build:\n steps:\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: npm ci\n`, + ), + "public", + ); + assert.match( + publicWrongSecret.jobs[0].violations.join(" "), + /PUBLIC_SOCKET_FIREWALL_TOKEN.*public/, + ); + + const publicRight = classify( + workflow( + ` build:\n steps:\n - uses: ${SETUP}\n with: { token: "${PUBLIC_TOKEN}" }\n - run: npm ci\n`, + ), + "public", + ); + assert.equal(publicRight.jobs[0].status, "protected"); + + const internalWrongSecret = classify( + workflow( + ` build:\n steps:\n - uses: ${SETUP}\n with: { token: "${PUBLIC_TOKEN}" }\n - run: npm ci\n`, + ), + "internal", + ); + assert.match( + internalWrongSecret.jobs[0].violations.join(" "), + /SOCKET_FIREWALL_TOKEN.*internal/, + ); +}); + +test("classifier: publish boundaries", () => { + const cleanPublish = classify( + workflow(` publish:\n steps:\n - run: npm publish\n`), + ); + assert.equal(cleanPublish.jobs[0].status, "safe-publish"); + + const restored = classify( + workflow( + ` release:\n steps:\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: pnpm install --frozen-lockfile\n - uses: ${TEARDOWN}\n - run: pnpm publish --no-git-checks\n`, + ), + ); + assert.equal(restored.jobs[0].status, "protected"); + + const unsafe = classify( + workflow( + ` release:\n steps:\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: pnpm install --frozen-lockfile\n - run: pnpm publish --no-git-checks\n`, + ), + ); + assert.equal(unsafe.jobs[0].status, "unsafe-publish"); + assert.equal(repositoryDisposition([unsafe]), "unsafe-publish"); + + const teardownBeforeFinalDownload = classify( + workflow( + ` release:\n steps:\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - uses: ${TEARDOWN}\n - run: pnpm install --frozen-lockfile\n - run: pnpm publish --no-git-checks\n`, + ), + ); + assert.equal(teardownBeforeFinalDownload.jobs[0].status, "unsafe-publish"); +}); + +test("classifier: yarn installs block", () => { + const result = classify( + workflow(` build:\n steps:\n - run: yarn install\n`), + ); + assert.equal(result.jobs[0].status, "blocked-yarn"); + assert.equal(repositoryDisposition([result]), "blocked-yarn"); +}); + +test("classifier: public privileged trigger with installs is unsafe", () => { + const result = classify( + workflow( + ` build:\n steps:\n - uses: actions/checkout@sha\n with: { ref: "\${{ github.event.pull_request.head.sha }}" }\n - run: npm ci\n`, + "pull_request_target", + ), + "public", + ); + assert.equal(result.jobs[0].status, "unsafe-trust"); + assert.equal(repositoryDisposition([result]), "blocked-trust"); +}); + +test("classifier: reusable call with inherited secrets is flagged", () => { + const result = classify( + `on: push\njobs:\n call:\n uses: workos/shared/.github/workflows/ci.yml@main\n secrets: inherit\n`, + ); + assert.equal(result.jobs[0].status, "reusable-call"); + assert.match(result.jobs[0].violations.join(" "), /inherits all secrets/); + assert.equal(repositoryDisposition([result]), "needs-review"); +}); + +test("classifier: pnpm action-setup run_install downloads", () => { + const result = classify( + workflow( + ` build:\n steps:\n - uses: pnpm/action-setup@sha\n with: { run_install: true }\n`, + ), + ); + assert.equal(result.jobs[0].status, "unprotected"); +}); + +test("classifier: local composite actions resolve or block", () => { + const localActions = new Map([ + [ + ".github/actions/install/action.yml", + `runs:\n using: composite\n steps:\n - run: npm ci\n shell: bash\n`, + ], + ]); + const resolved = classify( + workflow(` build:\n steps:\n - uses: ./.github/actions/install\n`), + "private", + localActions, + ); + assert.equal(resolved.jobs[0].status, "unprotected"); + assert.equal(resolved.jobs[0].operations[0].via, "./.github/actions/install"); + + const missing = classify( + workflow(` build:\n steps:\n - uses: ./.github/actions/mystery\n`), + ); + assert.equal(missing.jobs[0].status, "unknown"); + assert.equal(repositoryDisposition([missing]), "needs-review"); +}); + +test("classifier: parse errors and scope buckets fail closed", () => { + const broken = classify("on: [push\njobs: {"); + assert.notEqual(broken.parseError, undefined); + assert.equal(repositoryDisposition([broken]), "needs-review"); + + const python = classify( + workflow( + ` build:\n steps:\n - run: pip install -r requirements.txt\n`, + ), + ); + assert.equal(repositoryDisposition([python]), "out-of-scope"); + + assert.equal(repositoryDisposition([]), "no-ci"); + assert.equal(repositoryDisposition([], { error: "boom" }), "audit-error"); +}); + +function stubClient(overrides = {}) { + const restRepositories = [ + { + archived: false, + default_branch: "main", + name: "app", + visibility: "internal", + }, + { + archived: false, + default_branch: "main", + name: "empty-repo", + visibility: "private", + }, + ]; + return { + api: async (endpoint) => { + if (endpoint === "repos/workos/empty-repo") { + return { size: 0 }; + } + throw new Error(`unexpected api call: ${endpoint}`); + }, + getRef: async (repository) => { + if (repository === "workos/empty-repo") { + const error = new Error("Not Found"); + error.status = 404; + throw error; + } + return { object: { sha: "c".repeat(40) } }; + }, + getText: async (repository, path) => { + assert.equal(path, ".github/workflows/ci.yml"); + return workflow(` build:\n steps:\n - run: npm ci\n`); + }, + getTree: async () => ({ + tree: [ + { path: ".github/workflows/ci.yml", type: "blob" }, + { path: "package-lock.json", type: "blob" }, + ], + truncated: false, + }), + listGraphqlRepositories: async () => + restRepositories.map((repository) => ({ + isArchived: repository.archived, + name: repository.name, + visibility: repository.visibility, + })), + listRestRepositories: async () => restRepositories, + ...overrides, + }; +} + +test("audit: classifies live repositories and writes an atomic report", async () => { + const report = await runAudit(stubClient()); + assert.equal(report.inventory.activeCount, 2); + assert.deepEqual(report.dispositions, { empty: 1, "needs-sfw": 1 }); + const app = report.repositories.find((row) => row.name === "app"); + assert.equal(app.disposition, "needs-sfw"); + assert.deepEqual(app.managers, ["npm"]); + assert.deepEqual(app.lockfiles, ["package-lock.json"]); + + const directory = await mkdtemp(join(tmpdir(), "sfw-audit-")); + const reportPath = join(directory, "report.json"); + await writeReportAtomically(reportPath, report); + const written = JSON.parse(await readFile(reportPath, "utf8")); + assert.deepEqual(written.dispositions, report.dispositions); + assert.deepEqual(await readdir(directory), ["report.json"]); +}); + +test("audit: access failures become audit-error rows, never no-download", async () => { + const client = stubClient({ + getTree: async () => { + const error = new Error("Forbidden"); + error.status = 403; + throw error; + }, + }); + const row = await auditRepository(client, { + defaultBranch: "main", + name: "app", + visibility: "internal", + }); + assert.equal(row.disposition, "audit-error"); + assert.match(row.error, /Forbidden/); +}); + +test("audit: truncated trees fail closed", async () => { + const client = stubClient({ + getTree: async () => ({ tree: [], truncated: true }), + }); + const row = await auditRepository(client, { + defaultBranch: "main", + name: "app", + visibility: "internal", + }); + assert.equal(row.disposition, "audit-error"); + assert.match(row.error, /truncated/); +}); + +test("adapter: transient network failures retry bounded", async () => { + const { GitHubClient } = await import("./github.mjs"); + let attempts = 0; + const client = new GitHubClient({ + execute: async () => { + attempts += 1; + if (attempts < 3) { + throw new Error("dial tcp 140.82.113.6:443: i/o timeout"); + } + return { stdout: "{}" }; + }, + sleep: async () => {}, + }); + await client.api("repos/workos/example", "read example"); + assert.equal(attempts, 3); +}); + +test("adapter: non-transient failures do not retry", async () => { + const { GitHubClient } = await import("./github.mjs"); + let attempts = 0; + const client = new GitHubClient({ + execute: async () => { + attempts += 1; + const error = new Error("gh: Not Found (HTTP 404)"); + error.status = 404; + throw error; + }, + sleep: async () => {}, + }); + await assert.rejects(() => client.api("repos/workos/gone", "read gone")); + assert.equal(attempts, 1); +}); diff --git a/tools/rollout/cli.mjs b/tools/rollout/cli.mjs index f9870f9..8b89805 100644 --- a/tools/rollout/cli.mjs +++ b/tools/rollout/cli.mjs @@ -5,18 +5,40 @@ import { pathToFileURL } from "node:url"; import { GitHubClient } from "./github.mjs"; import { captureRepositoryInventory } from "./inventory.mjs"; import { ORGANIZATION } from "./constants.mjs"; +import { runAudit, writeReportAtomically } from "./audit.mjs"; import { verifyActionRelease } from "./release.mjs"; -const COMMANDS = new Set(["inventory", "verify-action"]); +const COMMANDS = new Set(["audit", "inventory", "verify-action"]); +const REPORT_URL = new URL("../../reports/live-audit.json", import.meta.url); export async function main(argv, options = {}) { if (argv.length !== 1 || !COMMANDS.has(argv[0])) { - throw new Error("usage: cli.mjs "); + throw new Error("usage: cli.mjs "); } const client = options.client ?? new GitHubClient(); const output = options.output ?? process.stdout; const command = argv[0]; + + if (command === "audit") { + const reportPath = options.reportPath ?? REPORT_URL.pathname; + const report = await runAudit(client, { + progress: (done, total) => { + if (done % 25 === 0 || done === total) { + process.stderr.write(`audited ${done}/${total} repositories\n`); + } + }, + }); + await writeReportAtomically(reportPath, report); + const summary = { + dispositions: report.dispositions, + inventory: report.inventory, + reportPath, + }; + output.write(`${JSON.stringify(summary, null, 2)}\n`); + return report; + } + const result = command === "verify-action" ? await verifyActionRelease({ client }) diff --git a/tools/rollout/github.mjs b/tools/rollout/github.mjs index e75a8da..f31297e 100644 --- a/tools/rollout/github.mjs +++ b/tools/rollout/github.mjs @@ -3,7 +3,7 @@ import { promisify } from "node:util"; import { setTimeout as sleep } from "node:timers/promises"; const execFile = promisify(execFileCallback); -const DEFAULT_MAX_ATTEMPTS = 3; +const DEFAULT_MAX_ATTEMPTS = 4; const DEFAULT_PAGE_SIZE = 100; const MAX_REPOSITORY_COUNT = 10_000; const MAX_REPOSITORY_PAGES = MAX_REPOSITORY_COUNT / DEFAULT_PAGE_SIZE; @@ -66,6 +66,7 @@ export class GitHubClient { this.sleep = options.sleep ?? sleep; this.maxAttempts = options.maxAttempts ?? DEFAULT_MAX_ATTEMPTS; this.defaultRetryDelayMs = options.defaultRetryDelayMs ?? 1_000; + this.transientRetryDelayMs = options.transientRetryDelayMs ?? 5_000; if (!Number.isInteger(this.maxAttempts) || this.maxAttempts < 1) { throw new Error("maxAttempts must be a positive integer"); @@ -81,16 +82,24 @@ export class GitHubClient { error.status === 403 && (error.retryAfterMs !== undefined || /rate.?limit/i.test(error.message)); - const retryable = error.status === 429 || rateLimited403; + const transientNetwork = + error.status === undefined && + /timeout|timed out|connection re|unexpected EOF|temporary failure/i.test( + error.message, + ); + const retryable = + error.status === 429 || rateLimited403 || transientNetwork; if (!retryable || attempt === this.maxAttempts) { throw new Error(`${description} failed: ${error.message}`, { cause: error, }); } + const baseDelayMs = transientNetwork + ? this.transientRetryDelayMs + : this.defaultRetryDelayMs; const delay = - error.retryAfterMs ?? - this.defaultRetryDelayMs * 2 ** Math.max(0, attempt - 1); + error.retryAfterMs ?? baseDelayMs * 2 ** Math.max(0, attempt - 1); await this.sleep(delay); } } From d1b1fb88611cd8d159ff1276b41d0a6f33c10c36 Mon Sep 17 00:00:00 2001 From: Matt Peake Date: Thu, 3 Sep 2026 13:28:17 -0400 Subject: [PATCH 04/19] fix: detect Corepack registry bypasses --- tools/rollout/classify.mjs | 34 +++++++++++++++++++++++++++++++-- tools/rollout/classify.test.mjs | 29 ++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 2 deletions(-) diff --git a/tools/rollout/classify.mjs b/tools/rollout/classify.mjs index 57c0e09..3a82fe9 100644 --- a/tools/rollout/classify.mjs +++ b/tools/rollout/classify.mjs @@ -166,9 +166,14 @@ export function classifyCommand(command) { if (name === "corepack") { const subcommand = firstSubcommand(words); if (subcommand === "enable" || subcommand === "disable") { - return { command, kind: "no-network" }; + return { command, corepack: subcommand, kind: "no-network" }; } - return { command, kind: "js-public-download", manager: "npm" }; + return { + command, + corepack: subcommand ?? "implicit", + kind: "js-public-download", + manager: "npm", + }; } if (name === "npm" || name === "pnpm" || name === "bun" || name === "yarn") { const subcommand = firstSubcommand(words); @@ -424,6 +429,31 @@ function collectViolations(operations, context, triggers, job) { } }); + const corepackEnable = operations.findIndex( + (operation) => operation.corepack === "enable", + ); + const firstPnpmDownload = operations.findIndex( + (operation) => + operation.kind === "js-public-download" && operation.manager === "pnpm", + ); + if (corepackEnable !== -1 && firstPnpmDownload > corepackEnable) { + violations.push( + "Corepack may lazily download pnpm directly from registry.npmjs.org; install the pinned pnpm package with npm through Socket Firewall before invoking pnpm", + ); + } + if ( + operations.some( + (operation) => + operation.corepack !== undefined && + operation.corepack !== "enable" && + operation.corepack !== "disable", + ) + ) { + violations.push( + "Corepack package-manager downloads do not use Socket Firewall npm configuration", + ); + } + const firstDownload = downloads[0]; const validSetup = setups.find( (index) => diff --git a/tools/rollout/classify.test.mjs b/tools/rollout/classify.test.mjs index 26b659a..c442d09 100644 --- a/tools/rollout/classify.test.mjs +++ b/tools/rollout/classify.test.mjs @@ -226,6 +226,35 @@ test("classifier: pnpm action-setup run_install downloads", () => { assert.equal(result.jobs[0].status, "unprotected"); }); +test("classifier: Corepack pnpm bootstrap bypasses SFW npm configuration", () => { + const lazyCorepack = classify( + workflow( + ` build:\n steps:\n - run: corepack enable\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: pnpm install --frozen-lockfile\n`, + ), + ); + assert.equal(lazyCorepack.jobs[0].status, "unprotected"); + assert.match(lazyCorepack.jobs[0].violations.join(" "), /Corepack.*lazily/); + + const corepackDownload = classify( + workflow( + ` build:\n steps:\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: corepack prepare pnpm@11.20.0 --activate\n`, + ), + ); + assert.equal(corepackDownload.jobs[0].status, "unprotected"); + assert.match( + corepackDownload.jobs[0].violations.join(" "), + /Corepack package-manager downloads/, + ); + + const npmBootstrap = classify( + workflow( + ` build:\n steps:\n - uses: ${SETUP}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: npm install --global pnpm@11.20.0 --ignore-scripts\n - run: pnpm install --frozen-lockfile\n`, + ), + ); + assert.equal(npmBootstrap.jobs[0].status, "protected"); + assert.deepEqual(npmBootstrap.jobs[0].violations, []); +}); + test("classifier: local composite actions resolve or block", () => { const localActions = new Map([ [ From 619c579fedc53405985f6e1de1dde40191c1cae3 Mon Sep 17 00:00:00 2001 From: Matt Peake Date: Mon, 14 Sep 2026 10:36:17 -0400 Subject: [PATCH 05/19] Add conservative read-only Socket Firewall gap detection --- .github/workflows/ci.yml | 4 +- .gitignore | 1 - CONTRIBUTING.md | 16 +- README.md | 45 +- package.json | 6 +- tools/rollout/audit.mjs | 114 ++-- tools/rollout/classify.mjs | 441 ++++++++++------ tools/rollout/classify.test.mjs | 545 +++++++++++++++++++- tools/rollout/cli.mjs | 44 +- tools/rollout/fixtures/release-manifest.txt | 5 + tools/rollout/github.mjs | 37 +- tools/rollout/regressions.test.mjs | 217 ++++++++ tools/rollout/release.mjs | 2 +- tools/rollout/rollout.test.mjs | 34 +- tools/rollout/yaml.mjs | 18 + 15 files changed, 1289 insertions(+), 240 deletions(-) create mode 100644 tools/rollout/fixtures/release-manifest.txt create mode 100644 tools/rollout/regressions.test.mjs create mode 100644 tools/rollout/yaml.mjs diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2ccd767..27ee976 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,9 +30,7 @@ jobs: run: npm ci --ignore-scripts --no-audit --no-fund - name: Remove Socket Firewall credentials before executing verifier source uses: workos/setup-socket-firewall/teardown@ca93dd8aa351f54f4729fe3377a9be23c631c25d # v1 - - name: Run verifier foundation tests - env: - GH_TOKEN: ${{ github.token }} + - name: Run offline verifier tests run: npm run check - name: Check shell style and test coverage policy run: | diff --git a/.gitignore b/.gitignore index a5f7024..c168e71 100644 --- a/.gitignore +++ b/.gitignore @@ -2,4 +2,3 @@ node_modules/ *.inventory.json reports/* !reports/.gitkeep -!reports/socket-firewall-ci-rollout-final.json diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 899ef9d..f16e791 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -6,7 +6,7 @@ Thanks for helping improve the WorkOS Socket Firewall GitHub Action. - Bash on Linux or macOS - Go, for the pinned `shfmt` check -- Node.js 22 and npm, for the source-only rollout verifier +- Node.js 22 and npm, for the source-only read-only CI gap detector - ShellCheck - Passwordless `sudo` and a disposable Linux runner for integration testing that modifies `/etc/hosts` @@ -28,9 +28,19 @@ npm ci --ignore-scripts --no-audit --no-fund npm run check ``` -CI runs the same static and unit checks on every pull request. Token-backed GitHub-hosted smoke jobs additionally exercise every supported package manager. +`npm run check` runs formatting and pure mocked Node tests offline after dependencies are installed. `npm test` and `npm run test:unit` both run all verifier suites. No credential or live discovery-ref lookup is needed for these tests. The install and pinned Go formatter command above may access the network; they are not offline tests. -The minimum test-coverage policy is one shell test suite for every executable shell source file. Verifier changes require Node fixture tests and must preserve exact REST/GraphQL inventory reconciliation. Changes to supported package-manager behavior must also include a token-backed frozen-lockfile smoke test. +CI runs the same static and unit checks on every pull request. Existing public/fork secret gates and token-backed GitHub-hosted smoke jobs are independent of the detector. Never add a live organization audit or release snapshot verification to normal source CI. + +The minimum test-coverage policy is one shell test suite for every executable shell source file. Verifier changes require synthetic Node fixtures for per-download setup/teardown boundaries, opaque execution, immutable-SHA reads, partial failures and private reporting. Preserve REST/GraphQL token-visible inventory reconciliation without claiming it proves organization-wide completeness. Changes to supported package-manager behavior must also include a token-backed frozen-lockfile smoke test. + +## Operator-only commands + +`npm run inventory` and `npm run audit:live` are read-only manual commands using an existing authorized `gh` session. Confirm organization-wide read access separately before making organization-wide claims. No scheduling or remediation is implicit. See README for the scan scope and limitations. + +Full results are owner-only, ignored `reports/inventory.json` and `reports/live-audit.json`; terminal JSON contains sanitized counts. Audit `scanErrors`/`scanStatus` distinguish operational failures from discovered gaps. Exit 1 includes partial scans; exit 0 can still include `needs-sfw` or `needs-review`. Do not force-add reports, expose inventory/source in logs, or upload full reports as artifacts. + +`npm run verify-action` is a separate live, strict historical release snapshot check, not a normal CI check. Advancing `v1` or changing the current release manifest can intentionally invalidate it. Mocked release tests use `tools/rollout/fixtures/release-manifest.txt`, preserving integrity regression tests without binding source CI to future manifest changes. Keep the actual runtime manifest and action-only publication boundaries intact. ## Pull request guidelines diff --git a/README.md b/README.md index 5868ea4..ad5906b 100644 --- a/README.md +++ b/README.md @@ -148,20 +148,51 @@ No maintainer runs local release commands. The workflow uses only the repository-scoped `GITHUB_TOKEN` with `contents: write`, serializes releases, skips stale successful commits when `main` has advanced, and can be retried through `workflow_dispatch`. A future breaking release must change the reviewed channel to `v2`; it must not repurpose `v1`. -Never publish the normal source commit as an action release: it contains tests and the one-time HELP-724 rollout verifier source. +Never publish the normal source commit as an action release: it contains tests and the read-only CI gap detector source. -## Rollout verifier +## Read-only CI gap detector -The source branch contains operator-only verification tooling that never enters the action release. Its foundation validates the signed action tree and independently reconciles REST and GraphQL WorkOS repository inventories: +The source branch includes an **operator-run candidate detector**, not runtime proof of protection. It never enters the action release and does not change repositories, create PRs, schedule scans, or diagnose general CI health. + +```bash +npm ci --ignore-scripts --no-audit --no-fund +npm run check # offline formatting and mocked Node tests +npm run inventory # read-only, token-visible WorkOS inventory +npm run audit:live # read-only, default-branch workflow candidate scan +``` + +`inventory` and `audit:live` require an existing authenticated `gh` session. Organization-wide conclusions require independently confirmed read access to **every intended repository**, including private/internal repositories (typically `repo` and `read:org`, with organization/SSO authorization as applicable). REST/GraphQL reconciliation checks consistency of the token-visible inventory; both APIs agreeing does **not** prove hidden repositories are absent. Archived repositories are counted but not scanned. Each repository's tree and workflow/local-action source use one captured immutable default-branch SHA; different repositories are not captured transactionally. + +### Results and privacy + +Terminal output is sanitized JSON counts, not repository names or workflow source. Full inventory and audit JSON go to `reports/inventory.json` and `reports/live-audit.json`, respectively. These paths are ignored; reports are atomically replaced with owner-only (`0600`) permissions. Keep them private: never force-add, publish, or upload reports as CI artifacts. Both commands take no CLI flags; a later run replaces the previous report. + +Audit JSON includes `schemaVersion`, `scanStatus`, `scanErrors`, disposition counts, and per-repository `headSha`, workflow/job operations and violations. `scanStatus: complete` means the token-visible scan completed, **not** that all jobs are protected. `partial` means one or more repositories have an `audit-error` row. CLI exit **0** means the scan completed, even when gaps/review candidates were found; exit **1** means an operational failure (including partial scans). Consumers should inspect dispositions separately from the exit code. Fatal inventory/API/command errors emit a sanitized error and exit 1; they do not refresh the report, so check its timestamp before use. + +| Repository disposition | Meaning | +| --- | --- | +| `needs-sfw` | A recognized download lacks a certain supported setup interval. | +| `needs-review` | Opaque behavior, reusable call, or malformed workflow prevents a conclusion. | +| `blocked-trust`, `unsafe-publish`, `blocked-yarn` | Existing trust, teardown/publication, or Yarn compatibility rule needs review. | +| `protected` | All recognized downloads satisfy the static rules, with no detected opaque operation; not runtime verification. | +| `audit-error` | A repository read failed or was incomplete; never a clean result. | +| `no-ci`, `no-js-ci`, `out-of-scope`, `empty` | No recognized in-scope candidate in the inspected source; not an organization-wide assurance. | + +Job statuses remain more detailed (`unprotected`, `unknown`, `reusable-call`, etc.). Ambiguous execution produces `needs-review`, with any recognized violations retained in the private report. A separate, clearly unprotected job still makes the repository a `needs-sfw` candidate. A `safe-publish` job is a static publication-only classification, not proof that lifecycle hooks are safe. + +### Static limits + +The small grammar recognizes direct npm/pnpm/Bun downloads, Yarn blockers, setup/teardown order and registry overrides. Every identified download needs an active approved-SHA setup using the visibility-appropriate token expression. Disabled setup cannot protect; conditional/continue-on-error boundaries, complex shell constructs, containers, unresolved scripts/actions and reusable workflows cannot earn `protected`. Referenced local composites are inspected with bounded nesting and cycle detection; dynamic inputs and inner/outer uncertainty remain review candidates. Public external-fork fallback is an explicit unprotected path, not verified protection. + +Only top-level `.github/workflows/*.yml|yaml` and `.depot/workflows/*.yml|yaml` on active repositories' default branches are inspected. The detector does not execute workflows, resolve package scripts/lifecycle hooks, validate credential existence, prove actual registry traffic, interpret arbitrary shell, inspect remote action/reusable-workflow implementations, or certify bespoke controls. Local-action resolution is deliberately limited; unsupported paths or execution contexts need review. Install hooks and downloaded code remain outside the runtime-proof claim. Treat results as a prioritized human-review queue. + +### Manual release snapshot verification ```bash -npm ci --ignore-scripts -npm test npm run verify-action -npm run inventory ``` -`inventory` requires an existing `gh` session with `repo` and `read:org`. It is read-only, accepts no configuration flags, and fails unless both inventory sources return exactly the same active repository set. Its output can contain internal repository names; do not attach it to a public issue or workflow log. +This separate **live, strict snapshot check** validates discovery refs, the approved signed action-only SHA, exact allowlisted tree, and runtime entrypoints against `tools/rollout/constants.mjs` and the local release manifest. It is not part of `npm run check` or ordinary source tests. It is expected to fail when `v1` advances or the source manifest no longer matches that historical snapshot; this alone is not evidence the current release is unsafe. Review/update the snapshot deliberately for a new release, rather than weakening integrity checks. Offline unit tests use an explicit historical manifest fixture so future manifest additions cannot break unrelated source CI. ## Contributing diff --git a/package.json b/package.json index 93fc9fb..35e8612 100644 --- a/package.json +++ b/package.json @@ -2,19 +2,19 @@ "name": "@workos/setup-socket-firewall-rollout", "version": "0.0.0", "private": true, - "description": "One-time HELP-724 rollout verification tooling", + "description": "Read-only Socket Firewall CI gap detector", "type": "module", "engines": { "node": ">=22" }, "scripts": { "audit:live": "node tools/rollout/cli.mjs audit", - "check": "npm run format:check && npm test && npm run verify-action", + "check": "npm run format:check && npm test", "format": "prettier --write package.json tools/rollout/*.mjs", "format:check": "prettier --check package.json tools/rollout/*.mjs", "inventory": "node tools/rollout/cli.mjs inventory", "test": "node --test tools/rollout/*.test.mjs", - "test:unit": "node --test tools/rollout/rollout.test.mjs", + "test:unit": "node --test tools/rollout/*.test.mjs", "verify-action": "node tools/rollout/cli.mjs verify-action" }, "dependencies": { diff --git a/tools/rollout/audit.mjs b/tools/rollout/audit.mjs index 29f06a1..c111e58 100644 --- a/tools/rollout/audit.mjs +++ b/tools/rollout/audit.mjs @@ -11,7 +11,7 @@ const LOCKFILE_PATTERN = const LOCAL_ACTION_PATTERN = /uses:\s*['"]?\.\/([^\s'"#]+)/g; const AUDIT_CONCURRENCY = 5; const MAX_WORKFLOWS_PER_REPOSITORY = 200; -const ADVISORY_WORKSPACE_PATTERN = /-ghsa(?:-[a-z0-9]{4}){3}$/; +const MAX_LOCAL_ACTIONS_PER_REPOSITORY = 200; function sortedByName(rows) { return [...rows].sort((left, right) => @@ -57,32 +57,22 @@ export async function auditRepository(client, repository) { try { const ref = await client.getRef( fullName, - `heads/${repository.defaultBranch}`, + `heads/${encodeURIComponent(repository.defaultBranch)}`, ); headSha = ref?.object?.sha; - if (typeof headSha !== "string" || headSha.length !== 40) { + if (typeof headSha !== "string" || !/^[0-9a-f]{40}$/.test(headSha)) { throw new Error(`unexpected head ref shape for ${fullName}`); } - tree = await client.getTree( - fullName, - encodeURIComponent(repository.defaultBranch), - true, - ); + tree = await client.getTree(fullName, headSha, true); } catch (error) { + const status = error.status ?? error.cause?.status; if ( - error.message?.includes("HTTP 404") && - ADVISORY_WORKSPACE_PATTERN.test(repository.name) + headSha === undefined && + (status === 404 || + status === 409 || + /HTTP (404|409)/.test(error.message)) && + (await isEmptyRepository(client, fullName)) ) { - return { - defaultBranch: repository.defaultBranch, - disposition: "advisory-workspace", - lockfiles: [], - name: repository.name, - visibility: repository.visibility, - workflows: [], - }; - } - if (await isEmptyRepository(client, fullName)) { return { defaultBranch: repository.defaultBranch, disposition: "empty", @@ -104,11 +94,27 @@ export async function auditRepository(client, repository) { }; } - if (tree.truncated !== false) { + if ( + tree?.truncated !== false || + !Array.isArray(tree.tree) || + tree.tree.some( + (entry) => + !entry || + typeof entry.path !== "string" || + !entry.path || + !["blob", "tree", "commit"].includes(entry.type) || + (entry.type === "blob" && + !["100644", "100755", "120000"].includes(entry.mode)) || + (entry.type === "tree" && entry.mode !== "040000") || + (entry.type === "commit" && entry.mode !== "160000"), + ) || + new Set(tree.tree.map((entry) => entry.path)).size !== tree.tree.length + ) { return { defaultBranch: repository.defaultBranch, disposition: "audit-error", - error: "default-branch tree listing is truncated", + error: + "default-branch tree listing is malformed, truncated, or contains unsupported file modes", headSha, lockfiles: [], name: repository.name, @@ -144,31 +150,61 @@ export async function auditRepository(client, repository) { } try { + const readSource = async (path) => { + if (tree.tree.find((entry) => entry.path === path)?.mode === "120000") { + throw new Error("workflow or local action source is a symlink"); + } + return client.getText(fullName, path, headSha); + }; const workflowTexts = await mapWithConcurrency( workflowPaths, AUDIT_CONCURRENCY, - (path) => client.getText(fullName, path, headSha), + (path) => readSource(path), ); + if ( + workflowTexts.some((text) => typeof text !== "string" || !text.trim()) + ) { + throw new Error("workflow source read is empty or malformed"); + } + // Fetch only referenced local actions, including nested composites. The set + // bounds cycles in fetching; the classifier separately bounds expansion. + const localActions = new Map(); + const pendingTexts = [...workflowTexts]; const localActionPaths = new Set(); - for (const text of workflowTexts) { - for (const match of text.matchAll(LOCAL_ACTION_PATTERN)) { - const base = match[1].replace(/\/+$/, ""); - for (const candidate of [`${base}/action.yml`, `${base}/action.yaml`]) { - if (blobPaths.has(candidate)) { - localActionPaths.add(candidate); + while (pendingTexts.length > 0) { + const paths = []; + for (const text of pendingTexts.splice(0)) { + for (const match of text.matchAll(LOCAL_ACTION_PATTERN)) { + const base = match[1].replace(/\/+$/, ""); + for (const candidate of [ + `${base}/action.yml`, + `${base}/action.yaml`, + ]) { + if (blobPaths.has(candidate) && !localActionPaths.has(candidate)) { + localActionPaths.add(candidate); + paths.push(candidate); + } } } } + if (localActionPaths.size > MAX_LOCAL_ACTIONS_PER_REPOSITORY) { + throw new Error( + "too many referenced local actions; refusing unbounded scan", + ); + } + await mapWithConcurrency( + paths.sort(), + AUDIT_CONCURRENCY, + async (path) => { + const text = await readSource(path); + if (typeof text !== "string" || !text.trim()) + throw new Error("local action source read is empty or malformed"); + localActions.set(path, text); + pendingTexts.push(text); + }, + ); } - const localActions = new Map(); - await mapWithConcurrency( - [...localActionPaths].sort(), - AUDIT_CONCURRENCY, - async (path) => { - localActions.set(path, await client.getText(fullName, path, headSha)); - }, - ); const workflows = workflowPaths.map((path, index) => classifyWorkflow(workflowTexts[index], { @@ -260,6 +296,10 @@ export async function runAudit(client, options = {}) { visibility: inventory.visibility, }, organization: ORGANIZATION, + scanErrors: dispositions["audit-error"] ?? 0, + scanStatus: dispositions["audit-error"] ? "partial" : "complete", + coverage: + "token-visible repositories only; organization-wide access is an operator prerequisite", repositories: rows, schemaVersion: 1, }; diff --git a/tools/rollout/classify.mjs b/tools/rollout/classify.mjs index 3a82fe9..b5f464d 100644 --- a/tools/rollout/classify.mjs +++ b/tools/rollout/classify.mjs @@ -1,4 +1,4 @@ -import { parse } from "yaml"; +import { parseYamlSource as parse } from "./yaml.mjs"; import { ACTION_REPOSITORY, APPROVED_RELEASE_SHA } from "./constants.mjs"; @@ -27,33 +27,9 @@ const NPM_DOWNLOAD = new Set([ "upgrade", "x", ]); -const NPM_NO_NETWORK = new Set([ - "build", - "cache", - "config", - "get", - "help", - "link", - "login", - "logout", - "ls", - "outdated", - "pack", - "ping", - "prune", - "rebuild", - "restart", - "run", - "run-script", - "set", - "start", - "stop", - "t", - "test", - "version", - "view", - "whoami", -]); +// Only literal informational commands are considered transparent. Scripts, +// lifecycle commands and executors can hide dependency installation. +const NPM_NO_NETWORK = new Set(["get", "help", "ls", "whoami"]); const PNPM_DOWNLOAD = new Set([ "add", "dlx", @@ -65,25 +41,9 @@ const PNPM_DOWNLOAD = new Set([ "up", "update", ]); -const PNPM_NO_NETWORK = new Set([ - "build", - "config", - "exec", - "lint", - "list", - "ls", - "pack", - "prune", - "rebuild", - "run", - "start", - "t", - "test", - "version", - "why", -]); +const PNPM_NO_NETWORK = new Set(["list", "ls", "why"]); const BUN_DOWNLOAD = new Set(["add", "i", "install", "update", "x"]); -const BUN_NO_NETWORK = new Set(["build", "run", "test"]); +const BUN_NO_NETWORK = new Set(); const YARN_DOWNLOAD = new Set(["add", "dlx", "i", "import", "install", "up"]); const OTHER_ECOSYSTEM_COMMANDS = new Set([ "apk", @@ -94,7 +54,6 @@ const OTHER_ECOSYSTEM_COMMANDS = new Set([ "cargo", "composer", "conda", - "docker", "dotnet", "gem", "go", @@ -110,7 +69,49 @@ const OTHER_ECOSYSTEM_COMMANDS = new Set([ "uv", ]); const WRAPPER_COMMANDS = new Set(["just", "make", "mise", "rake", "task"]); -const NO_NETWORK_ORCHESTRATORS = new Set(["lerna", "nx", "rush", "turbo"]); +const ORCHESTRATORS = new Set(["lerna", "nx", "rush", "turbo"]); +const SIMPLE_COMMANDS = new Set(["echo", "printf", "pwd", "true", "false"]); +const TRANSPARENT_INSTALL_FLAGS = new Set([ + "--ignore-scripts", + "--no-audit", + "--no-fund", + "--frozen-lockfile", + "--immutable", + "--offline", + "--prefer-offline", + "--no-progress", + "--no-save", + "--save-exact", + "--save-dev", + "--global", + "-g", + "-D", + "-y", +]); + +function isMapping(value) { + return value !== null && typeof value === "object" && !Array.isArray(value); +} + +function condition(value) { + if (value === undefined) return "always"; + const expression = normalizeExpression(value).replace( + /^\$\{\{(.*)\}\}$/, + "$1", + ); + if (expression === "false") return "never"; + if (expression === "true") return "always"; + return "uncertain"; +} + +function boundaryUncertain(step) { + return ( + condition(step.if) === "uncertain" || + (step["continue-on-error"] !== undefined && + condition(step["continue-on-error"]) !== "never") || + step.env !== undefined + ); +} function normalizeExpression(value) { return String(value ?? "").replaceAll(/\s+/g, ""); @@ -154,7 +155,17 @@ export function classifyCommand(command) { if (!program) { return { command, kind: "no-network" }; } - const name = program.replace(/^.*\//, ""); + if (program.includes("/")) return { command, kind: "unknown-wrapper" }; + const name = program; + + // Do not mistake an option's value (e.g. --prefix help) for the verb. + // Unsupported leading options remain review candidates, not a parsed shell. + if ( + ["npm", "pnpm", "bun", "yarn", "corepack"].includes(name) && + words[1]?.startsWith("-") + ) { + return { command, kind: "unknown-wrapper", manager: name }; + } if (name === "npx" || name === "bunx") { return { @@ -166,6 +177,8 @@ export function classifyCommand(command) { if (name === "corepack") { const subcommand = firstSubcommand(words); if (subcommand === "enable" || subcommand === "disable") { + // Targeted controls may affect Yarn without changing pnpm's shim. + if (words.length !== 2) return { command, kind: "unknown-wrapper" }; return { command, corepack: subcommand, kind: "no-network" }; } return { @@ -187,7 +200,7 @@ export function classifyCommand(command) { if (subcommand === undefined || YARN_DOWNLOAD.has(subcommand)) { return { command, kind: "yarn-blocked", manager: "yarn" }; } - return { command, kind: "no-network" }; + return { command, kind: "unknown-wrapper", manager: "yarn" }; } if (subcommand === "publish") { return { command, kind: "js-publish", manager: name }; @@ -210,23 +223,26 @@ export function classifyCommand(command) { } return { command, kind: "js-public-download", manager: name }; } - if ( - subcommand === undefined || - quiet.has(subcommand) || - (name === "bun" && /[./]/.test(subcommand)) - ) { + if (quiet.has(subcommand)) { return { command, kind: "no-network" }; } - return { command, kind: "unknown", manager: name }; + return { command, kind: "unknown-wrapper", manager: name }; } if (OTHER_ECOSYSTEM_COMMANDS.has(name)) { - return { command, kind: "other-ecosystem" }; + // Language/package executors can wrap JS installers just like shell scripts. + const executesProgram = words + .slice(1) + .some((word) => ["run", "exec", "generate", "shell"].includes(word)); + return { + command, + kind: executesProgram ? "unknown-wrapper" : "other-ecosystem", + }; } - if (NO_NETWORK_ORCHESTRATORS.has(name)) { + if (ORCHESTRATORS.has(name)) { if (name === "lerna" && words.includes("bootstrap")) { return { command, kind: "js-public-download", manager: "npm" }; } - return { command, kind: "no-network" }; + return { command, kind: "unknown-wrapper" }; } if ( WRAPPER_COMMANDS.has(name) || @@ -236,7 +252,10 @@ export function classifyCommand(command) { ) { return { command, kind: "unknown-wrapper" }; } - return { command, kind: "no-network" }; + return { + command, + kind: SIMPLE_COMMANDS.has(name) ? "no-network" : "unknown-wrapper", + }; } function parseUses(uses) { @@ -268,6 +287,16 @@ function classifyUsesStep(step, context) { const withInput = step.with ?? {}; if (uses.kind === "local") { + const stack = context.actionStack ?? []; + if (stack.includes(uses.path) || stack.length >= 20) { + return [ + { + kind: "unknown-local-action", + uses: step.uses, + reason: "cyclic or deeply nested local action", + }, + ]; + } const actionText = context.localActions?.get(`${uses.path}/action.yml`) ?? context.localActions?.get(`${uses.path}/action.yaml`); @@ -281,18 +310,25 @@ function classifyUsesStep(step, context) { return [{ kind: "unknown-local-action", uses: step.uses }]; } const steps = action?.runs?.steps; - if (action?.runs?.using !== "composite" || !Array.isArray(steps)) { - return [{ kind: "local-action", uses: step.uses }]; + if ( + action?.runs?.using !== "composite" || + !Array.isArray(steps) || + steps.length === 0 + ) { + return [{ kind: "unknown-local-action", uses: step.uses }]; } return steps.flatMap((inner) => - classifyStep(inner, context).map((operation) => ({ + classifyStep(inner, { + ...context, + actionStack: [...stack, uses.path], + }).map((operation) => ({ ...operation, via: step.uses, })), ); } if (uses.kind === "docker") { - return [{ kind: "other-ecosystem", uses: step.uses }]; + return [{ kind: "unknown", uses: step.uses }]; } if (uses.repository === ACTION_REPOSITORY) { @@ -302,6 +338,9 @@ function classifyUsesStep(step, context) { } return [ { + configureBun: normalizeExpression( + withInput["configure-bun"] ?? "false", + ), fallback: normalizeExpression( withInput["allow-external-fork-fallback"] ?? "false", ), @@ -312,7 +351,7 @@ function classifyUsesStep(step, context) { }, ]; } - if (uses.repository === "actions/setup-node") { + if (uses.repository === "actions/setup-node" && uses.subpath === "") { return [ { kind: "setup-node", @@ -321,10 +360,12 @@ function classifyUsesStep(step, context) { }, ]; } - if (uses.repository === "actions/checkout") { + if (uses.repository === "actions/checkout" && uses.subpath === "") { return [ { kind: "checkout", + uncertain: + withInput.ref !== undefined || withInput.repository !== undefined, persistCredentials: normalizeExpression( withInput["persist-credentials"] ?? "true", ), @@ -334,33 +375,80 @@ function classifyUsesStep(step, context) { }, ]; } - if (uses.repository === "pnpm/action-setup") { + if (uses.repository === "pnpm/action-setup" && uses.subpath === "") { const runInstall = withInput.run_install; if (runInstall !== undefined && String(runInstall) !== "false") { - return [{ kind: "js-public-download", manager: "pnpm", uses: step.uses }]; + return [ + { + kind: "js-public-download", + manager: "pnpm", + uses: step.uses, + uncertain: true, + }, + ]; } - return [{ kind: "toolchain", uses: step.uses }]; + return [{ kind: "unknown", uses: step.uses }]; } - if ( - uses.repository === "changesets/action" || - uses.repository === "JS-DevTools/npm-publish" - ) { - return [{ kind: "js-publish", manager: "npm", uses: step.uses }]; - } - return [{ kind: "remote-action", uses: step.uses }]; + return [{ kind: "unknown", uses: step.uses }]; } -export function classifyStep(step, context) { - if (step === null || typeof step !== "object") { - return [{ kind: "unknown" }]; - } - if (step.uses !== undefined) { - return classifyUsesStep(step, context); +export function classifyStep(step, context = {}) { + if (context.stepBudget && --context.stepBudget.remaining < 0) { + return [ + { + kind: "unknown-local-action", + reason: "local action expansion limit reached", + }, + ]; } - if (step.run !== undefined) { - return splitCommands(step.run).map((command) => classifyCommand(command)); + if (!isMapping(step)) return [{ kind: "unknown" }]; + if (condition(step.if) === "never") return []; + let operations; + let uncertain = boundaryUncertain(step); + if ( + step.uses !== undefined && + step.run === undefined && + typeof step.uses === "string" && + (step.with === undefined || isMapping(step.with)) + ) { + operations = classifyUsesStep(step, context); + } else if (typeof step.run === "string" && step.uses === undefined) { + // Deliberately not a shell interpreter: retain recognizable downloads but + // never certify control flow, substitutions, redirections or custom shells. + uncertain ||= + /[;'"$`|<>(){}\\]|(?:^|\s)(?:if|then|else|fi|for|while|case|eval|source|sudo)(?:\s|$)|(? { + const operation = classifyCommand(command); + if ( + /--(?:[^\s=]*:)?reg[a-z]*\b|--[^\s=]*(?:registry|userconfig)|(?:NPM|PNPM|BUN)_CONFIG_|npm_config_|\bnpm\s+config\s+(?:set|delete)\b/i.test( + command, + ) + ) { + operation.registryMutating = true; + } else if ( + operation.kind === "js-public-download" && + operation.corepack === undefined && + commandWords(command).some( + (word) => + word.startsWith("-") && !TRANSPARENT_INSTALL_FLAGS.has(word), + ) + ) { + // Package managers accept abbreviated/configuration flags. Rather than + // emulate each parser, keep unrecognized options as review candidates. + operation.uncertain = true; + } + return operation; + }); + } else { + operations = [{ kind: "unknown" }]; } - return [{ kind: "no-network" }]; + return operations.map((operation) => ({ + ...operation, + ...(uncertain || operation.uncertain ? { uncertain: true } : {}), + })); } function normalizeTriggers(workflow) { @@ -380,27 +468,68 @@ function normalizeTriggers(workflow) { function collectViolations(operations, context, triggers, job) { const violations = []; const downloads = []; - let lastRegistryMutation = -1; const setups = []; - const teardowns = []; const publishes = []; const expectedToken = expectedTokenExpression(context.visibility); + let activeSetup; + let registryInvalidated = false; + let publishBoundary = true; + let corepackEnabled = false; operations.forEach((operation, index) => { + if (operation.corepack === "enable") corepackEnabled = true; + if (operation.corepack === "disable") corepackEnabled = false; if (operation.kind === "js-public-download") { downloads.push(index); + if (registryInvalidated || operation.registryMutating) { + violations.push( + `registry-mutating configuration precedes download operation ${index + 1}`, + ); + } + if (operation.manager === "pnpm" && corepackEnabled) { + violations.push( + `Corepack may lazily download pnpm directly from registry.npmjs.org at operation ${index + 1}; install pinned pnpm with npm through Socket Firewall instead`, + ); + } + if ( + activeSetup === undefined || + operation.uncertain || + operation.registryMutating + ) { + violations.push( + `download operation ${index + 1} has no certain active Socket Firewall setup`, + ); + } + if (operation.manager === "bun" && activeSetup?.configureBun !== "true") { + violations.push( + `Bun download operation ${index + 1} requires configure-bun: true`, + ); + } + publishBoundary = false; } - if ( - operation.kind === "setup-node" && - operation.registryMutating === true - ) { - lastRegistryMutation = index; + if (operation.registryMutating === true) { + registryInvalidated = true; + activeSetup = undefined; } if (operation.kind === "js-publish") { publishes.push(index); + if (setups.length > 0 && !publishBoundary) { + violations.push( + "publish follows Socket Firewall setup without a same-SHA teardown boundary", + ); + } } if (operation.kind === "sfw-setup") { + registryInvalidated = false; setups.push(index); + publishBoundary = false; + activeSetup = + operation.ref === APPROVED_RELEASE_SHA && + operation.token === expectedToken && + !operation.uncertain && + operation.fallback === "false" + ? operation + : undefined; if (operation.ref !== APPROVED_RELEASE_SHA) { violations.push( FULL_SHA_PATTERN.test(operation.ref) @@ -420,7 +549,10 @@ function collectViolations(operations, context, triggers, job) { } } if (operation.kind === "sfw-teardown") { - teardowns.push(index); + registryInvalidated = false; + activeSetup = undefined; + publishBoundary = + operation.ref === APPROVED_RELEASE_SHA && !operation.uncertain; if (operation.ref !== APPROVED_RELEASE_SHA) { violations.push( `sfw-teardown ref "${operation.ref}" does not match the approved release SHA`, @@ -429,18 +561,6 @@ function collectViolations(operations, context, triggers, job) { } }); - const corepackEnable = operations.findIndex( - (operation) => operation.corepack === "enable", - ); - const firstPnpmDownload = operations.findIndex( - (operation) => - operation.kind === "js-public-download" && operation.manager === "pnpm", - ); - if (corepackEnable !== -1 && firstPnpmDownload > corepackEnable) { - violations.push( - "Corepack may lazily download pnpm directly from registry.npmjs.org; install the pinned pnpm package with npm through Socket Firewall before invoking pnpm", - ); - } if ( operations.some( (operation) => @@ -454,40 +574,8 @@ function collectViolations(operations, context, triggers, job) { ); } - const firstDownload = downloads[0]; - const validSetup = setups.find( - (index) => - operations[index].ref === APPROVED_RELEASE_SHA && - operations[index].token === expectedToken, - ); - if (firstDownload !== undefined && setups.length > 0) { - if (validSetup === undefined) { - violations.push("no valid Socket Firewall setup precedes downloads"); - } else if (validSetup > firstDownload) { - violations.push("Socket Firewall setup runs after the first download"); - } else if ( - lastRegistryMutation > validSetup && - lastRegistryMutation < firstDownload - ) { - violations.push( - "registry-mutating setup-node runs between Socket Firewall setup and the first download", - ); - } - } - - if (publishes.length > 0 && setups.length > 0) { - const lastDownload = downloads.at(-1) ?? -1; - const validTeardown = teardowns.find( - (index) => - operations[index].ref === APPROVED_RELEASE_SHA && - index > lastDownload && - index < publishes[0], - ); - if (validTeardown === undefined) { - violations.push( - "publish follows Socket Firewall setup without a same-SHA teardown boundary", - ); - } + if (downloads.length > 0 && setups.length > 0 && setups[0] > downloads[0]) { + violations.push("Socket Firewall setup runs after the first download"); } if (context.visibility === "public") { @@ -523,8 +611,8 @@ function collectViolations(operations, context, triggers, job) { return violations; } -export function classifyJob(jobName, job, context, triggers) { - if (job === null || typeof job !== "object") { +export function classifyJob(jobName, job, context = {}, triggers = []) { + if (!isMapping(job)) { return { job: jobName, managers: [], @@ -549,7 +637,31 @@ export function classifyJob(jobName, job, context, triggers) { } const steps = Array.isArray(job.steps) ? job.steps : []; - const operations = steps.flatMap((step) => classifyStep(step, context)); + const stepContext = { ...context, stepBudget: { remaining: 1000 } }; + const operations = + condition(job.if) === "never" + ? [] + : steps.flatMap((step, index) => + classifyStep(step, stepContext).map((operation) => ({ + ...operation, + step: index + 1, + })), + ); + if ( + !Array.isArray(job.steps) || + steps.length === 0 || + job.container !== undefined || + job.services !== undefined || + boundaryUncertain(job) || + context.workflowUncertain || + (job.defaults?.run?.shell !== undefined && + !["bash", "sh"].includes(job.defaults.run.shell)) + ) { + operations.push({ + kind: "unknown", + reason: "opaque job or workflow execution context", + }); + } const violations = collectViolations(operations, context, triggers, job); const kinds = new Set(operations.map((operation) => operation.kind)); @@ -568,7 +680,12 @@ export function classifyJob(jobName, job, context, triggers) { const hasUnknown = kinds.has("unknown") || kinds.has("unknown-wrapper") || - kinds.has("unknown-local-action"); + kinds.has("unknown-local-action") || + operations.some( + (operation) => + operation.uncertain || + (operation.kind === "sfw-setup" && operation.fallback !== "false"), + ); const publishUnsafe = violations.some((violation) => violation.includes("without a same-SHA teardown"), ); @@ -583,14 +700,18 @@ export function classifyJob(jobName, job, context, triggers) { status = "unsafe-trust"; } else if (publishUnsafe) { status = "unsafe-publish"; + } else if (hasUnknown) { + // A conditional setup, opaque wrapper, or supported fork fallback is not + // proof of a missing integration. Retain violations for private review. + status = "unknown"; } else if (kinds.has("yarn-blocked")) { status = "blocked-yarn"; - } else if (hasDownload && hasSetup && violations.length === 0) { + } else if (hasDownload && violations.length > 0) { + status = "unprotected"; + } else if (hasDownload && hasSetup) { status = "protected"; } else if (hasDownload) { status = "unprotected"; - } else if (hasUnknown) { - status = "unknown"; } else if (hasPublish) { status = hasSetup ? "unsafe-publish" : "safe-publish"; } else if (kinds.has("other-ecosystem")) { @@ -615,10 +736,14 @@ export function classifyWorkflow(text, context) { triggers: [], }; } - if (workflow === null || typeof workflow !== "object") { + if ( + !isMapping(workflow) || + !isMapping(workflow.jobs) || + Object.keys(workflow.jobs).length === 0 + ) { return { jobs: [], - parseError: "workflow is not a mapping", + parseError: "workflow or jobs is not a nonempty mapping", path: context.path, status: "unknown", triggers: [], @@ -626,11 +751,29 @@ export function classifyWorkflow(text, context) { } const triggers = normalizeTriggers(workflow); + if (triggers.length === 0) { + return { + jobs: [], + path: context.path, + triggers, + status: "unknown", + parseError: "workflow has no recognized trigger declaration", + }; + } const jobEntries = Object.entries(workflow.jobs ?? {}).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0, ); const jobs = jobEntries.map(([jobName, job]) => - classifyJob(jobName, job, context, triggers), + classifyJob( + jobName, + job, + { + ...context, + workflowUncertain: + workflow.env !== undefined || workflow.defaults !== undefined, + }, + triggers, + ), ); return { jobs, path: context.path, triggers }; diff --git a/tools/rollout/classify.test.mjs b/tools/rollout/classify.test.mjs index c442d09..3018a2c 100644 --- a/tools/rollout/classify.test.mjs +++ b/tools/rollout/classify.test.mjs @@ -1,5 +1,5 @@ import assert from "node:assert/strict"; -import { mkdtemp, readFile, readdir } from "node:fs/promises"; +import { mkdtemp, readFile, readdir, rm, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; import test from "node:test"; @@ -7,9 +7,12 @@ import test from "node:test"; import { auditRepository, runAudit, writeReportAtomically } from "./audit.mjs"; import { classifyCommand, + classifyJob, classifyWorkflow, repositoryDisposition, } from "./classify.mjs"; +import { main, scanExitCode } from "./cli.mjs"; +import { GitHubClient } from "./github.mjs"; import { APPROVED_RELEASE_SHA } from "./constants.mjs"; const SETUP = `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`; @@ -36,30 +39,30 @@ test("command grammar", () => { ["npm audit", "no-network"], ["npm audit fix", "js-public-download"], ["npm publish --access public", "js-publish"], - ["npm run build", "no-network"], - ["npm test", "no-network"], + ["npm run build", "unknown-wrapper"], + ["npm test", "unknown-wrapper"], ["npx changeset publish", "js-public-download"], ["pnpm install --frozen-lockfile", "js-public-download"], ["pnpm dlx create-thing", "js-public-download"], - ["pnpm exec vitest run", "no-network"], + ["pnpm exec vitest run", "unknown-wrapper"], ["pnpm publish --no-git-checks", "js-publish"], ["bun install --frozen-lockfile", "js-public-download"], ["bunx biome check", "js-public-download"], - ["bun run test", "no-network"], + ["bun run test", "unknown-wrapper"], ["bun publish", "js-publish"], ["yarn install --frozen-lockfile", "yarn-blocked"], ["yarn", "yarn-blocked"], ["yarn npm publish", "js-publish"], - ["yarn build", "no-network"], + ["yarn build", "unknown-wrapper"], ["corepack enable", "no-network"], ["corepack prepare pnpm@10 --activate", "js-public-download"], ["pip install -r requirements.txt", "other-ecosystem"], ["go build ./...", "other-ecosystem"], - ["docker build .", "other-ecosystem"], + ["docker build .", "unknown-wrapper"], ["make bootstrap", "unknown-wrapper"], ["./scripts/setup.sh", "unknown-wrapper"], ["bash scripts/install.sh", "unknown-wrapper"], - ["turbo run lint", "no-network"], + ["turbo run lint", "unknown-wrapper"], ["lerna bootstrap", "js-public-download"], ["CI=1 npm ci", "js-public-download"], ["echo done", "no-network"], @@ -117,6 +120,7 @@ test("classifier: mutable or mismatched refs are violations", () => { ` build:\n steps:\n - uses: workos/setup-socket-firewall@v1\n with: { token: "${PRIVATE_TOKEN}" }\n - run: npm ci\n`, ), ); + assert.equal(mutable.jobs[0].status, "unprotected"); assert.match(mutable.jobs[0].violations.join(" "), /mutable or short ref/); const stale = classify( @@ -124,6 +128,7 @@ test("classifier: mutable or mismatched refs are violations", () => { ` build:\n steps:\n - uses: workos/setup-socket-firewall@${"a".repeat(40)}\n with: { token: "${PRIVATE_TOKEN}" }\n - run: npm ci\n`, ), ); + assert.equal(stale.jobs[0].status, "unprotected"); assert.match(stale.jobs[0].violations.join(" "), /unapproved SHA/); }); @@ -134,6 +139,7 @@ test("classifier: token must match repository visibility", () => { ), "public", ); + assert.equal(publicWrongSecret.jobs[0].status, "unprotected"); assert.match( publicWrongSecret.jobs[0].violations.join(" "), /PUBLIC_SOCKET_FIREWALL_TOKEN.*public/, @@ -223,7 +229,8 @@ test("classifier: pnpm action-setup run_install downloads", () => { ` build:\n steps:\n - uses: pnpm/action-setup@sha\n with: { run_install: true }\n`, ), ); - assert.equal(result.jobs[0].status, "unprotected"); + assert.equal(result.jobs[0].status, "unknown"); + assert.equal(repositoryDisposition([result]), "needs-review"); }); test("classifier: Corepack pnpm bootstrap bypasses SFW npm configuration", () => { @@ -329,8 +336,8 @@ function stubClient(overrides = {}) { }, getTree: async () => ({ tree: [ - { path: ".github/workflows/ci.yml", type: "blob" }, - { path: "package-lock.json", type: "blob" }, + { path: ".github/workflows/ci.yml", type: "blob", mode: "100644" }, + { path: "package-lock.json", type: "blob", mode: "100644" }, ], truncated: false, }), @@ -345,7 +352,7 @@ function stubClient(overrides = {}) { }; } -test("audit: classifies live repositories and writes an atomic report", async () => { +test("audit: classifies mocked repositories and writes an atomic report", async (t) => { const report = await runAudit(stubClient()); assert.equal(report.inventory.activeCount, 2); assert.deepEqual(report.dispositions, { empty: 1, "needs-sfw": 1 }); @@ -355,8 +362,10 @@ test("audit: classifies live repositories and writes an atomic report", async () assert.deepEqual(app.lockfiles, ["package-lock.json"]); const directory = await mkdtemp(join(tmpdir(), "sfw-audit-")); + t.after(() => rm(directory, { recursive: true, force: true })); const reportPath = join(directory, "report.json"); await writeReportAtomically(reportPath, report); + assert.equal((await stat(reportPath)).mode & 0o777, 0o600); const written = JSON.parse(await readFile(reportPath, "utf8")); assert.deepEqual(written.dispositions, report.dispositions); assert.deepEqual(await readdir(directory), ["report.json"]); @@ -424,3 +433,515 @@ test("adapter: non-transient failures do not retry", async () => { await assert.rejects(() => client.api("repos/workos/gone", "read gone")); assert.equal(attempts, 1); }); + +const setupStep = { uses: SETUP, with: { token: PRIVATE_TOKEN } }; +const installStep = { run: "npm ci" }; +const teardownStep = { uses: TEARDOWN }; +function jobResult(steps, extra = {}, context = {}) { + return classifyJob( + "fixture", + { steps, ...extra }, + { visibility: "private", ...context }, + ["push"], + ); +} + +test("classifier: every download needs an active, unconditional setup interval", () => { + const cases = [ + ["disabled", [{ ...setupStep, if: false }, installStep]], + [ + "disabled expression", + [{ ...setupStep, if: "${{ false }}" }, installStep], + ], + [ + "conditional", + [{ ...setupStep, if: "github.ref == 'refs/heads/main'" }, installStep], + ], + [ + "continue on error", + [{ ...setupStep, "continue-on-error": true }, installStep], + ], + [ + "dynamic continue on error", + [ + { ...setupStep, "continue-on-error": "${{ matrix.optional }}" }, + installStep, + ], + ], + ["teardown before install", [setupStep, teardownStep, installStep]], + [ + "teardown after first install", + [setupStep, installStep, teardownStep, installStep], + ], + [ + "conditional teardown", + [setupStep, { ...teardownStep, if: "always()" }, installStep], + ], + [ + "later registry action", + [ + setupStep, + installStep, + { + uses: "actions/setup-node@fixture", + with: { "registry-url": "https://registry.npmjs.org/" }, + }, + installStep, + ], + ], + [ + "later registry flag", + [ + setupStep, + installStep, + { run: "npm ci --registry=https://registry.npmjs.org/" }, + ], + ], + [ + "later registry assignment", + [ + setupStep, + installStep, + { run: "NPM_CONFIG_REGISTRY=https://registry.npmjs.org/ npm ci" }, + ], + ], + ["install condition", [setupStep, { ...installStep, if: "always()" }]], + [ + "install continue on error", + [setupStep, { ...installStep, "continue-on-error": true }], + ], + [ + "fallback path", + [ + { + ...setupStep, + with: { ...setupStep.with, "allow-external-fork-fallback": "true" }, + }, + installStep, + ], + ], + ["bun configuration missing", [setupStep, { run: "bun install" }]], + ]; + const ambiguous = new Set([ + "conditional", + "continue on error", + "dynamic continue on error", + "conditional teardown", + "install condition", + "install continue on error", + "fallback path", + ]); + for (const [label, steps] of cases) { + const result = jobResult(steps); + assert.equal( + result.status, + ambiguous.has(label) ? "unknown" : "unprotected", + label, + ); + assert.equal( + repositoryDisposition([{ jobs: [result] }]), + ambiguous.has(label) ? "needs-review" : "needs-sfw", + label, + ); + assert.ok(result.violations.length > 0, label); + } + assert.equal( + jobResult([setupStep, installStep, teardownStep]).status, + "protected", + ); + assert.equal( + jobResult([setupStep, installStep, teardownStep, setupStep, installStep]) + .status, + "protected", + ); + assert.equal( + jobResult([ + { ...setupStep, if: true, "continue-on-error": false }, + installStep, + ]).status, + "protected", + ); + assert.equal( + jobResult([ + { ...setupStep, with: { ...setupStep.with, "configure-bun": "true" } }, + { run: "bun install" }, + ]).status, + "protected", + ); +}); + +test("classifier: opaque operations cannot lose to an otherwise protected install", () => { + const opaque = [ + { run: "npm run bootstrap" }, + { run: "npm test" }, + { run: "pnpm exec custom" }, + { run: "bash -c 'npm ci'" }, + { run: "node scripts/bootstrap.js" }, + { run: "./scripts/bootstrap" }, + { run: "docker build ." }, + { run: "custom-tool" }, + { uses: "docker://fixture/image:latest" }, + { uses: "example/opaque-action@fixture" }, + { uses: "./missing" }, + { run: "npm config set registry https://registry.npmjs.org/" }, + ]; + for (const step of opaque) { + assert.equal(jobResult([step]).status, "unknown", JSON.stringify(step)); + const result = jobResult([setupStep, installStep, step]); + assert.notEqual(result.status, "protected", JSON.stringify(step)); + assert.ok(["unknown", "unprotected"].includes(result.status)); + } + for (const extra of [ + { container: "node:22" }, + { if: "always()" }, + { "continue-on-error": true }, + { env: { NPM_CONFIG_REGISTRY: "custom" } }, + { defaults: { run: { shell: "python" } } }, + ]) { + assert.equal(jobResult([setupStep, installStep], extra).status, "unknown"); + } +}); + +test("classifier: small shell grammar keeps complex constructs uncertain", () => { + for (const run of [ + "npm ci || true", + "npm ci; npm ci", + "if true; then npm ci; fi", + "npm ci $(custom)", + "npm ci > log", + "sudo npm ci", + 'npm ci "$(custom)"', + ]) { + assert.notEqual(jobResult([setupStep, { run }]).status, "protected", run); + } + assert.equal( + jobResult([setupStep, { run: "npm ci && pnpm install" }]).status, + "protected", + ); + assert.equal( + jobResult([setupStep, { run: "npm ci", shell: "python" }]).status, + "unknown", + ); +}); + +test("classifier: local composite nesting, cycles and inner/outer uncertainty", () => { + const localActions = new Map([ + [ + "outer/action.yml", + "runs:\n using: composite\n steps:\n - uses: ./inner\n", + ], + [ + "inner/action.yml", + `runs:\n using: composite\n steps:\n - uses: ${SETUP}\n with: { token: '${PRIVATE_TOKEN}' }\n - run: npm ci\n shell: bash\n`, + ], + ]); + const call = { uses: "./outer" }; + assert.equal(jobResult([call], {}, { localActions }).status, "protected"); + for (const boundary of [{ if: "always()" }, { "continue-on-error": true }]) { + assert.equal( + jobResult([{ ...call, ...boundary }], {}, { localActions }).status, + "unknown", + ); + } + localActions.set( + "inner/action.yml", + `runs:\n using: composite\n steps:\n - uses: ${SETUP}\n if: false\n with: { token: '${PRIVATE_TOKEN}' }\n - run: npm ci\n shell: bash\n`, + ); + assert.equal(jobResult([call], {}, { localActions }).status, "unprotected"); + localActions.set( + "inner/action.yml", + "runs:\n using: composite\n steps:\n - uses: ./outer\n", + ); + assert.equal(jobResult([call], {}, { localActions }).status, "unknown"); + localActions.set( + "inner/action.yml", + "runs: { using: node24, main: index.js }", + ); + assert.equal( + jobResult([setupStep, installStep, call], {}, { localActions }).status, + "unknown", + ); +}); + +test("classifier: malformed workflow/jobs and unknown operations remain review candidates", () => { + for (const text of [ + "{}", + "jobs: []", + "jobs: {}", + "jobs: wrong", + "on: push\njobs: { build: [] }", + "on: push\njobs: { build: {} }", + "on: push\njobs: { build: { steps: wrong } }", + "on: push\njobs: { build: { steps: [null] } }", + ]) { + assert.equal(repositoryDisposition([classify(text)]), "needs-review", text); + } + const result = classify( + workflow(" build:\n steps:\n - run: npm run bootstrap\n"), + ); + assert.equal(repositoryDisposition([result]), "needs-review"); +}); + +const fixtureRepository = { + name: "app", + defaultBranch: "main", + visibility: "private", +}; +const blob = (path) => ({ path, type: "blob", mode: "100644" }); + +test("audit: moving branch cannot change tree or nested source snapshot", async () => { + const sha = "d".repeat(40); + const reads = []; + const sources = new Map([ + [ + ".github/workflows/ci.yml", + workflow(" build:\n steps:\n - uses: ./outer\n"), + ], + [ + "outer/action.yml", + "runs:\n using: composite\n steps:\n - uses: ./inner\n", + ], + [ + "inner/action.yml", + "runs:\n using: composite\n steps:\n - run: npm ci\n shell: bash\n - uses: ./outer\n", + ], + ]); + const row = await auditRepository( + stubClient({ + getRef: async () => ({ object: { sha } }), + getTree: async (_repo, ref, recursive) => { + assert.equal(ref, sha); // The simulated branch now points elsewhere. + assert.equal(recursive, true); + return { truncated: false, tree: [...sources.keys()].map(blob) }; + }, + getText: async (_repo, path, ref) => { + assert.equal(ref, sha); + reads.push(path); + return sources.get(path); + }, + }), + fixtureRepository, + ); + assert.equal(row.headSha, sha); + assert.equal(row.disposition, "needs-review"); + assert.deepEqual(reads.sort(), [...sources.keys()].sort()); + assert.ok( + row.workflows[0].jobs[0].operations.some( + (op) => op.kind === "unknown-local-action", + ), + ); +}); + +test("audit: malformed and partial reads never become clean", async () => { + const overrides = [ + { getRef: async () => ({ object: { sha: "z".repeat(40) } }) }, + ...[ + null, + {}, + { truncated: false }, + { truncated: false, tree: [null] }, + { truncated: false, tree: [{ type: "blob" }] }, + ].map((tree) => ({ getTree: async () => tree })), + ...[undefined, "", {}].map((text) => ({ getText: async () => text })), + { + getText: async () => { + throw new Error("fixture source read failed"); + }, + }, + { + getTree: async () => ({ + truncated: false, + tree: [{ ...blob(".github/workflows/ci.yml"), mode: "120000" }], + }), + }, + ]; + for (const override of overrides) { + assert.equal( + (await auditRepository(stubClient(override), fixtureRepository)) + .disposition, + "audit-error", + ); + } +}); + +test("adapter: reject partial/malformed base64, wrong sizes and invalid UTF-8", async () => { + const valid = { + type: "file", + encoding: "base64", + content: "bmFtZTogZml4dHVyZQo=", + size: 14, + }; + const responses = [ + null, + { ...valid, content: undefined }, + { ...valid, content: "%%%" }, + { ...valid, size: 15 }, + { ...valid, size: undefined }, + { ...valid, content: "/w==", size: 1 }, + ]; + for (const response of responses) { + const client = new GitHubClient({ + execute: async () => ({ stdout: JSON.stringify(response) }), + }); + await assert.rejects( + client.getText("example/fixture", "action.yml", "a".repeat(40)), + ); + } + const client = new GitHubClient({ + execute: async () => ({ stdout: JSON.stringify(valid) }), + }); + assert.equal( + await client.getText("example/fixture", "action.yml", "a".repeat(40)), + "name: fixture\n", + ); +}); + +test("CLI: partial scan errors are distinct from gaps and terminal output is sanitized", async (t) => { + const directory = await mkdtemp(join(tmpdir(), "sfw-cli-")); + t.after(() => rm(directory, { recursive: true, force: true })); + let output = ""; + const reportPath = join(directory, "report.json"); + const report = await main(["audit"], { + client: stubClient({ + getText: async () => { + throw new Error("private fixture detail"); + }, + }), + reportPath, + output: { + write: (text) => { + output += text; + }, + }, + }); + assert.equal(report.scanStatus, "partial"); + assert.equal(report.scanErrors, 1); + assert.equal(scanExitCode(report), 1); + const summary = JSON.parse(output); + assert.equal(summary.scanErrors, 1); + assert.deepEqual(summary.dispositions, { "audit-error": 1, empty: 1 }); + assert.doesNotMatch(output, /private fixture detail|empty-repo|"app"/); + assert.match(await readFile(reportPath, "utf8"), /private fixture detail/); + assert.equal((await stat(reportPath)).mode & 0o777, 0o600); + const gaps = await runAudit(stubClient()); + assert.equal(gaps.scanStatus, "complete"); + assert.equal(scanExitCode(gaps), 0); + const unknowns = await runAudit( + stubClient({ + getText: async () => + workflow(" build:\n steps:\n - run: npm run bootstrap\n"), + }), + ); + assert.equal(unknowns.dispositions["needs-review"], 1); + assert.equal(scanExitCode(unknowns), 0); +}); + +test("classifier: later Corepack activation and alternate checkout are not certified", () => { + const later = jobResult([ + setupStep, + { run: "pnpm install" }, + { run: "corepack enable" }, + { run: "pnpm install" }, + ]); + assert.equal(later.status, "unprotected"); + assert.match(later.violations.join(" "), /Corepack.*operation 4/); + for (const step of [ + { uses: "actions/setup-node/custom@fixture" }, + { uses: "actions/checkout@fixture", with: { ref: "other-branch" } }, + { + uses: "actions/checkout@fixture", + with: { repository: "example/alternate" }, + }, + ]) { + assert.equal(jobResult([step, setupStep, installStep]).status, "unknown"); + } +}); + +test("classifier: branching local composites have a bounded expansion", () => { + const localActions = new Map(); + for (let index = 0; index < 15; index += 1) { + localActions.set( + `action-${index}/action.yml`, + `runs:\n using: composite\n steps:\n - uses: ./action-${index + 1}\n - uses: ./action-${index + 1}\n`, + ); + } + const result = jobResult([{ uses: "./action-0" }], {}, { localActions }); + assert.equal(result.status, "unknown"); + assert.ok(result.operations.length < 1100); + assert.ok( + result.operations.some( + (op) => op.reason === "local action expansion limit reached", + ), + ); +}); + +test("adapter: source paths are escaped independently of immutable ref", async () => { + let endpoint; + const client = new GitHubClient({ + execute: async (args) => { + endpoint = args.at(-1); + return { + stdout: JSON.stringify({ + type: "file", + encoding: "base64", + content: "", + size: 0, + }), + }; + }, + }); + await client.getText( + "example/fixture", + ".github/workflows/ci#fixture.yml", + "e".repeat(40), + ); + assert.equal( + endpoint, + `repos/example/fixture/contents/.github/workflows/ci%23fixture.yml?ref=${"e".repeat(40)}`, + ); +}); + +test("classifier: composite inner boundaries survive expansion", () => { + for (const boundary of ["if: always()", "continue-on-error: true"]) { + const localActions = new Map([ + [ + "inner/action.yml", + `runs:\n using: composite\n steps:\n - uses: ${SETUP}\n ${boundary}\n with: { token: '${PRIVATE_TOKEN}' }\n - run: npm ci\n shell: bash\n`, + ], + ]); + const result = jobResult([{ uses: "./inner" }], {}, { localActions }); + assert.equal(result.status, "unknown", boundary); + assert.equal(result.operations[0].uncertain, true); + } + for (const boundary of [{ if: "always()" }, { "continue-on-error": true }]) { + assert.equal( + jobResult([ + setupStep, + installStep, + { ...teardownStep, ...boundary }, + { run: "npm publish" }, + ]).status, + "unsafe-publish", + ); + } +}); + +test("classifier: registry changes are evaluated at each download, not after the last one", () => { + const registryStep = { + uses: "actions/setup-node@fixture", + with: { "registry-url": "https://registry.npmjs.org/" }, + }; + assert.equal( + jobResult([setupStep, installStep, registryStep]).status, + "protected", + ); + assert.equal( + jobResult([setupStep, installStep, registryStep, installStep]).status, + "unprotected", + ); + assert.equal( + jobResult([setupStep, installStep, registryStep, setupStep, installStep]) + .status, + "protected", + ); +}); diff --git a/tools/rollout/cli.mjs b/tools/rollout/cli.mjs index 8b89805..07fcc9e 100644 --- a/tools/rollout/cli.mjs +++ b/tools/rollout/cli.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node -import { pathToFileURL } from "node:url"; +import { fileURLToPath, pathToFileURL } from "node:url"; import { GitHubClient } from "./github.mjs"; import { captureRepositoryInventory } from "./inventory.mjs"; @@ -10,6 +10,16 @@ import { verifyActionRelease } from "./release.mjs"; const COMMANDS = new Set(["audit", "inventory", "verify-action"]); const REPORT_URL = new URL("../../reports/live-audit.json", import.meta.url); +const INVENTORY_URL = new URL("../../reports/inventory.json", import.meta.url); + +function inventoryCounts(inventory) { + const { activeCount, archivedCount, totalCount, visibility } = inventory; + return { activeCount, archivedCount, totalCount, visibility }; +} + +export function scanExitCode(result) { + return result.scanErrors > 0 ? 1 : 0; +} export async function main(argv, options = {}) { if (argv.length !== 1 || !COMMANDS.has(argv[0])) { @@ -21,7 +31,7 @@ export async function main(argv, options = {}) { const command = argv[0]; if (command === "audit") { - const reportPath = options.reportPath ?? REPORT_URL.pathname; + const reportPath = options.reportPath ?? fileURLToPath(REPORT_URL); const report = await runAudit(client, { progress: (done, total) => { if (done % 25 === 0 || done === total) { @@ -32,7 +42,10 @@ export async function main(argv, options = {}) { await writeReportAtomically(reportPath, report); const summary = { dispositions: report.dispositions, - inventory: report.inventory, + inventory: inventoryCounts(report.inventory), + scanErrors: report.scanErrors, + scanStatus: report.scanStatus, + coverage: report.coverage, reportPath, }; output.write(`${JSON.stringify(summary, null, 2)}\n`); @@ -43,7 +56,15 @@ export async function main(argv, options = {}) { command === "verify-action" ? await verifyActionRelease({ client }) : await captureRepositoryInventory(client, ORGANIZATION); - output.write(`${JSON.stringify(result, null, 2)}\n`); + if (command === "inventory") { + const reportPath = options.reportPath ?? fileURLToPath(INVENTORY_URL); + await writeReportAtomically(reportPath, result); + output.write( + `${JSON.stringify({ ...inventoryCounts(result), scanStatus: "complete", coverage: "token-visible repositories only", reportPath }, null, 2)}\n`, + ); + } else { + output.write(`${JSON.stringify(result, null, 2)}\n`); + } return result; } @@ -51,8 +72,15 @@ const invokedPath = process.argv[1] ? pathToFileURL(process.argv[1]).href : undefined; if (import.meta.url === invokedPath) { - main(process.argv.slice(2)).catch((error) => { - process.stderr.write(`rollout verifier failed: ${error.message}\n`); - process.exitCode = 1; - }); + main(process.argv.slice(2)) + .then((result) => { + process.exitCode = scanExitCode(result); + }) + .catch(() => { + // API errors can contain private repository names or workflow source. + process.stderr.write( + `${JSON.stringify({ scanStatus: "failed", scanErrors: 1, error: "verifier failed; check command, access, API availability, and snapshot prerequisites privately" })}\n`, + ); + process.exitCode = 1; + }); } diff --git a/tools/rollout/fixtures/release-manifest.txt b/tools/rollout/fixtures/release-manifest.txt new file mode 100644 index 0000000..5cebab8 --- /dev/null +++ b/tools/rollout/fixtures/release-manifest.txt @@ -0,0 +1,5 @@ +LICENSE +action.yml +scripts/configure.sh +scripts/teardown.sh +teardown/action.yml diff --git a/tools/rollout/github.mjs b/tools/rollout/github.mjs index f31297e..67bc158 100644 --- a/tools/rollout/github.mjs +++ b/tools/rollout/github.mjs @@ -8,8 +8,8 @@ const DEFAULT_PAGE_SIZE = 100; const MAX_REPOSITORY_COUNT = 10_000; const MAX_REPOSITORY_PAGES = MAX_REPOSITORY_COUNT / DEFAULT_PAGE_SIZE; -function statusFromText(text) { - const match = String(text).match(/\(HTTP ([0-9]{3})\)/); +export function statusFromText(text) { + const match = String(text).match(/\bHTTP ([0-9]{3})(?:\)|:)/); return match ? Number(match[1]) : undefined; } @@ -139,18 +139,37 @@ export class GitHubClient { async getText(repository, path, ref) { const response = await this.api( - `repos/${repository}/contents/${path}?ref=${encodeURIComponent(ref)}`, + `repos/${repository}/contents/${path.split("/").map(encodeURIComponent).join("/")}?ref=${encodeURIComponent(ref)}`, `read ${repository}/${path} at ${ref}`, ); - if (response.type !== "file" || response.encoding !== "base64") { + const content = + typeof response?.content === "string" + ? response.content.replaceAll("\n", "") + : undefined; + if ( + response?.type !== "file" || + response.encoding !== "base64" || + content === undefined || + !Number.isSafeInteger(response.size) || + response.size < 0 || + !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test( + content, + ) + ) { throw new Error( - `${repository}/${path} at ${ref} is not a base64 GitHub file response`, + `${repository}/${path} at ${ref} is not a complete base64 GitHub file response`, ); } - return Buffer.from( - response.content.replaceAll("\n", ""), - "base64", - ).toString("utf8"); + const bytes = Buffer.from(content, "base64"); + if ( + bytes.length !== response.size || + bytes.toString("base64") !== content + ) { + throw new Error( + `${repository}/${path} at ${ref} has incomplete or malformed content`, + ); + } + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); } async listRestRepositories(organization) { diff --git a/tools/rollout/regressions.test.mjs b/tools/rollout/regressions.test.mjs new file mode 100644 index 0000000..7b198de --- /dev/null +++ b/tools/rollout/regressions.test.mjs @@ -0,0 +1,217 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { auditRepository, runAudit } from "./audit.mjs"; +import { + classifyJob, + classifyWorkflow, + repositoryDisposition, +} from "./classify.mjs"; +import { scanExitCode } from "./cli.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +import { parseYamlSource } from "./yaml.mjs"; +import { statusFromText } from "./github.mjs"; + +test("GitHub status parsing handles both CLI diagnostic formats", () => { + assert.equal(statusFromText("gh: API rate limit exceeded (HTTP 429)"), 429); + assert.equal(statusFromText("HTTP 429: rate limit exceeded"), 429); + assert.equal(statusFromText("HTTP 403: rate limit exceeded"), 403); + assert.equal(statusFromText("HTTP 404: Not Found"), 404); + assert.equal(statusFromText("network failure"), undefined); +}); + +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}" }, +}; +const install = { run: "npm ci" }; +const context = { visibility: "private", path: ".github/workflows/ci.yml" }; +function inspect(steps, overrides = {}) { + const job = classifyJob("build", { steps }, { ...context, ...overrides }, [ + "push", + ]); + return { job, disposition: repositoryDisposition([{ jobs: [job] }]) }; +} + +test("option values cannot masquerade as informational or publishing verbs", () => { + for (const run of [ + "npm --prefix help ci", + "npm --prefix publish ci", + "pnpm --filter list install", + "npm --prefix=help ci", + ]) { + for (const steps of [[{ run }], [setup, install, { run }]]) { + assert.equal(inspect(steps).disposition, "needs-review", run); + } + } +}); + +test("scoped and abbreviated registry options cannot earn protected", () => { + for (const run of [ + "npm install @fixture/pkg --@fixture:registry=https://registry.example.invalid", + "npm ci --reg=https://registry.example.invalid", + "npm ci --regis=https://registry.example.invalid", + "pnpm install --config.registry=https://registry.example.invalid", + ]) { + const result = inspect([setup, { run }]); + assert.equal(result.disposition, "needs-sfw", run); + assert.match(result.job.violations.join(" "), /registry-mutating/); + } + for (const run of [ + "npm ci --userc=other.npmrc", + "npm ci --future-config=other", + ]) { + assert.equal(inspect([setup, { run }]).disposition, "needs-review", run); + } + assert.equal( + inspect([setup, { run: "npm ci --ignore-scripts --no-audit --no-fund" }]) + .disposition, + "protected", + ); +}); + +test("ecosystem executors remain candidates rather than being excluded", () => { + for (const run of [ + "uv run npm ci", + "poetry run npm ci", + "conda run npm ci", + "pipenv run npm ci", + "go run bootstrap.go", + "cargo run --bin installer", + ]) { + assert.equal(inspect([{ run }]).disposition, "needs-review", run); + assert.equal( + inspect([setup, install, { run }]).disposition, + "needs-review", + run, + ); + } + assert.equal( + inspect([{ run: "pip install -r requirements.txt" }]).job.status, + "out-of-scope", + ); +}); + +test("targeted Corepack controls cannot clear a different manager's state", () => { + for (const enable of ["corepack enable", "corepack enable pnpm"]) { + assert.equal( + inspect([ + { run: enable }, + { run: "corepack disable yarn" }, + setup, + { run: "pnpm install" }, + ]).disposition, + "needs-review", + ); + } + assert.equal( + inspect([ + { run: "corepack enable" }, + { run: "corepack disable" }, + setup, + { run: "pnpm install" }, + ]).disposition, + "protected", + ); +}); + +test("conditional protection and legitimate fork fallback are review, not known gaps", () => { + for (const step of [ + { ...setup, if: "github.ref == 'refs/heads/main'" }, + { ...setup, "continue-on-error": true }, + ]) + assert.equal(inspect([step, install]).disposition, "needs-review"); + assert.equal( + inspect( + [ + { + ...setup, + with: { + token: "${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }}", + "allow-external-fork-fallback": true, + }, + }, + install, + ], + { visibility: "public" }, + ).disposition, + "needs-review", + ); + assert.equal( + inspect([{ ...setup, if: false }, install]).disposition, + "needs-sfw", + ); +}); + +test("unsupported YAML never emits source-bearing warnings or returns a clean workflow", () => { + const emitted = []; + const emitWarning = process.emitWarning; + process.emitWarning = (...args) => emitted.push(args); + try { + const invalidSources = [ + "name: !private-canary private-workflow-canary\n", + "? [private, canary]\n: value\n", + "---\nname: second-document\n", + "name: [unterminated\n", + ]; + for (const invalid of invalidSources) { + const source = `on: push\njobs:\n build:\n steps:\n - run: echo done\n${invalid}`; + assert.equal( + repositoryDisposition([classifyWorkflow(source, context)]), + "needs-review", + ); + assert.throws(() => parseYamlSource(source)); + } + const localActions = new Map([ + [ + "local/action.yml", + `name: !private-canary private-composite-canary\nruns:\n using: composite\n steps:\n - uses: ${setup.uses}\n with:\n token: '${setup.with.token}'\n - run: npm ci\n shell: bash\n`, + ], + ]); + assert.equal( + inspect([{ uses: "./local" }], { localActions }).disposition, + "needs-review", + ); + assert.deepEqual(emitted, []); + } finally { + process.emitWarning = emitWarning; + } +}); + +test("malformed non-blob modes produce partial audit errors, never no-ci", async () => { + const repository = { + name: "fixture", + defaultBranch: "main", + visibility: "private", + }; + for (const entry of [ + { type: "tree", mode: "100644" }, + { type: "tree" }, + { type: "commit", mode: "100644" }, + { type: "commit" }, + ]) { + const client = { + listRestRepositories: async () => [ + { ...repository, default_branch: "main", archived: false }, + ], + listGraphqlRepositories: async () => [ + { ...repository, isArchived: false }, + ], + getRef: async () => ({ object: { sha: "a".repeat(40) } }), + getTree: async () => ({ + truncated: false, + tree: [{ path: ".github/workflows/ci.yml", ...entry }], + }), + getText: async () => { + throw new Error("must not read malformed tree source"); + }, + }; + assert.equal( + (await auditRepository(client, repository)).disposition, + "audit-error", + ); + const report = await runAudit(client); + assert.equal(report.scanStatus, "partial"); + assert.equal(report.scanErrors, 1); + assert.equal(scanExitCode(report), 1); + } +}); diff --git a/tools/rollout/release.mjs b/tools/rollout/release.mjs index 906b7d2..8d724f2 100644 --- a/tools/rollout/release.mjs +++ b/tools/rollout/release.mjs @@ -1,5 +1,5 @@ import { readFile } from "node:fs/promises"; -import { parse as parseYaml } from "yaml"; +import { parseYamlSource as parseYaml } from "./yaml.mjs"; import { ACTION_REPOSITORY, diff --git a/tools/rollout/rollout.test.mjs b/tools/rollout/rollout.test.mjs index d64fddf..80fcf20 100644 --- a/tools/rollout/rollout.test.mjs +++ b/tools/rollout/rollout.test.mjs @@ -1,5 +1,7 @@ import assert from "node:assert/strict"; -import { readFile } from "node:fs/promises"; +import { mkdtemp, readFile, rm } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { describe, test } from "node:test"; import { @@ -17,6 +19,15 @@ import { import { main } from "./cli.mjs"; import { parseReleaseManifest, verifyActionRelease } from "./release.mjs"; +// Unit tests intentionally use a historical snapshot, not the evolving source +// manifest or live discovery refs. The manual verifier remains strict. +function verifyFixtureRelease(options) { + return verifyActionRelease({ + ...options, + manifestUrl: new URL("./fixtures/release-manifest.txt", import.meta.url), + }); +} + const TREE_SHA = "5cdbe39b0edafee9457767134320d95c61d91a60"; const ROOT_ACTION = ` @@ -106,7 +117,7 @@ describe("source dependency lockfile", () => { describe("release verification", () => { test("accepts the exact signed action-only release", async () => { - const result = await verifyActionRelease({ client: releaseClient() }); + const result = await verifyFixtureRelease({ client: releaseClient() }); assert.deepEqual(result, { branch: RELEASE_BRANCH, @@ -135,7 +146,7 @@ describe("release verification", () => { }); await assert.rejects( - verifyActionRelease({ client }), + verifyFixtureRelease({ client }), /release discovery refs do not match/, ); }); @@ -150,7 +161,7 @@ describe("release verification", () => { }); await assert.rejects( - verifyActionRelease({ client }), + verifyFixtureRelease({ client }), /release commit is not GitHub-verified/, ); }); @@ -168,7 +179,7 @@ describe("release verification", () => { }); await assert.rejects( - verifyActionRelease({ client }), + verifyFixtureRelease({ client }), /release tree differs from the reviewed manifest/, ); }); @@ -184,7 +195,7 @@ runs: `, }); - await assert.rejects(verifyActionRelease({ client }), /must execute only/); + await assert.rejects(verifyFixtureRelease({ client }), /must execute only/); }); test("rejects unsafe manifest paths", () => { @@ -394,7 +405,10 @@ describe("CLI", () => { await assert.rejects(main(["inventory", "--org", "other"], {}), /usage:/); }); - test("prints deterministic inventory JSON", async () => { + test("prints sanitized inventory JSON and keeps full inventory private", async (t) => { + const directory = await mkdtemp(join(tmpdir(), "sfw-inventory-")); + t.after(() => rm(directory, { recursive: true, force: true })); + const reportPath = join(directory, "inventory.json"); let output = ""; const client = { async listGraphqlRepositories() { @@ -407,6 +421,7 @@ describe("CLI", () => { await main(["inventory"], { client, + reportPath, output: { write(value) { output += value; @@ -415,5 +430,10 @@ describe("CLI", () => { }); assert.equal(JSON.parse(output).activeCount, 1); + assert.equal(JSON.parse(output).repositories, undefined); + assert.equal( + JSON.parse(await readFile(reportPath, "utf8")).repositories[0].name, + "one", + ); }); }); diff --git a/tools/rollout/yaml.mjs b/tools/rollout/yaml.mjs new file mode 100644 index 0000000..794db3a --- /dev/null +++ b/tools/rollout/yaml.mjs @@ -0,0 +1,18 @@ +import { parseDocument } from "yaml"; + +// yaml.parse() emits source-bearing warnings directly to stderr. Keep parsing +// non-emitting and reject unsupported constructs instead of guessing at them. +export function parseYamlSource(text) { + const document = parseDocument(text, { + logLevel: "error", + stringKeys: true, + prettyErrors: false, + }); + const diagnostics = [...document.errors, ...document.warnings]; + if (diagnostics.length > 0) { + throw new Error( + `YAML requires review: ${[...new Set(diagnostics.map((item) => item.code))].join(", ")}`, + ); + } + return document.toJS({ maxAliasCount: 100 }); +} From f59a7a23f87a3f2c1a8fc73b93903a99dd5a68af Mon Sep 17 00:00:00 2001 From: Matt Peake Date: Mon, 14 Sep 2026 13:11:12 -0400 Subject: [PATCH 06/19] Separate observed SFW gaps from execution assurance --- CONTRIBUTING.md | 2 + README.md | 27 +- tools/rollout/audit.mjs | 16 +- tools/rollout/classify.mjs | 318 ++++++++-- tools/rollout/classify.test.mjs | 6 +- tools/rollout/cli.mjs | 3 + tools/rollout/integration.mjs | 322 ++++++++++ tools/rollout/integration.test.mjs | 728 ++++++++++++++++++++++ tools/rollout/primary-boundaries.test.mjs | 164 +++++ 9 files changed, 1539 insertions(+), 47 deletions(-) create mode 100644 tools/rollout/integration.mjs create mode 100644 tools/rollout/integration.test.mjs create mode 100644 tools/rollout/primary-boundaries.test.mjs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index f133679..57afe39 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,6 +42,8 @@ The minimum test-coverage policy is one shell test suite for every executable sh `npm run inventory` and `npm run audit:live` are read-only manual commands using an existing authorized `gh` session. Confirm organization-wide read access separately before making organization-wide claims. No scheduling or remediation is implicit. See README for the scan scope and limitations. +Audit schema2 separates primary observed-install `dispositions` from strict `assuranceDispositions`; retain both views in regressions. Test that bounded ordinary package-script roles under valid setup do not erase observed configuration, while scripts before valid setup, under invalid/unresolved setup or in script-only sibling jobs remain visible. Cover persistent conditional Corepack state, malformed local source after covered installs, inline/YAML environment scope, unsupported env wrappers, literal logging/prologues versus shell source/substitutions, direct executor argument boundaries and exact npm lockfile-host replacement versus actual registry overrides. `integrated` is not runtime proof; strict opacity and security findings remain separate. Do not introduce a historical cohort, coverage percentage or repository allowlist. + Full results are owner-only, ignored `reports/inventory.json` and `reports/live-audit.json`; terminal JSON contains sanitized counts. Audit `scanErrors`/`scanStatus` distinguish operational failures from discovered gaps. Exit 1 includes partial scans; exit 0 can still include `needs-sfw` or `needs-review`. Do not force-add reports, expose inventory/source in logs, or upload full reports as artifacts. `npm run verify-action` is a separate live, strict historical release snapshot check, not a normal CI check. Advancing `v1` or changing the current release manifest can intentionally invalidate it. Mocked release tests use `tools/rollout/fixtures/release-manifest.txt`, preserving integrity regression tests without binding source CI to future manifest changes. Keep the actual runtime manifest and action-only publication boundaries intact. diff --git a/README.md b/README.md index 48199ee..0683a05 100644 --- a/README.md +++ b/README.md @@ -214,22 +214,33 @@ npm run audit:live # read-only, default-branch workflow candidate scan Terminal output is sanitized JSON counts, not repository names or workflow source. Full inventory and audit JSON go to `reports/inventory.json` and `reports/live-audit.json`, respectively. These paths are ignored; reports are atomically replaced with owner-only (`0600`) permissions. Keep them private: never force-add, publish, or upload reports as CI artifacts. Both commands take no CLI flags; a later run replaces the previous report. -Audit JSON includes `schemaVersion`, `scanStatus`, `scanErrors`, disposition counts, and per-repository `headSha`, workflow/job operations and violations. `scanStatus: complete` means the token-visible scan completed, **not** that all jobs are protected. `partial` means one or more repositories have an `audit-error` row. CLI exit **0** means the scan completed, even when gaps/review candidates were found; exit **1** means an operational failure (including partial scans). Consumers should inspect dispositions separately from the exit code. Fatal inventory/API/command errors emit a sanitized error and exit 1; they do not refresh the report, so check its timestamp before use. +Audit JSON schema **2** separates primary `dispositions` (observed dependency-install integration) from `assuranceDispositions` (strict execution/security review). Successfully inspected repositories retain both dispositions, their `headSha`, workflow/job operations and violations; read errors remain explicit error rows. Each job's `integration` includes per-download `covered`, `fork-exception`, `gap` or `unresolved` evidence, reason codes and diagnostic notes. `runtimeVerification` is always `not-performed`. `scanStatus: complete` means the token-visible scan completed, **not** that all jobs are protected. `partial` means one or more repositories have an `audit-error` row. CLI exit **0** means the scan completed, even when gaps/review candidates were found; exit **1** means an operational failure (including partial scans). Consumers should inspect dispositions separately from the exit code. Fatal inventory/API/command errors emit a sanitized error and exit 1; they do not refresh the report, so check its timestamp before use. | Repository disposition | Meaning | | --- | --- | -| `needs-sfw` | A recognized download lacks a certain supported setup interval. | -| `needs-review` | Opaque behavior, reusable call, or malformed workflow prevents a conclusion. | -| `blocked-trust`, `unsafe-publish`, `blocked-yarn` | Existing trust, teardown/publication, or Yarn compatibility rule needs review. | -| `protected` | All recognized downloads satisfy the static rules, with no detected opaque operation; not runtime verification. | +| `needs-sfw` | A recognized download has a definite missing/unsupported setup interval in its own job. | +| `needs-review` | An observed install interval, script-only installation-capable job, reusable call or malformed source needs investigation. | +| `integrated` | Recognized downloads have supported static setup intervals, possibly with an explicit public-fork exception; opaque execution remains unverified. | | `audit-error` | A repository read failed or was incomplete; never a clean result. | -| `no-ci`, `no-js-ci`, `out-of-scope`, `empty` | No recognized in-scope candidate in the inspected source; not an organization-wide assurance. | +| `no-ci`, `no-js-ci`, `empty` | No observed in-scope download candidate; not proof that remote actions or arbitrary code cannot download packages. | -Job statuses remain more detailed (`unprotected`, `unknown`, `reusable-call`, etc.). Ambiguous execution produces `needs-review`, with any recognized violations retained in the private report. A separate, clearly unprotected job still makes the repository a `needs-sfw` candidate. A `safe-publish` job is a static publication-only classification, not proof that lifecycle hooks are safe. +Primary integration is not a rollout percentage or a certification of every command. For example, setup → `npm ci` → `npm test` can be `integrated` while strict assurance remains `needs-review`. Without setup, the same direct install remains a `needs-sfw` candidate even though the later test is opaque. A new uncovered install job cannot be masked by an integrated sibling; a script-only installation-capable sibling remains `needs-review`. + +Strict job `status` and `assuranceDisposition` retain opaque execution and security findings (`blocked-trust`, `unsafe-publish`, `blocked-yarn`, etc.). **Inspect those findings separately; `integrated` does not dismiss them.** A strict `protected` or `safe-publish` status remains static evidence, not runtime or lifecycle-hook proof. Schema1 consumers must not interpret schema2 primary dispositions as the old assurance verdicts. ### Static limits -The small grammar recognizes direct npm/pnpm/Bun downloads, Yarn blockers, setup/teardown order and registry overrides. Every identified download needs an active approved-SHA setup using the visibility-appropriate token expression. Disabled setup cannot protect; conditional/continue-on-error boundaries, complex shell constructs, containers, unresolved scripts/actions and reusable workflows cannot earn `protected`. Referenced local composites are inspected with bounded nesting and cycle detection; dynamic inputs and inner/outer uncertainty remain review candidates. Public external-fork fallback is an explicit unprotected path, not verified protection. +The small grammar recognizes direct npm/pnpm/Bun downloads, Yarn blockers, setup/teardown order and registry overrides. Each recognized download is evaluated in its own job and interval, using the approved SHA and visibility-appropriate token expression. Disabled setup provides no integration evidence; uncertain setup/install conditions, complex shell constructs and actual job containers remain unresolved. Sibling service containers alone do not invalidate host setup ordering. A job-level guard affects shared reachability rather than relative setup ordering. Ordinary run-step guards retain GitHub's implicit success gating; explicit status-function guards, conditional setup/teardown and conditional Corepack transitions remain conservative. Unresolved setup token expressions (including composite input forwarding) require review, not a claim of a wrong token; literal wrong visibility-specific secrets remain gaps. + +Primary environment checks cover package-manager configuration prefixes, home/config paths, executable selection, shell startup, loader and proxy controls. Ordinary mapping keys for application credentials/settings and toolchain-version metadata do not alone invalidate observed configuration, including secret-valued expressions. Dynamic mappings, malformed keys and nonscalar values remain unresolved. The same rule applies to inline assignments: `CI=1 npm ci` and `env CI=1 npm ci` are supported; HOME/PATH/NODE_OPTIONS overrides and option-bearing `env` wrappers require review. Strict assurance retains blanket environment uncertainty. Literal Bash/sh defaults, exact `set -e` / `set -euo pipefail` prologues and literal `echo` logging do not alone invalidate primary evidence. Substitutions, imports, heredocs, redirections and unsupported shell switches are not interpreted. + +Opaque scripts/actions remain strict-assurance diagnostics. Primary integration recognizes a bounded ordinary package-script role (`npm`/`pnpm`/Bun/Yarn `run` with a literal script name, or `test`/`start`/`stop`/`restart`, with literal arguments; option arguments require a `--` separator). After a valid setup interval this role is not itself an observed routing invalidation, so setup → `npm test` → `npx wrangler deploy --env production` can be `integrated`. This does **not** declare scripts safe or no-network. Script-only jobs, execution before valid setup or under unresolved/invalid setup, explicit configuration changes and teardown remain visible; a later valid install cannot hide an earlier unresolved installation-capable path. Other opaque execution before a later install can make its routing unresolved; a fresh explicit setup can re-establish configuration, but cannot erase earlier unresolved execution. + +For direct `npx`/`bunx` only, a literal target (optionally preceded by `-y`/`--yes`) separates installer options from literal command arguments. Post-target `--env` or `--registry` belongs to the invoked command, not the installer's routing configuration; invoked code remains unverified. Pre-target registry overrides remain gaps; unsupported pre-target options, dynamic targets/arguments and npm-exec remain unresolved. Exact npm `--replace-registry-host=always` preserves the configured registry and is supported; other replacement modes require review. Registry/scoped/abbreviated overrides are not exempted. Corepack shim uncertainty persists across conditional/targeted controls and setup; a certain unqualified disable can reset it. + +Primary integration also makes a narrow toolchain-role assumption: standard `pnpm/action-setup` (no `run_install`, or literal false) and `oven-sh/setup-bun` with supported version inputs do not invalidate routing for subsequent explicit installs. Unsupported inputs and subpaths remain unresolved. This does **not** certify their bootstrap downloads or action implementations, and they never substitute for SFW setup. + +Referenced local composites have bounded nesting and cycle detection; unresolved installation-capable paths remain candidates. Missing, malformed or unresolvable local-action source retains a separate error marker and requires review even after a covered install. Public external-fork fallback is recorded per download as `fork-exception`: an explicit unprotected path, not verified protection. No historical repository cohort or exception allowlist is used. Only top-level `.github/workflows/*.yml|yaml` and `.depot/workflows/*.yml|yaml` on active repositories' default branches are inspected. The detector does not execute workflows, resolve package scripts/lifecycle hooks, validate credential existence, prove actual registry traffic, interpret arbitrary shell, inspect remote action/reusable-workflow implementations, or certify bespoke controls. Local-action resolution is deliberately limited; unsupported paths or execution contexts need review. Install hooks and downloaded code remain outside the runtime-proof claim. Treat results as a prioritized human-review queue. diff --git a/tools/rollout/audit.mjs b/tools/rollout/audit.mjs index c111e58..87d92ee 100644 --- a/tools/rollout/audit.mjs +++ b/tools/rollout/audit.mjs @@ -4,6 +4,7 @@ import { dirname, join } from "node:path"; import { classifyWorkflow, repositoryDisposition } from "./classify.mjs"; import { captureRepositoryInventory } from "./inventory.mjs"; import { ORGANIZATION } from "./constants.mjs"; +import { integrationDisposition } from "./integration.mjs"; const WORKFLOW_PATH_PATTERN = /^\.(?:github|depot)\/workflows\/[^/]+\.ya?ml$/; const LOCKFILE_PATTERN = @@ -230,7 +231,8 @@ export async function auditRepository(client, repository) { return { defaultBranch: repository.defaultBranch, - disposition: repositoryDisposition(workflows), + disposition: integrationDisposition(workflows), + assuranceDisposition: repositoryDisposition(workflows), headSha, jobs: { byStatus: Object.fromEntries( @@ -279,14 +281,24 @@ export async function runAudit(client, options = {}) { const rows = sortedByName(repositories); const dispositions = {}; + const assuranceDispositions = {}; for (const row of rows) { dispositions[row.disposition] = (dispositions[row.disposition] ?? 0) + 1; + const assurance = row.assuranceDisposition ?? row.disposition; + assuranceDispositions[assurance] = + (assuranceDispositions[assurance] ?? 0) + 1; } return { dispositions: Object.fromEntries( Object.entries(dispositions).sort(([a], [b]) => (a < b ? -1 : 1)), ), + assuranceDispositions: Object.fromEntries( + Object.entries(assuranceDispositions).sort(([a], [b]) => + a < b ? -1 : 1, + ), + ), + runtimeVerification: "not-performed", generatedAt: new Date().toISOString(), inventory: { activeCount: inventory.activeCount, @@ -301,7 +313,7 @@ export async function runAudit(client, options = {}) { coverage: "token-visible repositories only; organization-wide access is an operator prerequisite", repositories: rows, - schemaVersion: 1, + schemaVersion: 2, }; } diff --git a/tools/rollout/classify.mjs b/tools/rollout/classify.mjs index b5f464d..e489a38 100644 --- a/tools/rollout/classify.mjs +++ b/tools/rollout/classify.mjs @@ -1,6 +1,13 @@ import { parseYamlSource as parse } from "./yaml.mjs"; import { ACTION_REPOSITORY, APPROVED_RELEASE_SHA } from "./constants.mjs"; +import { + certainTeardown, + defaultsUncertain, + environmentUncertain, + observedIntegration, + toolchainPreservesRouting, +} from "./integration.mjs"; const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/; const UNSAFE_PUBLIC_TRIGGERS = new Set([ @@ -117,6 +124,19 @@ function normalizeExpression(value) { return String(value ?? "").replaceAll(/\s+/g, ""); } +// An ordinary run-step guard still has GitHub's implicit success() gating. +// It can skip an install/script, but cannot run it after failed setup. Keep +// explicit status predicates conservative and never apply this to uses/composites. +function primaryRunGuard(step) { + if ( + typeof step.run === "string" && + typeof step.if === "string" && + !/\b(?:always|cancelled|failure|success)\s*\(/i.test(step.if) + ) + return undefined; + return step.if; +} + function expectedTokenExpression(visibility) { const secret = visibility === "public" @@ -127,22 +147,75 @@ function expectedTokenExpression(visibility) { function splitCommands(script) { return String(script) - .split(/\r?\n|&&|\|\||;|(? + literalLogging(line.trim()) + ? [line] + : line.split(/&&|\|\||;|(? command.trim()) .filter((command) => command.length > 0 && !command.startsWith("#")); } -function commandWords(command) { +function commandParts(command) { const words = command.split(/\s+/).filter((word) => word.length > 0); + const environment = {}; let start = 0; - while ( - start < words.length && - (/^[A-Za-z_][A-Za-z0-9_]*=/.test(words[start]) || - ["sudo", "time", "env", "exec"].includes(words[start])) - ) { + const assignments = () => { + while (/^[A-Za-z_][A-Za-z0-9_]*=/.test(words[start] ?? "")) { + // Keep keys for scope checks, never copy values to new diagnostics. + Object.defineProperty(environment, words[start].split("=", 1)[0], { + value: true, + enumerable: true, + configurable: true, + }); + start += 1; + } + }; + assignments(); + if (words[start] === "env" && !words[start + 1]?.startsWith("-")) { start += 1; + assignments(); } - return words.slice(start); + return { words: words.slice(start), environment }; +} + +function commandWords(command) { + return commandParts(command).words; +} + +// Literal logging only: quoted source, substitution and shell controls are not +// interpreted. These lines are transparent only to observed configuration. +function literalLogging(command) { + // GitHub interpolates expressions before the shell sees even single quotes. + if (command.includes("${{")) return false; + return /^echo(?:\s+(?:"[^"$`\\\\]*"|'[^']*'|[A-Za-z0-9_.,:/@+=-]+))*\s*$/.test( + command, + ); +} + +function safePrologue(command) { + return /^set\s+(?:-e|-euo\s+pipefail)$/.test(command); +} + +const REGISTRY_MUTATION = + /--(?:[^\s=]*:)?reg[a-z]*\b|--[^\s=]*(?:registry|userconfig)|(?:NPM|PNPM|BUN)_CONFIG_|npm_config_|\bnpm\s+config\s+(?:set|delete)\b/i; + +function directExecutor(words) { + if (!["npx", "bunx"].includes(words[0])) return undefined; + let target = 1; + while (["-y", "--yes"].includes(words[target])) target += 1; + // Only literal package/binary targets and literal arguments; no npm-exec + // grammar, shell interpretation or unsupported installer options. + if (!/^[A-Za-z0-9_@][A-Za-z0-9_@./:+-]*$/.test(words[target] ?? "")) + return undefined; + if ( + words + .slice(target + 1) + .some((word) => !/^[A-Za-z0-9_@./:=,+%-]+$/.test(word)) + ) + return undefined; + return words.slice(0, target).join(" "); } function firstSubcommand(words) { @@ -155,7 +228,8 @@ export function classifyCommand(command) { if (!program) { return { command, kind: "no-network" }; } - if (program.includes("/")) return { command, kind: "unknown-wrapper" }; + if (program.includes("/")) + return { command, program, kind: "unknown-wrapper" }; const name = program; // Do not mistake an option's value (e.g. --prefix help) for the verb. @@ -178,7 +252,13 @@ export function classifyCommand(command) { const subcommand = firstSubcommand(words); if (subcommand === "enable" || subcommand === "disable") { // Targeted controls may affect Yarn without changing pnpm's shim. - if (words.length !== 2) return { command, kind: "unknown-wrapper" }; + if (words.length !== 2) + return { + command, + program, + corepackControl: subcommand, + kind: "unknown-wrapper", + }; return { command, corepack: subcommand, kind: "no-network" }; } return { @@ -236,13 +316,14 @@ export function classifyCommand(command) { return { command, kind: executesProgram ? "unknown-wrapper" : "other-ecosystem", + installationCapable: executesProgram, }; } if (ORCHESTRATORS.has(name)) { if (name === "lerna" && words.includes("bootstrap")) { return { command, kind: "js-public-download", manager: "npm" }; } - return { command, kind: "unknown-wrapper" }; + return { command, program, kind: "unknown-wrapper" }; } if ( WRAPPER_COMMANDS.has(name) || @@ -250,10 +331,11 @@ export function classifyCommand(command) { ((name === "bash" || name === "sh" || name === "zsh") && words.slice(1).some((word) => /\.sh$/.test(word))) ) { - return { command, kind: "unknown-wrapper" }; + return { command, program, kind: "unknown-wrapper" }; } return { command, + program, kind: SIMPLE_COMMANDS.has(name) ? "no-network" : "unknown-wrapper", }; } @@ -294,6 +376,7 @@ function classifyUsesStep(step, context) { kind: "unknown-local-action", uses: step.uses, reason: "cyclic or deeply nested local action", + sourceError: "unresolved-local-action-expansion", }, ]; } @@ -301,13 +384,40 @@ function classifyUsesStep(step, context) { context.localActions?.get(`${uses.path}/action.yml`) ?? context.localActions?.get(`${uses.path}/action.yaml`); if (actionText === undefined) { - return [{ kind: "unknown-local-action", uses: step.uses }]; + return [ + { + kind: "unknown-local-action", + uses: step.uses, + sourceError: "unresolved-local-action-source", + }, + ]; } let action; try { action = parse(actionText); } catch { - return [{ kind: "unknown-local-action", uses: step.uses }]; + return [ + { + kind: "unknown-local-action", + uses: step.uses, + sourceError: "local-action-parse-error", + }, + ]; + } + if ( + !isMapping(action) || + !isMapping(action.runs) || + typeof action.runs.using !== "string" || + (action.runs.using === "composite" && + (!Array.isArray(action.runs.steps) || action.runs.steps.length === 0)) + ) { + return [ + { + kind: "unknown-local-action", + uses: step.uses, + sourceError: "malformed-local-action", + }, + ]; } const steps = action?.runs?.steps; if ( @@ -345,6 +455,12 @@ function classifyUsesStep(step, context) { withInput["allow-external-fork-fallback"] ?? "false", ), kind, + id: step.id, + condition: step.if, + boundaryUncertainWithoutCondition: boundaryUncertain({ + ...step, + if: undefined, + }), ref: uses.ref, token: normalizeExpression(withInput.token ?? ""), uses: step.uses, @@ -387,9 +503,21 @@ function classifyUsesStep(step, context) { }, ]; } - return [{ kind: "unknown", uses: step.uses }]; + return [ + { + kind: "unknown", + uses: step.uses, + integrationToolchain: toolchainPreservesRouting(uses, withInput), + }, + ]; } - return [{ kind: "unknown", uses: step.uses }]; + return [ + { + kind: "unknown", + uses: step.uses, + integrationToolchain: toolchainPreservesRouting(uses, withInput), + }, + ]; } export function classifyStep(step, context = {}) { @@ -398,13 +526,24 @@ export function classifyStep(step, context = {}) { { kind: "unknown-local-action", reason: "local action expansion limit reached", + sourceError: "unresolved-local-action-expansion", }, ]; } if (!isMapping(step)) return [{ kind: "unknown" }]; if (condition(step.if) === "never") return []; + if ( + isMapping(step.with) && + Object.values(step.with).some( + (value) => value !== null && typeof value === "object", + ) + ) + return [{ kind: "unknown", sourceError: "malformed-action-input" }]; let operations; let uncertain = boundaryUncertain(step); + let integrationUncertain = + boundaryUncertain({ ...step, env: undefined, if: primaryRunGuard(step) }) || + environmentUncertain(step.env); if ( step.uses !== undefined && step.run === undefined && @@ -415,31 +554,99 @@ export function classifyStep(step, context = {}) { } else if (typeof step.run === "string" && step.uses === undefined) { // Deliberately not a shell interpreter: retain recognizable downloads but // never certify control flow, substitutions, redirections or custom shells. - uncertain ||= + const complexShell = (script) => /[;'"$`|<>(){}\\]|(?:^|\s)(?:if|then|else|fi|for|while|case|eval|source|sudo)(?:\s|$)|(? !literalLogging(line.trim())) + .join("\n"), + ); operations = splitCommands(step.run).map((command) => { const operation = classifyCommand(command); - if ( - /--(?:[^\s=]*:)?reg[a-z]*\b|--[^\s=]*(?:registry|userconfig)|(?:NPM|PNPM|BUN)_CONFIG_|npm_config_|\bnpm\s+config\s+(?:set|delete)\b/i.test( - command, - ) - ) { - operation.registryMutating = true; - } else if ( + const { words, environment } = commandParts(command); + const executorPrefix = directExecutor(words); + // npm's exact 'always' mode rewrites lockfile hosts to the configured + // registry; it does not select another registry. Other modes need review. + const replacementFlags = words.filter((word) => + word.startsWith("--replace-registry-host"), + ); + const configurationWords = command + .split(/\s+/) + .filter((word) => !word.startsWith("--replace-registry-host")); + const supportedReplacement = + words[0] === "npm" && + replacementFlags.every( + (word) => word === "--replace-registry-host=always", + ); + const registryMutating = + !literalLogging(command) && + REGISTRY_MUTATION.test(configurationWords.join(" ")); + const unsupportedFlags = operation.kind === "js-public-download" && operation.corepack === undefined && - commandWords(command).some( - (word) => - word.startsWith("-") && !TRANSPARENT_INSTALL_FLAGS.has(word), - ) - ) { - // Package managers accept abbreviated/configuration flags. Rather than - // emulate each parser, keep unrecognized options as review candidates. + words + .filter( + (word) => + !( + supportedReplacement && + word === "--replace-registry-host=always" + ), + ) + .some( + (word) => + word.startsWith("-") && !TRANSPARENT_INSTALL_FLAGS.has(word), + ); + if (registryMutating) operation.registryMutating = true; + else if ( + unsupportedFlags || + (replacementFlags.length > 0 && !supportedReplacement) + ) + operation.uncertain = true; + if (Object.keys(environment).length && !registryMutating) operation.uncertain = true; + operation.integrationUncertain = + environmentUncertain(environment) || + (replacementFlags.length > 0 && !supportedReplacement) || + (words[0] === "npm" && ["exec", "x"].includes(words[1])) || + (unsupportedFlags && !registryMutating); + if (["npx", "bunx"].includes(words[0])) { + // Strict assurance still sees unknown executor code/configuration flags. + operation.uncertain = true; + operation.integrationExecutor = executorPrefix !== undefined; + operation.integrationUncertain = + environmentUncertain(environment) || + (executorPrefix === undefined && !registryMutating); + if (executorPrefix !== undefined) + operation.registryMutating = REGISTRY_MUTATION.test(executorPrefix); } + // Reachability is not a certain state transition: a skipped disable + // cannot clear a persistent Corepack shim established by an earlier step. + if ( + operation.corepackControl || + ["enable", "disable"].includes(operation.corepack) + ) + operation.integrationUncertain ||= condition(step.if) === "uncertain"; + operation.integrationTransparent = + safePrologue(command) || literalLogging(command); + operation.integrationScript = + ["npm", "pnpm", "bun", "yarn"].includes(words[0]) && + ((words[1] === "run" && /^[A-Za-z0-9_:.+-]+$/.test(words[2] ?? "")) || + ["test", "start", "stop", "restart"].includes(words[1])) && + words + .slice(2) + .every( + (word, index) => + /^[A-Za-z0-9_@./:=,+%-]+$/.test(word) && + (!word.startsWith("-") || + word === "--" || + words.slice(2, index + 2).includes("--")), + ) && + !registryMutating; return operation; }); } else { @@ -448,6 +655,9 @@ export function classifyStep(step, context = {}) { return operations.map((operation) => ({ ...operation, ...(uncertain || operation.uncertain ? { uncertain: true } : {}), + integrationUncertain: + integrationUncertain || + (operation.integrationUncertain ?? operation.uncertain ?? false), })); } @@ -472,6 +682,7 @@ function collectViolations(operations, context, triggers, job) { const publishes = []; const expectedToken = expectedTokenExpression(context.visibility); let activeSetup; + let lastSetup; let registryInvalidated = false; let publishBoundary = true; let corepackEnabled = false; @@ -520,6 +731,7 @@ function collectViolations(operations, context, triggers, job) { } } if (operation.kind === "sfw-setup") { + lastSetup = operation; registryInvalidated = false; setups.push(index); publishBoundary = false; @@ -551,8 +763,7 @@ function collectViolations(operations, context, triggers, job) { if (operation.kind === "sfw-teardown") { registryInvalidated = false; activeSetup = undefined; - publishBoundary = - operation.ref === APPROVED_RELEASE_SHA && !operation.uncertain; + publishBoundary = certainTeardown(operation, lastSetup); if (operation.ref !== APPROVED_RELEASE_SHA) { violations.push( `sfw-teardown ref "${operation.ref}" does not match the approved release SHA`, @@ -619,9 +830,30 @@ export function classifyJob(jobName, job, context = {}, triggers = []) { operations: [], status: "unknown", violations: ["job definition is not a mapping"], + integration: { + disposition: "needs-review", + downloads: [], + notes: ["malformed-job"], + runtimeVerification: "not-performed", + }, }; } + if (job.uses !== undefined && condition(job.if) === "never") { + return { + job: jobName, + managers: [], + operations: [], + status: "no-in-scope-download", + violations: [], + integration: { + disposition: "no-js-ci", + downloads: [], + notes: [], + runtimeVerification: "not-performed", + }, + }; + } if (job.uses !== undefined) { const violations = job.secrets === "inherit" @@ -633,6 +865,12 @@ export function classifyJob(jobName, job, context = {}, triggers = []) { operations: [{ kind: "reusable-call", uses: String(job.uses) }], status: "reusable-call", violations, + integration: { + disposition: "needs-review", + downloads: [], + notes: ["unresolved-reusable-workflow"], + runtimeVerification: "not-performed", + }, }; } @@ -720,7 +958,14 @@ export function classifyJob(jobName, job, context = {}, triggers = []) { status = "no-in-scope-download"; } - return { job: jobName, managers, operations, status, violations }; + return { + job: jobName, + managers, + operations, + status, + violations, + integration: observedIntegration(operations, job, context), + }; } export function classifyWorkflow(text, context) { @@ -771,6 +1016,9 @@ export function classifyWorkflow(text, context) { ...context, workflowUncertain: workflow.env !== undefined || workflow.defaults !== undefined, + integrationContextUncertain: + environmentUncertain(workflow.env) || + defaultsUncertain(workflow.defaults), }, triggers, ), diff --git a/tools/rollout/classify.test.mjs b/tools/rollout/classify.test.mjs index 3018a2c..dc722a9 100644 --- a/tools/rollout/classify.test.mjs +++ b/tools/rollout/classify.test.mjs @@ -723,7 +723,9 @@ test("audit: moving branch cannot change tree or nested source snapshot", async fixtureRepository, ); assert.equal(row.headSha, sha); - assert.equal(row.disposition, "needs-review"); + // The certain install precedes the unresolved cycle; retain both signals. + assert.equal(row.disposition, "needs-sfw"); + assert.equal(row.assuranceDisposition, "needs-review"); assert.deepEqual(reads.sort(), [...sources.keys()].sort()); assert.ok( row.workflows[0].jobs[0].operations.some( @@ -921,7 +923,7 @@ test("classifier: composite inner boundaries survive expansion", () => { { ...teardownStep, ...boundary }, { run: "npm publish" }, ]).status, - "unsafe-publish", + boundary.if === "always()" ? "unknown" : "unsafe-publish", ); } }); diff --git a/tools/rollout/cli.mjs b/tools/rollout/cli.mjs index 07fcc9e..bb8ae7b 100644 --- a/tools/rollout/cli.mjs +++ b/tools/rollout/cli.mjs @@ -41,7 +41,10 @@ export async function main(argv, options = {}) { }); await writeReportAtomically(reportPath, report); const summary = { + schemaVersion: report.schemaVersion, dispositions: report.dispositions, + assuranceDispositions: report.assuranceDispositions, + runtimeVerification: report.runtimeVerification, inventory: inventoryCounts(report.inventory), scanErrors: report.scanErrors, scanStatus: report.scanStatus, diff --git a/tools/rollout/integration.mjs b/tools/rollout/integration.mjs new file mode 100644 index 0000000..ed495ea --- /dev/null +++ b/tools/rollout/integration.mjs @@ -0,0 +1,322 @@ +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; + +// Configuration/executable/startup controls, not application credentials. +const RELEVANT_ENV = + /^(?:(?:npm|pnpm|bun|yarn|corepack)_|HOME$|USERPROFILE$|XDG_|APPDATA$|LOCALAPPDATA$|PATH$|NODE_OPTIONS$|NODE_PATH$|BASH_ENV$|ENV$|SHELL$|SHELLOPTS$|BASHOPTS$|CDPATH$|LD_|DYLD_|BASH_FUNC_|HTTP_PROXY$|HTTPS_PROXY$|ALL_PROXY$|NO_PROXY$|NODE_EXTRA_CA_CERTS$|NODE_TLS_REJECT_UNAUTHORIZED$|SSL_CERT_|CURL_CA_BUNDLE$)/i; +// Workflow toolchain-version metadata does not select a registry/config path. +const VERSION_ENV = new Set([ + "NODE_VERSION", + "PNPM_VERSION", + "BUN_VERSION", + "XCODE_VERSION", +]); +const normalize = (value) => String(value ?? "").replaceAll(/\s+/g, ""); +const expression = (value) => + normalize(value).replace(/^\$\{\{(.*)\}\}$/, "$1"); + +export function environmentUncertain(env) { + if (env === undefined) return false; + if (!env || typeof env !== "object" || Array.isArray(env)) return true; + return Object.entries(env).some( + ([key, value]) => + !/^[A-Za-z_][A-Za-z0-9_]*$/.test(key) || + (RELEVANT_ENV.test(key) && !VERSION_ENV.has(key.toUpperCase())) || + !["string", "number", "boolean"].includes(typeof value), + ); +} + +export function defaultsUncertain(defaults) { + if (defaults === undefined) return false; + return ( + !defaults || + typeof defaults !== "object" || + Array.isArray(defaults) || + Object.keys(defaults).some((key) => key !== "run") || + !defaults.run || + typeof defaults.run !== "object" || + Array.isArray(defaults.run) || + Object.entries(defaults.run).some( + ([key, value]) => key !== "shell" || !["bash", "sh"].includes(value), + ) + ); +} + +// Only the documented toolchain role is assumed here, not bootstrap traffic or +// arbitrary action implementation safety. Strict assurance keeps these opaque. +export function toolchainPreservesRouting(uses, inputs) { + if (uses.subpath !== "" || uses.kind !== "remote" || !uses.ref) return false; + const supported = + uses.repository === "pnpm/action-setup" + ? ["version", "package_json_file", "run_install"] + : uses.repository === "oven-sh/setup-bun" + ? ["bun-version", "bun-version-file"] + : undefined; + if (!supported || Object.keys(inputs).some((key) => !supported.includes(key))) + return false; + return ( + inputs.run_install === undefined || + inputs.run_install === false || + inputs.run_install === "false" + ); +} + +export function certainTeardown(operation, setup) { + if (operation.ref !== APPROVED_RELEASE_SHA) return false; + if (!operation.uncertain) return true; + if (operation.via || operation.boundaryUncertainWithoutCondition) + return false; + const guard = expression(operation.condition); + if (guard === "always()") return true; + const active = guard.match( + /^always\(\)&&steps\.([A-Za-z_][\w-]*)\.outputs\.active==(['"])true\2$/, + ); + return Boolean( + active && + !setup?.via && + setup?.id === active[1] && + !setup.uncertain && + setup.ref === operation.ref, + ); +} + +function installationCapable(operation) { + if (operation.corepackControl) return false; + if (operation.kind === "unknown-local-action") return true; + if (operation.kind !== "unknown-wrapper") return false; + const program = + operation.program ?? + String(operation.command ?? "") + .trim() + .split(/\s+/)[0]; + return Boolean( + operation.manager || + operation.installationCapable || + program.includes("/") || + /['"$`=]/.test(program) || + /\.(?:[cm]?js|sh|py|rb)$/.test(program) || + [ + "env", + "npm", + "pnpm", + "bun", + "yarn", + "node", + "command", + "sudo", + "time", + "exec", + "eval", + "source", + ".", + "python", + "python3", + "ruby", + "perl", + "bash", + "sh", + "zsh", + "make", + "just", + "mise", + "rake", + "task", + "corepack", + "lerna", + "nx", + "rush", + "turbo", + ].includes(program), + ); +} + +function contextUncertain(job, context) { + return ( + environmentUncertain(job.env) || + defaultsUncertain(job.defaults) || + (context.integrationContextUncertain ?? context.workflowUncertain) || + job.container !== undefined + ); +} + +export function observedIntegration(operations, job, context) { + const downloads = []; + const notes = new Set(); + const expectedToken = `\${{secrets.${context.visibility === "public" ? "PUBLIC_SOCKET_FIREWALL_TOKEN" : "SOCKET_FIREWALL_TOKEN"}}}`; + let setup; + let state; + let opaque = false; + let corepack = "disabled"; + let potentialInstall = false; + let unresolvedEarlyInstall = false; + const uncertainContext = contextUncertain(job, context); + const malformed = + !Array.isArray(job.steps) || + job.steps.length === 0 || + operations.some( + (operation) => + operation.sourceError || + (operation.kind === "unknown" && !operation.uses && !operation.reason), + ); + if (malformed) notes.add("malformed-job-or-step"); + + for (const [index, operation] of operations.entries()) { + const uncertain = operation.integrationUncertain ?? operation.uncertain; + if (operation.corepackControl) corepack = "unresolved"; + if (["enable", "disable"].includes(operation.corepack)) + corepack = uncertain + ? "unresolved" + : operation.corepack === "enable" + ? "enabled" + : "disabled"; + if (operation.kind === "sfw-setup") { + setup = operation; + opaque = false; + if (uncertain || !["false", "true"].includes(operation.fallback)) { + state = { + status: "unresolved", + reason: "conditional-or-uncertain-setup", + }; + } else if ( + operation.ref !== APPROVED_RELEASE_SHA || + (operation.token !== expectedToken && + (!operation.token.includes("${{") || + /^\$\{\{secrets\.[A-Za-z_][\w]*\}\}$/.test(operation.token))) || + (operation.fallback === "true" && context.visibility !== "public") + ) { + state = { status: "gap", reason: "unsupported-setup-configuration" }; + } else if (operation.token !== expectedToken) { + state = { status: "unresolved", reason: "unresolved-setup-token" }; + } else { + state = { + status: operation.fallback === "true" ? "fork-exception" : "covered", + reason: "approved-setup-interval", + }; + } + continue; + } + if (operation.kind === "sfw-teardown") { + const certain = certainTeardown(operation, setup); + state = certain + ? undefined + : { status: "unresolved", reason: "conditional-or-uncertain-teardown" }; + setup = undefined; + opaque = false; + continue; + } + if ( + operation.kind === "js-public-download" || + operation.kind === "yarn-blocked" + ) { + let result; + if (uncertainContext || uncertain || opaque) { + result = { + status: "unresolved", + reason: uncertainContext + ? "install-execution-context" + : opaque + ? "opaque-operation-before-install" + : "uncertain-install-syntax-or-condition", + }; + } else if (operation.registryMutating) { + result = { status: "gap", reason: "registry-override" }; + } else if (operation.kind === "yarn-blocked") { + result = { status: "unresolved", reason: "unsupported-yarn-install" }; + } else if (operation.manager === "pnpm" && corepack === "unresolved") { + result = { status: "unresolved", reason: "uncertain-corepack-shims" }; + } else if ( + operation.corepack || + (operation.manager === "pnpm" && corepack === "enabled") + ) { + result = { + status: "gap", + reason: "corepack-download-bypasses-registry", + }; + } else if (!state) { + result = { status: "gap", reason: "no-active-setup" }; + } else if (state.status === "unresolved" || state.status === "gap") { + result = state; + } else if ( + operation.manager === "bun" && + setup?.configureBun !== "true" + ) { + result = { + status: setup?.configureBun === "false" ? "gap" : "unresolved", + reason: "bun-configuration-missing-or-unresolved", + }; + } else { + result = state; + } + downloads.push({ + operation: index + 1, + step: operation.step, + manager: operation.manager, + ...result, + }); + } + if (operation.registryMutating) { + state = uncertain + ? { status: "unresolved", reason: "uncertain-registry-mutation" } + : { status: "gap", reason: "registry-override" }; + setup = undefined; + } + const unknown = operation.kind.startsWith("unknown"); + if (unknown || uncertain) + notes.add( + unknown ? "opaque-execution" : "conditional-or-uncertain-operation", + ); + const validInterval = ["covered", "fork-exception"].includes(state?.status); + const scriptRole = + operation.integrationScript && + validInterval && + !uncertainContext && + !uncertain && + !opaque; + if (installationCapable(operation)) { + potentialInstall = true; + if (!validInterval || uncertainContext || uncertain || opaque) + unresolvedEarlyInstall = true; + } + if (scriptRole) notes.add("package-script-role-not-routing-invalidation"); + if (operation.integrationExecutor) notes.add("executor-code-unverified"); + if ( + (unknown && + !operation.integrationToolchain && + !operation.integrationTransparent && + !scriptRole) || + (uncertain && operation.kind !== "js-public-download") + ) + opaque = true; + } + const statuses = new Set(downloads.map((download) => download.status)); + if (unresolvedEarlyInstall) notes.add("unresolved-install-path-before-setup"); + const disposition = statuses.has("gap") + ? "needs-sfw" + : statuses.has("unresolved") || unresolvedEarlyInstall || malformed + ? "needs-review" + : downloads.length + ? "integrated" + : potentialInstall + ? "needs-review" + : "no-js-ci"; + return { + disposition, + downloads, + notes: [...notes].sort(), + runtimeVerification: "not-performed", + }; +} + +export function integrationDisposition(workflows) { + const dispositions = new Set( + workflows.flatMap((workflow) => [ + ...(workflow.parseError !== undefined ? ["needs-review"] : []), + ...workflow.jobs.map( + (job) => job.integration?.disposition ?? "needs-review", + ), + ]), + ); + for (const disposition of ["needs-sfw", "needs-review", "integrated"]) { + if (dispositions.has(disposition)) return disposition; + } + return workflows.length ? "no-js-ci" : "no-ci"; +} diff --git a/tools/rollout/integration.test.mjs b/tools/rollout/integration.test.mjs new file mode 100644 index 0000000..aa97ac2 --- /dev/null +++ b/tools/rollout/integration.test.mjs @@ -0,0 +1,728 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { classifyJob, classifyWorkflow } from "./classify.mjs"; +import { integrationDisposition } from "./integration.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +import { runAudit } from "./audit.mjs"; + +const setup = { + id: "sfw", + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { + token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}", + "configure-bun": true, + }, +}; +const teardown = { + uses: `workos/setup-socket-firewall/teardown@${APPROVED_RELEASE_SHA}`, +}; +const install = { run: "npm ci" }; +const build = { run: "npm run build" }; +const context = { visibility: "private" }; +function job(steps, properties = {}, overrides = {}) { + return classifyJob( + "fixture", + { steps, ...properties }, + { ...context, ...overrides }, + ["push"], + ); +} +const primary = (...args) => job(...args).integration.disposition; + +test("observed install coverage and opaque execution are separate", () => { + const result = job([setup, install, build]); + assert.equal(result.status, "unknown"); + assert.equal(result.integration.disposition, "integrated"); + assert.equal(result.integration.downloads[0].status, "covered"); + assert.ok(result.integration.notes.includes("opaque-execution")); + assert.equal(primary([install, build]), "needs-sfw"); + assert.equal(primary([setup, install, teardown, build]), "needs-review"); + assert.equal(primary([setup, install, build, install]), "integrated"); + assert.equal(primary([build, setup, install]), "needs-review"); +}); + +test("an integrated sibling cannot hide a missing or unresolved install job", () => { + assert.equal( + integrationDisposition([ + { jobs: [job([setup, install]), job([install, build])] }, + ]), + "needs-sfw", + ); + assert.equal( + integrationDisposition([{ jobs: [job([setup, install]), job([build])] }]), + "needs-review", + ); + assert.equal(primary([{ uses: "./unresolved" }]), "needs-review"); + assert.equal( + primary([ + { run: "go build ./..." }, + { uses: "actions/upload-artifact@v4" }, + ]), + "no-js-ci", + ); + assert.equal(primary([{ run: "git status" }]), "no-js-ci"); +}); + +test("supported toolchain roles preserve intervals without synthesizing setup", () => { + for (const tool of [ + { uses: "pnpm/action-setup@v4" }, + { + uses: "pnpm/action-setup@v4", + with: { version: "10", run_install: false }, + }, + { uses: "oven-sh/setup-bun@v2", with: { "bun-version": "1.3.14" } }, + ]) { + assert.equal(primary([setup, tool, install, build]), "integrated"); + assert.equal(primary([tool, install, build]), "needs-sfw"); + assert.equal(job([setup, tool, install]).status, "unknown"); + } + for (const tool of [ + { uses: "pnpm/action-setup@v4", with: { run_install: true } }, + { + uses: "pnpm/action-setup@v4", + with: { run_install: "${{ inputs.install }}" }, + }, + { uses: "pnpm/action-setup@v4", with: { custom: "anything" } }, + { uses: "pnpm/action-setup/other@v4" }, + { uses: "untrusted/action-setup@v4" }, + { + uses: "oven-sh/setup-bun@v2", + with: { "bun-download-url": "https://example.invalid/bun" }, + }, + { uses: "oven-sh/setup-bun/other@v2" }, + ]) + assert.equal(primary([setup, tool, install]), "needs-review"); +}); + +test("job reachability and harmless context do not alter install ordering", () => { + assert.equal( + primary([setup, install], { + if: "github.ref == 'refs/heads/main'", + env: { CI: true }, + defaults: { run: { shell: "bash" } }, + }), + "integrated", + ); + assert.equal(primary([setup, install], { if: false }), "no-js-ci"); + const workflow = classifyWorkflow( + "on: push\ndefaults:\n run:\n shell: bash\nenv:\n CI: true\njobs:\n test:\n steps:\n - uses: " + + setup.uses + + "\n with:\n token: '${{ secrets.SOCKET_FIREWALL_TOKEN }}'\n - run: npm ci\n", + { ...context, path: ".github/workflows/ci.yml" }, + ); + assert.equal(integrationDisposition([workflow]), "integrated"); + assert.equal( + primary([setup, { ...install, env: { NODE_ENV: "production" } }]), + "integrated", + ); + for (const env of [ + { NPM_CONFIG_REGISTRY: "https://example.invalid" }, + { PATH: "/custom" }, + { HOME: "/custom" }, + { BASH_ENV: "startup.sh" }, + { "${{ inputs.key }}": "value" }, + { UNMODELED: {} }, + ]) { + assert.equal(primary([setup, install], { env }), "needs-review"); + assert.equal(primary([setup, { ...install, env }]), "needs-review"); + } + assert.equal( + primary([setup, install], { defaults: { run: { shell: "pwsh" } } }), + "needs-review", + ); +}); + +test("public fallback is recorded as an exception, not missing integration", () => { + const fallback = { + ...setup, + with: { + token: "${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }}", + "allow-external-fork-fallback": true, + }, + }; + const result = job([fallback, install, build], {}, { visibility: "public" }); + assert.equal(result.integration.disposition, "integrated"); + assert.equal(result.integration.downloads[0].status, "fork-exception"); + assert.equal(result.status, "unknown"); + assert.equal( + primary([ + { + ...setup, + with: { ...setup.with, "allow-external-fork-fallback": true }, + }, + install, + ]), + "needs-sfw", + ); + assert.equal( + primary([ + { + ...setup, + with: { + ...setup.with, + "allow-external-fork-fallback": "${{ inputs.fallback }}", + }, + }, + install, + ]), + "needs-review", + ); +}); + +test("per-download negative controls remain actionable or unresolved", () => { + for (const steps of [ + [{ ...setup, if: false }, install, build], + [install, setup, install], + [setup, install, teardown, install, build], + [{ ...setup, uses: "workos/setup-socket-firewall@v1" }, install], + [{ ...setup, with: { token: "wrong" } }, install], + [ + { ...setup, with: { ...setup.with, "configure-bun": false } }, + { run: "bun install" }, + ], + [setup, { run: "npm ci --reg=https://example.invalid" }], + [{ run: "corepack enable" }, setup, { run: "pnpm install" }], + ]) + assert.equal(primary(steps), "needs-sfw"); + for (const steps of [ + [{ ...setup, if: "github.ref == 'refs/heads/main'" }, install], + [{ ...setup, "continue-on-error": true }, install], + [setup, { run: "npm ci --future-option=unknown" }], + [setup, { run: "npm --prefix help ci" }], + [setup, { run: "cat <<'EOF'\nnpm ci\nEOF" }], + [setup, { uses: "./unresolved" }, install], + [setup, { run: "corepack enable pnpm" }, { run: "pnpm install" }], + ]) + assert.equal(primary(steps), "needs-review"); +}); + +test("teardown guards establish only the appropriate success-path boundary", () => { + for (const condition of [ + "always()", + "${{ always() && steps.sfw.outputs.active == 'true' }}", + ]) { + const clean = job([ + setup, + install, + { ...teardown, if: condition }, + { run: "npm publish" }, + ]); + assert.equal(clean.integration.disposition, "integrated"); + assert.equal(clean.status, "unknown"); + assert.ok( + !clean.violations.some((v) => v.includes("without a same-SHA teardown")), + ); + assert.equal( + primary([setup, install, { ...teardown, if: condition }, install]), + "needs-sfw", + ); + assert.notEqual( + job([ + setup, + install, + { ...teardown, if: condition }, + { run: "npm publish", if: "always()" }, + ]).status, + "protected", + ); + } + for (const boundary of [ + { ...teardown, if: "always() && steps.other.outputs.active == 'true'" }, + { ...teardown, if: "always()", "continue-on-error": true }, + { ...teardown, if: "inputs.cleanup" }, + ]) { + assert.equal( + job([setup, install, boundary, { run: "npm publish" }]).status, + "unsafe-publish", + ); + assert.equal(primary([setup, install, boundary, install]), "needs-review"); + } +}); + +test("audit v2 separates primary integration counts from assurance and errors", async () => { + const workflow = `on: push\njobs:\n build:\n steps:\n - uses: ${setup.uses}\n with: { token: '${setup.with.token}' }\n - run: npm ci\n - run: npm test\n`; + const client = { + listRestRepositories: async () => + ["app", "broken"].map((name) => ({ + name, + default_branch: "main", + archived: false, + visibility: "private", + })), + listGraphqlRepositories: async () => + ["app", "broken"].map((name) => ({ + name, + isArchived: false, + visibility: "private", + })), + getRef: async () => ({ object: { sha: "a".repeat(40) } }), + getTree: async (repo) => ({ + truncated: repo.endsWith("broken"), + tree: [ + { path: ".github/workflows/ci.yml", type: "blob", mode: "100644" }, + ], + }), + getText: async () => workflow, + }; + const report = await runAudit(client); + assert.equal(report.schemaVersion, 2); + assert.deepEqual(report.dispositions, { "audit-error": 1, integrated: 1 }); + assert.deepEqual(report.assuranceDispositions, { + "audit-error": 1, + "needs-review": 1, + }); + assert.equal(report.scanStatus, "partial"); + assert.equal(report.scanErrors, 1); + assert.equal(report.runtimeVerification, "not-performed"); + assert.equal( + integrationDisposition([ + classifyWorkflow("on: [", { ...context, path: "fixture" }), + ]), + "needs-review", + ); +}); + +test("malformed steps cannot become no-js-ci or integrated", () => { + for (const steps of [ + undefined, + null, + [], + [null], + [{}], + [setup, install, {}], + ]) { + assert.equal(job(steps).integration.disposition, "needs-review"); + } + assert.equal( + integrationDisposition([ + classifyWorkflow("on: push\njobs:\n bad: false\n", { + visibility: "private", + }), + ]), + "needs-review", + ); +}); + +test("installation-capable ecosystem and script executors stay review candidates", () => { + for (const run of [ + "uv run npm ci", + "poetry run npm ci", + "go run bootstrap.go", + "python scripts/bootstrap.py", + "CI=1 make bootstrap", + "CI=1 node scripts/bootstrap.mjs", + "CI=1 ./scripts/bootstrap.sh", + "${{ inputs.command }}", + "command npm ci", + "npm --prefix help ci", + ]) { + assert.equal(job([{ run }]).integration.disposition, "needs-review", run); + assert.equal( + integrationDisposition([ + { jobs: [job([setup, install]), job([{ run }])] }, + ]), + "needs-review", + run, + ); + } + assert.equal( + job([], { + uses: "example/shared/.github/workflows/build.yml@main", + if: false, + }).integration.disposition, + "no-js-ci", + ); +}); + +test("dynamic Bun configuration and uncertain local boundaries stay unresolved", () => { + assert.equal( + job([ + { + ...setup, + with: { ...setup.with, "configure-bun": "${{ inputs.bun }}" }, + }, + { run: "bun install" }, + ]).integration.disposition, + "needs-review", + ); + const localActions = new Map([ + [ + "inner/action.yml", + `runs:\n using: composite\n steps:\n - uses: ${setup.uses}\n with: { token: '${setup.with.token}' }\n - run: npm ci\n shell: bash\n`, + ], + ]); + const nested = classifyJob( + "fixture", + { steps: [{ uses: "./inner", if: "inputs.install" }] }, + { visibility: "private", localActions }, + ["push"], + ); + assert.equal(nested.integration.disposition, "needs-review"); + assert.equal( + job([setup, { uses: "pnpm/action-setup" }, install]).integration + .disposition, + "needs-review", + ); +}); + +test("active-output teardown does not confuse composite-local step IDs with job IDs", () => { + const localActions = new Map([ + [ + "inner/action.yml", + `runs:\n using: composite\n steps:\n - uses: ${setup.uses}\n id: sfw\n with: { token: '${setup.with.token}' }\n - run: npm ci\n shell: bash\n`, + ], + ]); + const result = classifyJob( + "fixture", + { + steps: [ + { uses: "./inner" }, + { ...teardown, if: "always() && steps.sfw.outputs.active == 'true'" }, + { run: "npm publish" }, + ], + }, + { visibility: "private", localActions }, + ["push"], + ); + assert.equal(result.status, "unsafe-publish"); + assert.equal(primary([{ run: "git diff file.js" }]), "no-js-ci"); +}); + +test("Corepack uncertainty survives setup and conditional or targeted controls", () => { + for (const steps of [ + [ + { run: "corepack enable" }, + { run: "corepack disable", if: "inputs.disable_corepack" }, + ], + [{ run: "corepack enable", if: "inputs.enable_corepack" }], + [{ run: "corepack enable pnpm", if: "inputs.enable_corepack" }], + [{ run: "corepack enable pnpm" }], + [ + { run: "corepack enable" }, + { run: "corepack disable yarn", if: "inputs.disable_corepack" }, + ], + ]) { + const result = job([...steps, setup, { run: "pnpm install" }]); + assert.equal(result.integration.disposition, "needs-review"); + assert.equal(result.integration.downloads[0].status, "unresolved"); + } + assert.equal( + primary([ + { run: "corepack enable", if: "inputs.enable_corepack" }, + { run: "corepack disable" }, + setup, + { run: "pnpm install" }, + ]), + "integrated", + ); + assert.equal( + primary([ + { run: "corepack enable pnpm", if: "inputs.enable_corepack" }, + { run: "corepack disable" }, + setup, + { run: "pnpm install" }, + ]), + "integrated", + ); + assert.equal( + primary([{ run: "corepack enable" }, setup, { run: "pnpm install" }]), + "needs-sfw", + ); +}); + +test("scripts under absent, unresolved or invalid setup cannot be erased by later setup", () => { + for (const before of [ + [], + [{ ...setup, if: "inputs.enable_sfw" }], + [{ ...setup, with: { token: "wrong" } }], + [setup, install, teardown], + ]) { + const result = job([ + ...before, + { run: "npm run bootstrap" }, + setup, + install, + ]); + assert.equal(result.integration.disposition, "needs-review"); + assert.ok( + result.integration.notes.includes("unresolved-install-path-before-setup"), + ); + } +}); + +test("malformed local source remains review even after a covered install", () => { + for (const text of [ + "runs: [", + "runs: false", + "runs: { using: composite, steps: false }", + "false", + ]) { + const result = job( + [setup, install, { uses: "./broken" }], + {}, + { + localActions: new Map([["broken/action.yml", text]]), + }, + ); + assert.equal(result.integration.disposition, "needs-review"); + assert.ok(result.operations.some((operation) => operation.sourceError)); + assert.equal(result.integration.downloads[0].status, "covered"); + } +}); + +test("inline environment and unsupported env wrappers retain installer candidates", () => { + for (const run of [ + "HOME=/tmp npm ci", + "PATH=/tmp npm ci", + "NODE_OPTIONS=--require=./bootstrap.cjs npm ci", + "env HOME=/tmp npm ci", + "env -i npm ci", + "env --ignore-environment node scripts/bootstrap.mjs", + "env --unset=HOME npm ci", + "1BAD=x npm ci", + "sudo npm ci", + "time npm ci", + "exec npm ci", + ]) { + assert.equal(primary([setup, { run }]), "needs-review", run); + assert.equal(primary([{ run }]), "needs-review", run); + assert.equal( + integrationDisposition([ + { jobs: [job([setup, install]), job([{ run }])] }, + ]), + "needs-review", + run, + ); + } + for (const run of ["CI=1 npm ci", "env CI=1 npm ci"]) { + assert.equal(primary([setup, { run }]), "integrated", run); + assert.equal(primary([{ run }]), "needs-sfw", run); + assert.notEqual(job([setup, { run }]).status, "protected"); + } + assert.equal( + primary([{ run: "git status" }, { uses: "actions/upload-artifact@v4" }]), + "no-js-ci", + ); +}); + +test("only bounded literal logging and safe shell prologues preserve direct installs", () => { + for (const prefix of [ + "set -e", + "set -euo pipefail", + 'echo "Installing dependencies"', + "echo 'Installing dependencies'", + 'echo "source: npm ci; npm ci"', + 'echo "--registry=https://example.invalid"', + ]) { + const run = `${prefix}\nnpm ci`; + assert.equal(primary([setup, { run }]), "integrated", run); + assert.equal(primary([{ run }]), "needs-sfw", run); + assert.notEqual(job([setup, { run }]).status, "protected"); + } + for (const run of [ + 'echo "$(node bootstrap.mjs)"\nnpm ci', + "source ./env.sh\nnpm ci", + "cat <<'EOF'\nnpm ci\nEOF", + "set +e\nnpm ci", + "set -o posix\nnpm ci", + 'echo "Installing" > .npmrc\nnpm ci', + "echo `node bootstrap.mjs`\nnpm ci", + 'echo "literal source: npm ci"', + 'echo "source: npm ci; npm ci"', + "echo 'source: npm ci && npm ci'", + ]) { + assert.notEqual(primary([setup, { run }]), "integrated", run); + assert.notEqual(primary([{ run }]), "needs-sfw", run); + } +}); + +test("ordinary package script roles preserve observed setup, never assure script code", () => { + for (const run of [ + "npm run build", + "npm test", + "npm test -- --runInBand", + "pnpm run lint", + "bun run build", + "yarn test", + ]) { + const result = job([ + setup, + { run }, + { run: "npx wrangler deploy --env production" }, + ]); + assert.equal(result.integration.disposition, "integrated", run); + assert.equal(result.status, "unknown", run); + assert.ok( + result.integration.notes.includes( + "package-script-role-not-routing-invalidation", + ), + ); + assert.equal(primary([setup, { run }]), "needs-review", run); + assert.equal(primary([{ run }, setup, install]), "needs-review", run); + } + for (const run of [ + "node scripts/build.mjs", + "npm --prefix scripts run build", + "source ./env.sh", + "npm run $TASK", + "npm run build --script-shell=./bootstrap.sh", + "npm run build --prefix=/tmp", + "npm run build --registry=https://example.invalid", + ]) { + assert.notEqual(primary([setup, { run }, install]), "integrated", run); + } + assert.equal(primary([setup, build, teardown, install]), "needs-sfw"); + assert.equal( + primary([ + setup, + build, + { run: "npm config set registry https://example.invalid" }, + install, + ]), + "needs-review", + ); +}); + +test("direct executor target arguments are not installer configuration", () => { + for (const run of [ + "npx wrangler deploy --env production", + "npx -y wrangler deploy --env production", + "bunx biome check --write", + "npx tool --registry=https://example.invalid", + ]) { + const result = job([setup, { run }]); + assert.equal(result.integration.disposition, "integrated", run); + assert.equal(result.status, "unknown", run); + assert.ok(result.integration.notes.includes("executor-code-unverified")); + assert.equal(primary([{ run }]), "needs-sfw", run); + } + for (const run of [ + "npx --registry=https://example.invalid wrangler", + "bunx --reg=https://example.invalid tool", + "npx --@scope:registry=https://example.invalid tool", + ]) { + assert.equal(primary([setup, { run }]), "needs-sfw", run); + } + for (const run of [ + "npx --future-option tool", + "npx --package=tool tool", + "npx $TOOL --env production", + "npx tool $(node bootstrap.mjs)", + "npx", + "npm exec tool -- --env production", + "npm exec tool", + "env -i npx tool", + "npx tool --env $ENVIRONMENT", + ]) + assert.equal(primary([setup, { run }]), "needs-review", run); +}); + +test("application environment and sibling services do not change observed host routing", () => { + const env = { + APP_TOKEN: "${{ secrets.APP_TOKEN }}", + CI: "${{ inputs.ci }}", + DEPLOY_ENV: "production", + }; + assert.equal(primary([setup, install], { env }), "integrated"); + assert.equal(primary([setup, { ...install, env }]), "integrated"); + assert.equal(job([setup, install], { env }).status, "unknown"); + const workflow = classifyWorkflow( + `on: push\nenv:\n APP_TOKEN: '${env.APP_TOKEN}'\njobs:\n test:\n steps:\n - uses: ${setup.uses}\n with: { token: '${setup.with.token}' }\n - run: npm ci\n`, + context, + ); + assert.equal(integrationDisposition([workflow]), "integrated"); + for (const key of [ + "npm_config_registry", + "PNPM_HOME", + "BUN_CONFIG", + "YARN_RC_FILENAME", + "HOME", + "USERPROFILE", + "XDG_CONFIG_HOME", + "APPDATA", + "LOCALAPPDATA", + "PATH", + "NODE_OPTIONS", + "NODE_PATH", + "BASH_ENV", + "ENV", + "SHELL", + "LD_PRELOAD", + "DYLD_INSERT_LIBRARIES", + "HTTPS_PROXY", + "NODE_EXTRA_CA_CERTS", + "bad-key", + ]) { + assert.equal( + primary([setup, install], { env: { [key]: "${{ secrets.VALUE }}" } }), + "needs-review", + key, + ); + } + for (const env of ["${{ inputs.environment }}", [], null, { APP_TOKEN: {} }]) + assert.equal(primary([setup, install], { env }), "needs-review"); + const services = { database: { image: "postgres:17" } }; + assert.equal(primary([setup, install], { services }), "integrated"); + assert.equal(primary([install], { services }), "needs-sfw"); + assert.equal(job([setup, install], { services }).status, "unknown"); + assert.equal( + primary([setup, install], { container: "node:22" }), + "needs-review", + ); +}); + +test("unresolved setup token forwarding is not evidence of a wrong token", () => { + for (const token of [ + "${{ inputs.token }}", + "${{ secrets[inputs.token_name] }}", + ]) { + const result = job([{ ...setup, with: { token } }, install]); + assert.equal(result.integration.disposition, "needs-review"); + assert.equal( + result.integration.downloads[0].reason, + "unresolved-setup-token", + ); + assert.notEqual(result.status, "protected"); + } + for (const token of [ + "wrong", + "", + "${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }}", + ]) + assert.equal( + primary([{ ...setup, with: { token } }, install]), + "needs-sfw", + ); + assert.equal(primary([install]), "needs-sfw"); +}); + +test("exact npm lockfile host replacement preserves the configured registry", () => { + assert.equal( + primary([setup, { run: "npm ci --replace-registry-host=always" }]), + "integrated", + ); + assert.equal( + primary([{ run: "npm ci --replace-registry-host=always" }]), + "needs-sfw", + ); + for (const option of [ + "--registry=https://example.invalid", + "--reg=https://example.invalid", + "--@scope:registry=https://example.invalid", + ]) + assert.equal( + primary([ + setup, + { run: `npm ci --replace-registry-host=always ${option}` }, + ]), + "needs-sfw", + ); + for (const run of [ + "npm ci --replace-registry-host=never", + "npm ci --replace-registry-host=npmjs", + "npm ci --replace-registry-host=example.invalid", + "npm ci --replace-registry-host always", + "pnpm install --replace-registry-host=always", + ]) + assert.equal(primary([setup, { run }]), "needs-review", run); +}); diff --git a/tools/rollout/primary-boundaries.test.mjs b/tools/rollout/primary-boundaries.test.mjs new file mode 100644 index 0000000..9a451c2 --- /dev/null +++ b/tools/rollout/primary-boundaries.test.mjs @@ -0,0 +1,164 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { classifyJob } from "./classify.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}" }, +}; +const install = { run: "npm ci" }; +const context = { visibility: "private" }; +const inspect = (steps, extra = {}) => + classifyJob("fixture", { steps }, { ...context, ...extra }, ["push"]); + +test("ordinary run guards preserve observed ordering, not boundary reachability", () => { + for (const guard of [ + "github.ref == 'refs/heads/main'", + "${{ github.event_name == 'push' }}", + ]) { + assert.equal( + inspect([setup, { ...install, if: guard }]).integration.disposition, + "integrated", + ); + assert.equal( + inspect([{ ...install, if: guard }]).integration.disposition, + "needs-sfw", + ); + const row = inspect([ + setup, + install, + { run: "npm run deploy", if: guard }, + { run: "npx wrangler deploy --env production", if: guard }, + ]); + assert.equal(row.integration.disposition, "integrated"); + assert.equal(row.status, "unknown"); + assert.equal( + inspect([{ ...setup, if: guard }, install]).integration.disposition, + "needs-review", + ); + } + for (const guard of [ + "always()", + "Always()", + "failure()", + "!cancelled()", + "success() || true", + ]) { + assert.equal( + inspect([setup, { ...install, if: guard }]).integration.disposition, + "needs-review", + ); + } + const localActions = new Map([ + [ + "configure/action.yml", + `runs:\n using: composite\n steps:\n - uses: ${setup.uses}\n with:\n token: '${setup.with.token}'\n`, + ], + ]); + assert.equal( + inspect( + [{ uses: "./configure", if: "github.ref == 'refs/heads/main'" }, install], + { localActions }, + ).integration.disposition, + "needs-review", + ); +}); + +test("toolchain version metadata is distinct from package-manager configuration", () => { + const steps = [setup, install]; + assert.equal( + classifyJob( + "fixture", + { steps, env: { BUN_VERSION: "1.3.14", PNPM_VERSION: "9.15.9" } }, + context, + ["push"], + ).integration.disposition, + "integrated", + ); + for (const env of [ + { BUN_INSTALL: "/tmp/other" }, + { PNPM_CONFIG_REGISTRY: "https://registry.example.invalid" }, + ]) { + assert.equal( + classifyJob("fixture", { steps, env }, context, ["push"]).integration + .disposition, + "needs-review", + ); + } +}); + +test("GitHub expressions are not literal shell logging", () => { + for (const run of [ + "echo '${{ inputs.payload }}'\nnpm ci", + 'echo "${{ inputs.payload }}"\nnpm ci', + ]) { + assert.equal( + inspect([setup, { run }]).integration.disposition, + "needs-review", + ); + assert.equal(inspect([{ run }]).integration.disposition, "needs-review"); + } + assert.equal( + inspect([setup, { run: "echo '$SHELL_LITERAL'\nnpm ci" }]).integration + .disposition, + "integrated", + ); +}); + +test("exhausted local expansion retains covered evidence but requires review", () => { + const steps = [ + ...Array.from({ length: 997 }, () => ({ run: "echo ok", shell: "bash" })), + { run: "npm ci --registry=https://example.invalid", shell: "bash" }, + ]; + const localActions = new Map([ + [ + "large/action.yml", + JSON.stringify({ runs: { using: "composite", steps } }), + ], + ]); + const result = inspect([setup, install, { uses: "./large" }], { + localActions, + }); + assert.equal( + result.operations.at(-1).sourceError, + "unresolved-local-action-expansion", + ); + assert.equal(result.integration.downloads[0].status, "covered"); + assert.equal(result.integration.disposition, "needs-review"); +}); + +test("missing, cyclic and malformed local/action source stays visible after install", () => { + assert.equal( + inspect([setup, install, { uses: "./missing" }]).integration.disposition, + "needs-review", + ); + const localActions = new Map([ + [ + "cycle/action.yml", + "runs:\n using: composite\n steps:\n - uses: ./cycle\n", + ], + ]); + assert.equal( + inspect([setup, install, { uses: "./cycle" }], { localActions }).integration + .disposition, + "needs-review", + ); + for (const value of [{ nested: true }, ["not", "a", "scalar"]]) { + assert.equal( + inspect([ + setup, + { uses: "pnpm/action-setup@v4", with: { version: value } }, + install, + ]).integration.disposition, + "needs-review", + ); + assert.equal( + inspect([ + setup, + install, + { uses: "actions/upload-artifact@v4", with: { path: value } }, + ]).integration.disposition, + "needs-review", + ); + } +}); From 0518a0575bbd332276c3110d175d9d9b5663c0d7 Mon Sep 17 00:00:00 2001 From: Matt Peake Date: Mon, 14 Sep 2026 17:01:52 -0400 Subject: [PATCH 07/19] Fix gap classification to track observed SFW configuration --- CONTRIBUTING.md | 2 +- README.md | 22 +- tools/rollout/audit.mjs | 24 +- tools/rollout/classify.mjs | 465 +++++++++++++++--- tools/rollout/classify.test.mjs | 2 +- tools/rollout/commands.mjs | 145 ++++++ tools/rollout/configuration-presence.test.mjs | 356 ++++++++++++++ .../configuration-regressions.test.mjs | 348 +++++++++++++ tools/rollout/integration.mjs | 326 +++++++----- tools/rollout/integration.test.mjs | 88 ++-- tools/rollout/primary-boundaries.test.mjs | 7 +- 11 files changed, 1533 insertions(+), 252 deletions(-) create mode 100644 tools/rollout/commands.mjs create mode 100644 tools/rollout/configuration-presence.test.mjs create mode 100644 tools/rollout/configuration-regressions.test.mjs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 57afe39..daf54da 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,7 +42,7 @@ The minimum test-coverage policy is one shell test suite for every executable sh `npm run inventory` and `npm run audit:live` are read-only manual commands using an existing authorized `gh` session. Confirm organization-wide read access separately before making organization-wide claims. No scheduling or remediation is implicit. See README for the scan scope and limitations. -Audit schema2 separates primary observed-install `dispositions` from strict `assuranceDispositions`; retain both views in regressions. Test that bounded ordinary package-script roles under valid setup do not erase observed configuration, while scripts before valid setup, under invalid/unresolved setup or in script-only sibling jobs remain visible. Cover persistent conditional Corepack state, malformed local source after covered installs, inline/YAML environment scope, unsupported env wrappers, literal logging/prologues versus shell source/substitutions, direct executor argument boundaries and exact npm lockfile-host replacement versus actual registry overrides. `integrated` is not runtime proof; strict opacity and security findings remain separate. Do not introduce a historical cohort, coverage percentage or repository allowlist. +Audit schema2 separates primary observed configuration from strict `assuranceDispositions`. Acceptance tests must show that arbitrary execution before or after setup, pipelines, package scripts and script-only siblings do not invent missing installs or erase configuration. An actual uncovered install, explicit configuration conflict or unresolved relevant source must remain visible. Cover conditional Corepack state, scoped environment/registry changes, quoted data versus executable commands, executor payload boundaries, composite input binding and snapshot-bound reusable-workflow references. Preserve strict security/opacity diagnostics and source acquisition limits. `integrated` is configuration evidence, not runtime proof. `no-js-ci` is absence of an observed install path, not assurance about script bodies. Do not introduce a historical cohort, coverage percentage or repository allowlist. Full results are owner-only, ignored `reports/inventory.json` and `reports/live-audit.json`; terminal JSON contains sanitized counts. Audit `scanErrors`/`scanStatus` distinguish operational failures from discovered gaps. Exit 1 includes partial scans; exit 0 can still include `needs-sfw` or `needs-review`. Do not force-add reports, expose inventory/source in logs, or upload full reports as artifacts. diff --git a/README.md b/README.md index 0683a05..97626d9 100644 --- a/README.md +++ b/README.md @@ -214,35 +214,33 @@ npm run audit:live # read-only, default-branch workflow candidate scan Terminal output is sanitized JSON counts, not repository names or workflow source. Full inventory and audit JSON go to `reports/inventory.json` and `reports/live-audit.json`, respectively. These paths are ignored; reports are atomically replaced with owner-only (`0600`) permissions. Keep them private: never force-add, publish, or upload reports as CI artifacts. Both commands take no CLI flags; a later run replaces the previous report. -Audit JSON schema **2** separates primary `dispositions` (observed dependency-install integration) from `assuranceDispositions` (strict execution/security review). Successfully inspected repositories retain both dispositions, their `headSha`, workflow/job operations and violations; read errors remain explicit error rows. Each job's `integration` includes per-download `covered`, `fork-exception`, `gap` or `unresolved` evidence, reason codes and diagnostic notes. `runtimeVerification` is always `not-performed`. `scanStatus: complete` means the token-visible scan completed, **not** that all jobs are protected. `partial` means one or more repositories have an `audit-error` row. CLI exit **0** means the scan completed, even when gaps/review candidates were found; exit **1** means an operational failure (including partial scans). Consumers should inspect dispositions separately from the exit code. Fatal inventory/API/command errors emit a sanitized error and exit 1; they do not refresh the report, so check its timestamp before use. +Audit JSON schema **2** separates primary `dispositions` (observed dependency-install integration) from `assuranceDispositions` (strict execution/security review). Successfully inspected repositories retain both dispositions, their `headSha`, workflow/job operations and violations; read errors remain explicit error rows. Each job's `integration` includes per-download `covered`, `fork-exception`, `gap` or `unresolved` configuration evidence, reason codes and diagnostic notes. `additionalJsPaths` records explicit JS invocations outside the direct-install grammar, distinguishing `setup-observed` from unresolved configuration. `runtimeVerification` is always `not-performed`. `scanStatus: complete` means the token-visible scan completed, **not** that all jobs are protected. `partial` means one or more repositories have an `audit-error` row. CLI exit **0** means the scan completed, even when gaps/review candidates were found; exit **1** means an operational failure (including partial scans). Consumers should inspect dispositions separately from the exit code. Fatal inventory/API/command errors emit a sanitized error and exit 1; they do not refresh the report, so check its timestamp before use. | Repository disposition | Meaning | | --- | --- | | `needs-sfw` | A recognized download has a definite missing/unsupported setup interval in its own job. | -| `needs-review` | An observed install interval, script-only installation-capable job, reusable call or malformed source needs investigation. | -| `integrated` | Recognized downloads have supported static setup intervals, possibly with an explicit public-fork exception; opaque execution remains unverified. | +| `needs-review` | Install configuration, an explicit JS invocation, or relevant workflow/local-action source could not be resolved. Ordinary opaque scripts alone do not cause this result. | +| `integrated` | SFW configuration is observed for the identified install paths, possibly with an explicit public-fork exception. This is not execution or traffic assurance. | | `audit-error` | A repository read failed or was incomplete; never a clean result. | | `no-ci`, `no-js-ci`, `empty` | No observed in-scope download candidate; not proof that remote actions or arbitrary code cannot download packages. | -Primary integration is not a rollout percentage or a certification of every command. For example, setup → `npm ci` → `npm test` can be `integrated` while strict assurance remains `needs-review`. Without setup, the same direct install remains a `needs-sfw` candidate even though the later test is opaque. A new uncovered install job cannot be masked by an integrated sibling; a script-only installation-capable sibling remains `needs-review`. +Primary integration is not a rollout percentage or a certification of every command. For example, setup → `npm ci` → `npm test` can be `integrated` while strict assurance remains `needs-review`. Without setup, the same direct install remains a `needs-sfw` candidate even though the later test is opaque. A new uncovered install job cannot be masked by an integrated sibling. An ordinary script-only sibling does not invent another install path: its code remains unverified in assurance diagnostics. Strict job `status` and `assuranceDisposition` retain opaque execution and security findings (`blocked-trust`, `unsafe-publish`, `blocked-yarn`, etc.). **Inspect those findings separately; `integrated` does not dismiss them.** A strict `protected` or `safe-publish` status remains static evidence, not runtime or lifecycle-hook proof. Schema1 consumers must not interpret schema2 primary dispositions as the old assurance verdicts. ### Static limits -The small grammar recognizes direct npm/pnpm/Bun downloads, Yarn blockers, setup/teardown order and registry overrides. Each recognized download is evaluated in its own job and interval, using the approved SHA and visibility-appropriate token expression. Disabled setup provides no integration evidence; uncertain setup/install conditions, complex shell constructs and actual job containers remain unresolved. Sibling service containers alone do not invalidate host setup ordering. A job-level guard affects shared reachability rather than relative setup ordering. Ordinary run-step guards retain GitHub's implicit success gating; explicit status-function guards, conditional setup/teardown and conditional Corepack transitions remain conservative. Unresolved setup token expressions (including composite input forwarding) require review, not a claim of a wrong token; literal wrong visibility-specific secrets remain gaps. +The primary question is **whether SFW is configured for identified dependency-install paths**, not whether every command can be proved to preserve protection. Arbitrary scripts, remote actions, `curl`/shell bootstraps and package-script bodies remain unverified; they do not erase or synthesize observed setup. A script before setup or after teardown is not presumed to install dependencies. An explicit install in that position is still a gap. `no-js-ci` means no identified dependency-install path in the supported workflow source—not that the repository contains no JavaScript or that its scripts cannot install packages. -Primary environment checks cover package-manager configuration prefixes, home/config paths, executable selection, shell startup, loader and proxy controls. Ordinary mapping keys for application credentials/settings and toolchain-version metadata do not alone invalidate observed configuration, including secret-valued expressions. Dynamic mappings, malformed keys and nonscalar values remain unresolved. The same rule applies to inline assignments: `CI=1 npm ci` and `env CI=1 npm ci` are supported; HOME/PATH/NODE_OPTIONS overrides and option-bearing `env` wrappers require review. Strict assurance retains blanket environment uncertainty. Literal Bash/sh defaults, exact `set -e` / `set -euo pipefail` prologues and literal `echo` logging do not alone invalidate primary evidence. Substitutions, imports, heredocs, redirections and unsupported shell switches are not interpreted. +The small grammar recognizes npm/pnpm/Bun installs, download-capable executors, Yarn blockers and setup/teardown ordering. A lexical boundary scanner keeps quoted data and command substitutions inside their containing command rather than inventing installs from text. Pipelines and executor payload arguments do not invalidate configuration simply because their execution is opaque. For `npx`/`bunx`, a literal target after optional `-y`/`--yes` separates installer options from program arguments, including quoted or variable-valued payloads. Pre-target registry overrides remain gaps; unsupported installer options, dynamic installer targets and explicit context changes can still require review. Uninterpreted shell structure can establish configuration presence without proving whether a particular command executes; it cannot establish a definite missing install when the command may only be data. -Opaque scripts/actions remain strict-assurance diagnostics. Primary integration recognizes a bounded ordinary package-script role (`npm`/`pnpm`/Bun/Yarn `run` with a literal script name, or `test`/`start`/`stop`/`restart`, with literal arguments; option arguments require a `--` separator). After a valid setup interval this role is not itself an observed routing invalidation, so setup → `npm test` → `npx wrangler deploy --env production` can be `integrated`. This does **not** declare scripts safe or no-network. Script-only jobs, execution before valid setup or under unresolved/invalid setup, explicit configuration changes and teardown remain visible; a later valid install cannot hide an earlier unresolved installation-capable path. Other opaque execution before a later install can make its routing unresolved; a fresh explicit setup can re-establish configuration, but cannot erase earlier unresolved execution. +Configuration checks retain the approved action SHA, visibility-appropriate token, Bun configuration and public-fork exception rules. Disabled, conditional or continued-on-error setup cannot silently cover an install. Corepack uncertainty survives conditional or targeted controls. The action-exported `SFW_BUN_CONFIG_PATH` is recognized in Bun's `--config` argument when Bun setup is enabled; explicit overrides of that variable remain configuration questions. Whole-value composite input references in `with`/`env` are bound to caller values/defaults without evaluating conditions or interpolating shell text. Missing/malformed local source, cycles and expansion limits remain review findings. -For direct `npx`/`bunx` only, a literal target (optionally preceded by `-y`/`--yes`) separates installer options from literal command arguments. Post-target `--env` or `--registry` belongs to the invoked command, not the installer's routing configuration; invoked code remains unverified. Pre-target registry overrides remain gaps; unsupported pre-target options, dynamic targets/arguments and npm-exec remain unresolved. Exact npm `--replace-registry-host=always` preserves the configured registry and is supported; other replacement modes require review. Registry/scoped/abbreviated overrides are not exempted. Corepack shim uncertainty persists across conditional/targeted controls and setup; a certain unqualified disable can reset it. +Known registry writes and teardown invalidate the relevant configuration; a command-local override does not contaminate later installs. npm/pnpm registry setters, configuration-file writes (including `tee`), standalone relevant assignments and GitHub environment-file writes remain findings rather than being treated as generic opacity. Reading a configuration variable is not a mutation. HOME/PATH/NODE_OPTIONS overrides, relevant exported variables within a run block, unsupported environment wrappers, dynamic mappings and job containers remain configuration questions. Ordinary application secrets/settings, version metadata and sibling service containers do not invalidate host setup ordering. Job-level guards affect shared reachability; explicit status predicates and conditional configuration transitions remain conservative. Exact npm `--replace-registry-host=always` preserves the configured registry; other modes require review. -Primary integration also makes a narrow toolchain-role assumption: standard `pnpm/action-setup` (no `run_install`, or literal false) and `oven-sh/setup-bun` with supported version inputs do not invalidate routing for subsequent explicit installs. Unsupported inputs and subpaths remain unresolved. This does **not** certify their bootstrap downloads or action implementations, and they never substitute for SFW setup. +Local reusable workflows are already read at the same repository SHA and contribute their actual primary result instead of an automatic unknown. Missing targets and cycles remain reviewable. A remote reusable call can be resolved as having no observed JS install only when that workflow was already captured in the same organization's inventory at the requested default branch or exact SHA. A different pin/tag is never replaced with the current body. Remote callees with installs require caller-specific context and remain unresolved; no additional sources are fetched by this cross-reference step. -Referenced local composites have bounded nesting and cycle detection; unresolved installation-capable paths remain candidates. Missing, malformed or unresolvable local-action source retains a separate error marker and requires review even after a covered install. Public external-fork fallback is recorded per download as `fork-exception`: an explicit unprotected path, not verified protection. No historical repository cohort or exception allowlist is used. - -Only top-level `.github/workflows/*.yml|yaml` and `.depot/workflows/*.yml|yaml` on active repositories' default branches are inspected. The detector does not execute workflows, resolve package scripts/lifecycle hooks, validate credential existence, prove actual registry traffic, interpret arbitrary shell, inspect remote action/reusable-workflow implementations, or certify bespoke controls. Local-action resolution is deliberately limited; unsupported paths or execution contexts need review. Install hooks and downloaded code remain outside the runtime-proof claim. Treat results as a prioritized human-review queue. +Only top-level `.github/workflows/*.yml|yaml`, `.depot/workflows/*.yml|yaml` and referenced local actions in the captured scope are inspected. The detector does not execute code, inspect arbitrary script/action implementations, validate credentials, prove traffic routing or certify bespoke controls. Security findings and unverified execution remain separate assurance diagnostics. No historical cohort, coverage percentage or repository exception allowlist is used. ### Manual release snapshot verification diff --git a/tools/rollout/audit.mjs b/tools/rollout/audit.mjs index 87d92ee..f0b1794 100644 --- a/tools/rollout/audit.mjs +++ b/tools/rollout/audit.mjs @@ -4,7 +4,11 @@ import { dirname, join } from "node:path"; import { classifyWorkflow, repositoryDisposition } from "./classify.mjs"; import { captureRepositoryInventory } from "./inventory.mjs"; import { ORGANIZATION } from "./constants.mjs"; -import { integrationDisposition } from "./integration.mjs"; +import { + integrationDisposition, + resolveLocalWorkflowCalls, + resolveNoInstallWorkflowCalls, +} from "./integration.mjs"; const WORKFLOW_PATH_PATTERN = /^\.(?:github|depot)\/workflows\/[^/]+\.ya?ml$/; const LOCKFILE_PATTERN = @@ -207,12 +211,14 @@ export async function auditRepository(client, repository) { ); } - const workflows = workflowPaths.map((path, index) => - classifyWorkflow(workflowTexts[index], { - localActions, - path, - visibility: repository.visibility, - }), + const workflows = resolveLocalWorkflowCalls( + workflowPaths.map((path, index) => + classifyWorkflow(workflowTexts[index], { + localActions, + path, + visibility: repository.visibility, + }), + ), ); const managers = [ @@ -279,7 +285,9 @@ export async function runAudit(client, options = {}) { }, ); - const rows = sortedByName(repositories); + const rows = sortedByName( + resolveNoInstallWorkflowCalls(repositories, ORGANIZATION), + ); const dispositions = {}; const assuranceDispositions = {}; for (const row of rows) { diff --git a/tools/rollout/classify.mjs b/tools/rollout/classify.mjs index e489a38..50ae574 100644 --- a/tools/rollout/classify.mjs +++ b/tools/rollout/classify.mjs @@ -1,4 +1,5 @@ import { parseYamlSource as parse } from "./yaml.mjs"; +import { shellCommands, shellTokens } from "./commands.mjs"; import { ACTION_REPOSITORY, APPROVED_RELEASE_SHA } from "./constants.mjs"; import { @@ -6,7 +7,7 @@ import { defaultsUncertain, environmentUncertain, observedIntegration, - toolchainPreservesRouting, + unresolvedJsInvocation, } from "./integration.mjs"; const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/; @@ -145,26 +146,16 @@ function expectedTokenExpression(visibility) { return `\${{secrets.${secret}}}`; } -function splitCommands(script) { - return String(script) - .split(/\r?\n/) - .flatMap((line) => - literalLogging(line.trim()) - ? [line] - : line.split(/&&|\|\||;|(? command.trim()) - .filter((command) => command.length > 0 && !command.startsWith("#")); -} - function commandParts(command) { - const words = command.split(/\s+/).filter((word) => word.length > 0); + const tokens = shellTokens(command); + const words = tokens.words; const environment = {}; let start = 0; - const assignments = () => { - while (/^[A-Za-z_][A-Za-z0-9_]*=/.test(words[start] ?? "")) { + const assignments = (envArguments = false) => { + const values = envArguments ? words : tokens.commands; + while (/^[A-Za-z_][A-Za-z0-9_]*=/.test(values[start] ?? "")) { // Keep keys for scope checks, never copy values to new diagnostics. - Object.defineProperty(environment, words[start].split("=", 1)[0], { + Object.defineProperty(environment, values[start].split("=", 1)[0], { value: true, enumerable: true, configurable: true, @@ -175,9 +166,13 @@ function commandParts(command) { assignments(); if (words[start] === "env" && !words[start + 1]?.startsWith("-")) { start += 1; - assignments(); + assignments(true); } - return { words: words.slice(start), environment }; + return { + words: words.slice(start), + rawWords: tokens.commands.slice(start), + environment, + }; } function commandWords(command) { @@ -194,30 +189,137 @@ function literalLogging(command) { ); } -function safePrologue(command) { - return /^set\s+(?:-e|-euo\s+pipefail)$/.test(command); +const REGISTRY_FLAG = + /^--(?:config\.)?(?:(?:@[^:\s=]+:)?(?:reg|regi|regis|regist|registr|registry)|userconfig|globalconfig)(?:=|$)/i; +const REGISTRY_KEY = /^(?:(?:@[^:]+:)?registry|userconfig|globalconfig)$/i; +function registrySetter(words) { + return ( + ["npm", "pnpm"].includes(words[0]) && + words[1] === "config" && + ["set", "delete", "unset"].includes(words[2]) && + REGISTRY_KEY.test((words[3] ?? "").split("=", 1)[0]) + ); +} +function writeTargets(tokens, includeRemovals = true) { + const target = (index) => ({ + value: tokens.words[index] ?? "", + raw: tokens.commands[index] ?? "", + }); + const targets = tokens.commands.flatMap((word, index) => + /^>+$/.test(word) ? [target(index + 1)] : [], + ); + const program = tokens.words[0]; + if ( + program === "tee" || + (includeRemovals && ["rm", "unlink", "mv", "truncate"].includes(program)) || + (["sed", "perl"].includes(program) && + tokens.words.some((word) => /^-[^-]*i|^--in-place(?:=|$)/.test(word))) + ) + return [ + ...targets, + ...tokens.words.slice(1).map((_, index) => target(index + 1)), + ]; + if (["cp", "install", "mv"].includes(program) && tokens.words.length > 1) + return [...targets, target(tokens.words.length - 1)]; + return targets; +} +function expandedTarget(target) { + return target.raw.startsWith('"') && target.raw.endsWith('"') + ? target.raw.slice(1, -1) + : target.raw; +} +function configTarget(target) { + if ( + /(?:^|\/)\.npmrc$/.test(target.value) || + /^\$(?:NPM_CONFIG_USERCONFIG|\{NPM_CONFIG_USERCONFIG(?::\?[^}]*)?\})$/.test( + expandedTarget(target), + ) + ) + return "npm"; + if ( + /(?:^|\/)bunfig\.toml$/.test(target.value) || + /^\$(?:SFW_BUN_CONFIG_PATH|\{SFW_BUN_CONFIG_PATH(?::\?[^}]*)?\})$/.test( + expandedTarget(target), + ) + ) + return "bun"; + return undefined; } -const REGISTRY_MUTATION = - /--(?:[^\s=]*:)?reg[a-z]*\b|--[^\s=]*(?:registry|userconfig)|(?:NPM|PNPM|BUN)_CONFIG_|npm_config_|\bnpm\s+config\s+(?:set|delete)\b/i; +function assignmentKeys(words) { + return Object.fromEntries( + words.flatMap((word) => { + const match = word.match(/^([A-Za-z_][A-Za-z0-9_]*)(?:=|<<)/); + return match ? [[match[1], true]] : []; + }), + ); +} +function changesStepEnvironment(command) { + const { words, environment } = commandParts(command); + if (!words.length) return environmentUncertain(environment); + if (["export", "declare", "typeset", "unset"].includes(words[0])) { + const keys = + words[0] === "unset" + ? Object.fromEntries(words.slice(1).map((key) => [key, true])) + : assignmentKeys(words.slice(1)); + return environmentUncertain(keys); + } + return false; +} +function writesEnvironment(tokens) { + const targets = writeTargets(tokens, false).map(expandedTarget); + const keys = assignmentKeys(tokens.words); + return ( + targets.some((path) => + /^\$(?:GITHUB_PATH|\{GITHUB_PATH(?::\?[^}]*)?\})$/.test(path), + ) || + (targets.some((path) => + /^\$(?:GITHUB_ENV|\{GITHUB_ENV(?::\?[^}]*)?\})$/.test(path), + ) && + (!Object.keys(keys).length || environmentUncertain(keys))) + ); +} + +function approvedBunArguments(words, rawWords) { + const indexes = new Set(); + if (words[0] !== "bun") return indexes; + const approvedPath = (value) => + /^\$(?:SFW_BUN_CONFIG_PATH|\{SFW_BUN_CONFIG_PATH(?::\?)?\})$/.test( + value?.startsWith('"') && value.endsWith('"') + ? value.slice(1, -1) + : (value ?? ""), + ); + rawWords.forEach((word, index) => { + if (word.startsWith("--config=") && approvedPath(word.slice(9))) + indexes.add(index); + if (word === "--config" && approvedPath(rawWords[index + 1])) { + indexes.add(index); + indexes.add(index + 1); + } + }); + return indexes; +} function directExecutor(words) { if (!["npx", "bunx"].includes(words[0])) return undefined; let target = 1; while (["-y", "--yes"].includes(words[target])) target += 1; - // Only literal package/binary targets and literal arguments; no npm-exec - // grammar, shell interpretation or unsupported installer options. + // Installer options end at the literal package/binary target. Arguments to + // that program are payload, not npx/bunx registry configuration. if (!/^[A-Za-z0-9_@][A-Za-z0-9_@./:+-]*$/.test(words[target] ?? "")) return undefined; - if ( - words - .slice(target + 1) - .some((word) => !/^[A-Za-z0-9_@./:=,+%-]+$/.test(word)) - ) - return undefined; return words.slice(0, target).join(" "); } +function installerArguments(words) { + const prefix = directExecutor(words); + const args = (prefix === undefined ? words : shellTokens(prefix).words).slice( + 1, + ); + const end = args.indexOf("--"); + return end === -1 ? args : args.slice(0, end); +} + function firstSubcommand(words) { return words.slice(1).find((word) => !word.startsWith("-")); } @@ -231,6 +333,19 @@ export function classifyCommand(command) { if (program.includes("/")) return { command, program, kind: "unknown-wrapper" }; const name = program; + if ( + ["npm", "pnpm", "bun", "yarn", "npx", "bunx", "corepack"].includes(name) && + words.length === 2 && + ["--version", "-v", "--help", "-h"].includes(words[1]) + ) + return { command, kind: "no-network" }; + + if ( + ["npm", "pnpm"].includes(name) && + words[1] === "config" && + ["get", "list", "ls"].includes(words[2]) + ) + return { command, kind: "no-network" }; // Do not mistake an option's value (e.g. --prefix help) for the verb. // Unsupported leading options remain review candidates, not a parsed shell. @@ -364,6 +479,35 @@ function parseUses(uses) { }; } +// Bind only whole-value composite input references; never evaluate expressions +// or interpolate shell text. This preserves caller token names through helpers. +function bindInputs(value, inputs) { + if (typeof value !== "string") return value; + const match = value.match(/^\$\{\{\s*inputs\.([\w-]+)\s*\}\}$/); + return match + ? Object.hasOwn(inputs, match[1]) + ? inputs[match[1]] + : "" + : value; +} + +function bindStepInputs(step, inputs) { + if (!isMapping(step)) return step; + return Object.fromEntries( + Object.entries(step).map(([key, value]) => [ + key, + ["with", "env"].includes(key) && isMapping(value) + ? Object.fromEntries( + Object.entries(value).map(([name, item]) => [ + name, + bindInputs(item, inputs), + ]), + ) + : value, + ]), + ); +} + function classifyUsesStep(step, context) { const uses = parseUses(step.uses); const withInput = step.with ?? {}; @@ -427,8 +571,17 @@ function classifyUsesStep(step, context) { ) { return [{ kind: "unknown-local-action", uses: step.uses }]; } + const inputs = { + ...Object.fromEntries( + Object.entries(action.inputs ?? {}).map(([key, input]) => [ + key, + input?.default ?? "", + ]), + ), + ...withInput, + }; return steps.flatMap((inner) => - classifyStep(inner, { + classifyStep(bindStepInputs(inner, inputs), { ...context, actionStack: [...stack, uses.path], }).map((operation) => ({ @@ -472,6 +625,7 @@ function classifyUsesStep(step, context) { { kind: "setup-node", registryMutating: withInput["registry-url"] !== undefined, + registryPersisting: withInput["registry-url"] !== undefined, uses: step.uses, }, ]; @@ -500,6 +654,9 @@ function classifyUsesStep(step, context) { manager: "pnpm", uses: step.uses, uncertain: true, + integrationConfigurationUncertain: ![true, "true"].includes( + runInstall, + ), }, ]; } @@ -507,7 +664,6 @@ function classifyUsesStep(step, context) { { kind: "unknown", uses: step.uses, - integrationToolchain: toolchainPreservesRouting(uses, withInput), }, ]; } @@ -515,7 +671,6 @@ function classifyUsesStep(step, context) { { kind: "unknown", uses: step.uses, - integrationToolchain: toolchainPreservesRouting(uses, withInput), }, ]; } @@ -544,6 +699,7 @@ export function classifyStep(step, context = {}) { let integrationUncertain = boundaryUncertain({ ...step, env: undefined, if: primaryRunGuard(step) }) || environmentUncertain(step.env); + const configurationBoundaryUncertain = integrationUncertain; if ( step.uses !== undefined && step.run === undefined && @@ -560,38 +716,90 @@ export function classifyStep(step, context = {}) { ) || (step.shell !== undefined && !["bash", "sh"].includes(step.shell)); uncertain ||= complexShell(step.run); - integrationUncertain ||= complexShell( - step.run - .split(/\r?\n/) - .filter((line) => !literalLogging(line.trim())) - .join("\n"), - ); - operations = splitCommands(step.run).map((command) => { + const shell = shellCommands(step.run); + integrationUncertain ||= + shell.ambiguous || + (step.shell !== undefined && !["bash", "sh"].includes(step.shell)); + const conditionalControl = + shell.conditional || + shell.commands.some((command) => + /^(?:if|case|for|while|until|select)\b/.test(command), + ); + operations = shell.commands.map((command) => { const operation = classifyCommand(command); - const { words, environment } = commandParts(command); + const { words, rawWords, environment } = commandParts(command); + const bunArguments = approvedBunArguments(words, rawWords); + const tokens = shellTokens(command); + const nestedCommands = tokens.substitutions.flatMap( + (body) => shellCommands(body).commands, + ); + const nested = nestedCommands.map((body) => commandParts(body).words); + const nestedEnvironment = nestedCommands.some( + (body, index) => + environmentUncertain(commandParts(body).environment) || + (classifyCommand(body).kind === "unknown-wrapper" && + !classifyCommand(body).manager && + unresolvedJsInvocation(classifyCommand(body))) || + changesStepEnvironment(body) || + (/^(?:env|sudo|command|time|exec)$/.test(nested[index][0] ?? "") && + unresolvedJsInvocation(classifyCommand(body))), + ); const executorPrefix = directExecutor(words); // npm's exact 'always' mode rewrites lockfile hosts to the configured // registry; it does not select another registry. Other modes need review. const replacementFlags = words.filter((word) => word.startsWith("--replace-registry-host"), ); - const configurationWords = command - .split(/\s+/) - .filter((word) => !word.startsWith("--replace-registry-host")); + const configurationWords = installerArguments(words).filter( + (word) => !word.startsWith("--replace-registry-host"), + ); const supportedReplacement = words[0] === "npm" && replacementFlags.every( (word) => word === "--replace-registry-host=always", ); + const targets = [ + ...writeTargets(tokens), + ...nestedCommands.flatMap((body) => writeTargets(shellTokens(body))), + ]; + const configKinds = targets.map(configTarget); + const configFileWrite = configKinds.some(Boolean); + const bunFileOnly = configFileWrite && !configKinds.includes("npm"); + const knownRegistryContent = + ["echo", "printf"].includes(words[0]) && + tokens.words.some((word) => + /(?:^|\n)(?:@[^:]+:)?registry\s*=/.test(word), + ); + const setter = registrySetter(words) || nested.some(registrySetter); + const dynamicSetter = [words, ...nested].some( + (args) => + ["npm", "pnpm"].includes(args[0]) && + args[1] === "config" && + ["set", "delete", "unset"].includes(args[2]) && + /[$`]/.test(args.slice(3).join(" ")), + ); + const nestedOverride = nested.some( + (args) => + ["npm", "pnpm", "bun", "yarn", "npx", "bunx"].includes(args[0]) && + installerArguments(args).some((word) => REGISTRY_FLAG.test(word)), + ); const registryMutating = - !literalLogging(command) && - REGISTRY_MUTATION.test(configurationWords.join(" ")); + setter || + dynamicSetter || + configFileWrite || + nestedOverride || + (["npm", "pnpm", "bun", "yarn", "npx", "bunx"].includes(words[0]) && + configurationWords.some((word) => REGISTRY_FLAG.test(word))) || + Object.keys(environment).some((key) => + /^(?:npm|pnpm|bun)_CONFIG_/i.test(key), + ); const unsupportedFlags = operation.kind === "js-public-download" && operation.corepack === undefined && words .filter( - (word) => + (word, index) => + !bunArguments.has(index) && !( supportedReplacement && word === "--replace-registry-host=always" @@ -601,8 +809,12 @@ export function classifyStep(step, context = {}) { (word) => word.startsWith("-") && !TRANSPARENT_INSTALL_FLAGS.has(word), ); - if (registryMutating) operation.registryMutating = true; - else if ( + if (registryMutating) { + operation.registryMutating = true; + operation.registryPersisting = + setter || dynamicSetter || configFileWrite; + if (bunFileOnly) operation.registryManager = "bun"; + } else if ( unsupportedFlags || (replacementFlags.length > 0 && !supportedReplacement) ) @@ -613,7 +825,14 @@ export function classifyStep(step, context = {}) { environmentUncertain(environment) || (replacementFlags.length > 0 && !supportedReplacement) || (words[0] === "npm" && ["exec", "x"].includes(words[1])) || - (unsupportedFlags && !registryMutating); + (unsupportedFlags && !registryMutating) || + (operation.kind === "js-public-download" && + /[$`]/.test( + words.filter((_, index) => !bunArguments.has(index)).join(" "), + )) || + (configFileWrite && !knownRegistryContent) || + nestedOverride || + dynamicSetter; if (["npx", "bunx"].includes(words[0])) { // Strict assurance still sees unknown executor code/configuration flags. operation.uncertain = true; @@ -621,34 +840,105 @@ export function classifyStep(step, context = {}) { operation.integrationUncertain = environmentUncertain(environment) || (executorPrefix === undefined && !registryMutating); - if (executorPrefix !== undefined) - operation.registryMutating = REGISTRY_MUTATION.test(executorPrefix); + operation.integrationUncertain ||= nestedOverride; + // Persistence comes from executable setters/writes, not payload text. } + if (setter && nested.some(registrySetter)) + operation.integrationUncertain = true; // Reachability is not a certain state transition: a skipped disable // cannot clear a persistent Corepack shim established by an earlier step. if ( operation.corepackControl || ["enable", "disable"].includes(operation.corepack) ) - operation.integrationUncertain ||= condition(step.if) === "uncertain"; - operation.integrationTransparent = - safePrologue(command) || literalLogging(command); - operation.integrationScript = - ["npm", "pnpm", "bun", "yarn"].includes(words[0]) && - ((words[1] === "run" && /^[A-Za-z0-9_:.+-]+$/.test(words[2] ?? "")) || - ["test", "start", "stop", "restart"].includes(words[1])) && - words - .slice(2) - .every( - (word, index) => - /^[A-Za-z0-9_@./:=,+%-]+$/.test(word) && - (!word.startsWith("-") || - word === "--" || - words.slice(2, index + 2).includes("--")), - ) && - !registryMutating; + operation.integrationUncertain ||= + condition(step.if) === "uncertain" || conditionalControl; + if ( + operation.registryMutating && + (conditionalControl || condition(step.if) === "uncertain") + ) + operation.integrationUncertain = true; + operation.commandGroup = context.stepBudget?.remaining; + operation.stepEnvironmentUncertain = changesStepEnvironment(command); + operation.environmentPersisting = + writesEnvironment(tokens) || + nestedCommands.some((body) => writesEnvironment(shellTokens(body))); + const nestedCandidates = nestedCommands.map(classifyCommand); + const nestedInvocations = nestedCandidates.filter((candidate, index) => { + const args = nested[index]; + const configRead = + args[1] === "config" && ["get", "list", "ls"].includes(args[2]); + return ( + candidate.kind === "js-public-download" || + candidate.kind === "yarn-blocked" || + (unresolvedJsInvocation(candidate) && + !configRead && + !registrySetter(args) && + !scriptInvocation(args, candidate.kind)) + ); + }); + operation.explicitJsInvocation = + nestedOverride || nestedInvocations.length > 0; + const directInvocation = + !scriptInvocation(words, operation.kind) && + unresolvedJsInvocation({ ...operation, explicitJsInvocation: false }); + const identifiedInvocations = directInvocation + ? [...nestedInvocations, operation] + : nestedInvocations; + operation.integrationManagers = [ + ...new Set( + identifiedInvocations + .map((candidate) => { + const manager = + candidate.manager ?? + shellTokens(candidate.command).words.find((word) => + ["npm", "pnpm", "bun", "yarn", "npx", "bunx"].includes(word), + ); + return manager === "npx" + ? "npm" + : manager === "bunx" + ? "bun" + : manager; + }) + .filter(Boolean), + ), + ]; + operation.integrationCorepackDownload = nestedInvocations.some( + (candidate) => candidate.corepack, + ); + operation.integrationCorepackUncertain = nestedCandidates.some( + (candidate) => + candidate.corepackControl || + ["enable", "disable"].includes(candidate.corepack), + ); + operation.integrationConfigurationUncertain = + operation.integrationUncertain || + (operation.explicitJsInvocation && nestedEnvironment) || + (directInvocation && !operation.manager) || + (step.shell !== undefined && !["bash", "sh"].includes(step.shell)) || + (operation.kind === "unknown-wrapper" && + operation.manager !== undefined && + (words[1]?.startsWith("-") || /[$`]/.test(words[1] ?? ""))) || + /^(?![A-Za-z_][A-Za-z0-9_]*=)[^\s=]+=/.test(command) || + /^(?:\.?\.?\/|\/)[^\s]*\/(?:npm|pnpm|npx|bun|bunx|yarn|corepack)\b/.test( + command, + ) || + (/^(?:env|sudo|command|time|exec)$/.test(words[0] ?? "") && + /(?:^|\s)(?:npm|pnpm|npx|bun|bunx|yarn|corepack)\b/.test(command)) || + (operation.registryMutating && shell.ambiguous); + operation.integrationSyntaxUncertain = shell.ambiguous; + operation.integrationLiteral = literalLogging(command); + // Script bodies are unverified code, not presumed dependency installs. + // pnpm/Yarn support script-name shorthands; their configuration/toolchain + // commands are not shorthands. Explicit nested JS commands stay reviewable. + operation.integrationScript = scriptInvocation(words, operation.kind); return operation; }); + if (shell.limit || shell.lexError) + operations.push({ + kind: "unknown", + sourceError: "unresolved-shell-source", + }); } else { operations = [{ kind: "unknown" }]; } @@ -658,9 +948,34 @@ export function classifyStep(step, context = {}) { integrationUncertain: integrationUncertain || (operation.integrationUncertain ?? operation.uncertain ?? false), + integrationConfigurationUncertain: + configurationBoundaryUncertain || + (operation.integrationConfigurationUncertain ?? + operation.integrationUncertain ?? + operation.uncertain ?? + false), })); } +function scriptInvocation(words, kind) { + return ( + kind === "unknown-wrapper" && + ["npm", "pnpm", "bun", "yarn"].includes(words[0]) && + (/^(?:run|test|start|stop|restart)$/.test(words[1] ?? "") || + (words[0] === "bun" && words[1] === "build") || + (["pnpm", "yarn"].includes(words[0]) && + /^[A-Za-z0-9_:.+-]+$/.test(words[1] ?? "") && + !/^(?:config|env|setup|set|create|self-update|plugin|policies|patch|patch-commit)$/.test( + words[1], + ) && + !words + .slice(2) + .some((word) => + /^(?:npm|pnpm|npx|bun|bunx|yarn|corepack)$/.test(word), + ))) + ); +} + function normalizeTriggers(workflow) { const triggers = workflow?.on ?? workflow?.[true]; if (typeof triggers === "string") { @@ -862,7 +1177,15 @@ export function classifyJob(jobName, job, context = {}, triggers = []) { return { job: jobName, managers: [], - operations: [{ kind: "reusable-call", uses: String(job.uses) }], + operations: + typeof job.uses === "string" && + job.steps === undefined && + (job.with === undefined || isMapping(job.with)) && + (job.secrets === undefined || + job.secrets === "inherit" || + isMapping(job.secrets)) + ? [{ kind: "reusable-call", uses: job.uses }] + : [{ kind: "unknown", sourceError: "malformed-reusable-call" }], status: "reusable-call", violations, integration: { diff --git a/tools/rollout/classify.test.mjs b/tools/rollout/classify.test.mjs index dc722a9..6cb0c87 100644 --- a/tools/rollout/classify.test.mjs +++ b/tools/rollout/classify.test.mjs @@ -831,7 +831,7 @@ test("CLI: partial scan errors are distinct from gaps and terminal output is san const unknowns = await runAudit( stubClient({ getText: async () => - workflow(" build:\n steps:\n - run: npm run bootstrap\n"), + workflow(" build:\n steps:\n - run: npm $INSTALL_COMMAND\n"), }), ); assert.equal(unknowns.dispositions["needs-review"], 1); diff --git a/tools/rollout/commands.mjs b/tools/rollout/commands.mjs new file mode 100644 index 0000000..6c9ec8f --- /dev/null +++ b/tools/rollout/commands.mjs @@ -0,0 +1,145 @@ +// Lexical boundaries only: no expansion, execution or control-flow evaluation. +// Keep quoted values and substitutions atomic at both command and word level. +function lex(source, words = false) { + const commands = []; + const substitutions = []; + const groups = []; + const backticks = []; + let command = ""; + let quote; + let escaped = false; + let conditional = false; + let limit = false; + let ambiguous = /<<|^\s*(?:function\s|[\w-]+\s*\(\s*\)\s*\{)/m.test(source); + const flush = () => { + if (command.trim()) commands.push(command.trim()); + command = ""; + }; + const capture = (text) => { + if (substitutions.length < 20) substitutions.push(text); + else limit = true; + }; + for (let index = 0; index < source.length; index += 1) { + const char = source[index]; + if (escaped) { + command += char === "\n" ? "" : `\\${char}`; + escaped = false; + continue; + } + if (char === "\\" && quote !== "'") { + escaped = true; + continue; + } + if (char === "`" && quote !== "'") { + if (quote === "`") { + const frame = backticks.pop(); + capture(source.slice(frame.start + 1, index)); + quote = frame.quote; + } else { + backticks.push({ start: index, quote }); + quote = "`"; + } + command += char; + continue; + } + if ( + char === "$" && + ["(", "{"].includes(source[index + 1]) && + !["'", "`"].includes(quote) + ) { + const open = source[index + 1]; + groups.push({ + close: open === "(" ? ")" : "}", + quote, + start: index, + capture: open === "(", + }); + quote = undefined; + command += `$${open}`; + index += 1; + continue; + } + if (quote) { + command += char; + if (char === quote) quote = undefined; + continue; + } + if (["'", '"'].includes(char)) { + quote = char; + command += char; + continue; + } + if ( + char === "#" && + groups.at(-1)?.close !== "}" && + (!command || /\s$/.test(command)) + ) { + while (index < source.length && source[index] !== "\n") index += 1; + if (groups.length) command += "\n"; + else flush(); + continue; + } + if (char === "(") groups.push({ close: ")" }); + else if (groups.at(-1)?.close === char) { + const frame = groups.pop(); + if (frame.capture) capture(source.slice(frame.start + 2, index)); + quote = frame.quote; + } else if (char === ")" && !groups.length) ambiguous = true; + if (groups.length > 100) + return { + commands: [source], + substitutions: [], + ambiguous: true, + limit: true, + conditional, + }; + if (words && !groups.length && [">", "<"].includes(char)) { + flush(); + let operator = char; + while (source[index + 1] === char) operator += source[++index]; + commands.push(operator); + } else if ( + !groups.length && + (words ? /\s/.test(char) : ["\n", ";", "|", "&"].includes(char)) && + !(char === "&" && /[<>]$/.test(command)) + ) { + if (["&", "|"].includes(char) && source[index + 1] === char) + conditional = true; + flush(); + } else command += char; + } + if (escaped) command += "\\"; + flush(); + const lexError = Boolean(quote || groups.length || escaped); + return { + commands, + substitutions, + conditional, + lexError, + limit, + ambiguous: ambiguous || lexError || limit, + }; +} + +export const shellCommands = (source) => lex(source); +function wordValue(word) { + let quote; + let value = ""; + for (let index = 0; index < word.length; index += 1) { + const char = word[index]; + if (char === "\\" && quote !== "'") { + const next = word[index + 1]; + if (next !== undefined && (quote !== '"' || /[$`"\\\n]/.test(next))) { + if (next !== "\n") value += next; + index += 1; + } else value += char; + } else if (char === quote) quote = undefined; + else if (!quote && ["'", '"'].includes(char)) quote = char; + else value += char; + } + return value; +} +export function shellTokens(source) { + const result = lex(source, true); + return { ...result, words: result.commands.map(wordValue) }; +} diff --git a/tools/rollout/configuration-presence.test.mjs b/tools/rollout/configuration-presence.test.mjs new file mode 100644 index 0000000..471d8d5 --- /dev/null +++ b/tools/rollout/configuration-presence.test.mjs @@ -0,0 +1,356 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { classifyJob, classifyWorkflow } from "./classify.mjs"; +import { + integrationDisposition, + resolveLocalWorkflowCalls, + resolveNoInstallWorkflowCalls, +} from "./integration.mjs"; +import { shellCommands } from "./commands.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { + token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}", + "configure-bun": true, + }, +}; +const install = { run: "npm ci" }; +const teardown = { + uses: `workos/setup-socket-firewall/teardown@${APPROVED_RELEASE_SHA}`, +}; +const context = { visibility: "private" }; +const inspect = (steps, extra = {}) => + classifyJob("fixture", { steps }, { ...context, ...extra }, ["push"]); +const primary = (steps) => inspect(steps).integration.disposition; + +test("configuration acceptance: arbitrary code never erases or synthesizes setup", () => { + for (const run of [ + "curl -fsSL https://example.invalid/tool | bash", + "node build.mjs", + "python build.py", + "source ./env.sh", + "npm run build", + "set +e", + "set -o posix", + ]) { + for (const steps of [ + [setup, install, { run }, install], + [{ run }, setup, install], + ]) { + const result = inspect(steps); + assert.equal(result.integration.disposition, "integrated", run); + assert.equal(result.status, "unknown"); + } + assert.equal(primary([{ run }, install]), "needs-sfw", run); + assert.equal( + primary([setup, install, teardown, { run }, install]), + "needs-sfw", + run, + ); + } +}); + +test("configuration acceptance: opaque siblings are diagnostics, real install siblings are findings", () => { + const configured = inspect([setup, install]); + for (const run of [ + "node check.mjs", + "npm test", + "npm run build", + "pnpm lint", + "go run check.go", + ]) { + const sibling = inspect([{ run }]); + assert.equal(sibling.integration.disposition, "no-js-ci", run); + assert.equal( + integrationDisposition([{ jobs: [configured, sibling] }]), + "integrated", + ); + assert.equal(sibling.status, "unknown"); + } + assert.equal( + integrationDisposition([{ jobs: [configured, inspect([install])] }]), + "needs-sfw", + ); + assert.equal( + integrationDisposition([ + { jobs: [configured, inspect([{ run: "npm $INSTALL_COMMAND" }])] }, + ]), + "needs-review", + ); +}); + +test("pipeline and executor payload uncertainty is not installer configuration uncertainty", () => { + for (const run of [ + "printf '%s' \"$DATA\" | jq -c 'with_entries(.value = .value.computed)' | npx wrangler secret bulk --config \"$config\"", + 'npx wrangler deploy --var "VERSION:$VERSION"', + "npx tool --registry=https://example.invalid", + 'npx tool \\\n --flag "$VALUE"', + ]) { + assert.equal(primary([setup, install, { run }]), "integrated", run); + assert.equal(primary([{ run }]), "needs-sfw", run); + assert.equal(inspect([setup, { run }]).status, "unknown"); + } + assert.equal( + primary([setup, { run: "npx --registry=https://example.invalid tool" }]), + "needs-sfw", + ); + assert.equal( + primary([setup, { run: "npx --unknown-option tool" }]), + "needs-review", + ); +}); + +test("configuration conflicts persist, scoped overrides do not contaminate later installs", () => { + for (const run of [ + "npm config set registry https://example.invalid", + "echo 'registry=https://example.invalid' > .npmrc", + ]) { + assert.equal(primary([setup, { run }, install]), "needs-sfw"); + } + assert.equal( + primary([ + setup, + { + run: "npm config set registry https://example.invalid", + if: "inputs.override", + }, + install, + ]), + "needs-review", + ); + const result = inspect([ + setup, + { run: "npm ci --registry=https://example.invalid" }, + install, + ]); + assert.deepEqual( + result.integration.downloads.map((path) => path.status), + ["gap", "covered"], + ); + assert.equal( + primary([setup, { run: 'test -n "${NPM_CONFIG_USERCONFIG:-}"' }, install]), + "integrated", + ); + assert.equal( + primary([setup, { run: "export HOME=/tmp\nnpm ci" }]), + "needs-review", + ); + assert.equal( + primary([setup, { run: "export HOME=/tmp" }, install]), + "integrated", + ); + assert.equal(primary([setup, { run: "HOME=/tmp npm ci" }]), "needs-review"); +}); + +test("lexical boundaries do not turn quoted data into installs", () => { + for (const run of [ + "echo 'npm ci; npm ci'", + 'printf "%s" "example: npm ci; npm install"', + "echo '${{ inputs.payload }}; npm ci; npm install'", + "echo '$(npm ci)'", + ]) { + assert.equal(inspect([{ run }]).integration.downloads.length, 0, run); + assert.notEqual(primary([{ run }]), "needs-sfw", run); + } + assert.deepEqual(shellCommands('echo "a;b|c" | npx tool # npm ci').commands, [ + 'echo "a;b|c"', + "npx tool", + ]); + assert.deepEqual( + shellCommands( + 'VALUE="$(node --print \'require("./package.json").name\')"\nnpm ci', + ).commands, + ['VALUE="$(node --print \'require("./package.json").name\')"', "npm ci"], + ); + assert.equal(shellCommands("cat <<'EOF'\nnpm ci\nEOF").ambiguous, true); + assert.equal(primary([{ run: "cat <<'EOF'\nnpm ci\nEOF" }]), "needs-review"); +}); + +test("unparsed explicit JS invocation retains setup evidence, not execution assurance", () => { + for (const run of [ + "RESULT=$(npx tool --output json)", + "npm pack package-name", + ]) { + const result = inspect([setup, { run }]); + assert.equal(result.integration.disposition, "integrated", run); + assert.equal( + result.integration.additionalJsPaths[0].status, + "setup-observed", + ); + assert.equal(result.status, "unknown"); + assert.equal(primary([{ run }]), "needs-review"); + } + for (const run of [ + "uv run npm ci", + "env -i npm ci", + "npm --future-option ci", + "npm $COMMAND", + ]) + assert.equal(primary([setup, { run }]), "needs-review", run); +}); + +const workflow = (path, jobs) => + classifyWorkflow(JSON.stringify({ on: ["workflow_call"], jobs }), { + ...context, + path, + }); +test("local reusable calls use the already-read snapshot and preserve gap precedence", () => { + for (const steps of [ + [setup, install], + [install], + [setup, { run: "npm --future-option ci" }], + ]) { + const target = workflow(".github/workflows/shared.yml", { + build: { steps }, + }); + const caller = workflow(".github/workflows/ci.yml", { + call: { uses: "./.github/workflows/shared.yml" }, + own: { steps: [setup, install] }, + }); + const resolved = resolveLocalWorkflowCalls([caller, target]); + assert.equal( + resolved[0].jobs[0].integration.disposition, + target.jobs[0].integration.disposition, + ); + assert.equal(resolved[0].jobs[0].status, "reusable-call"); + assert.equal( + resolved[0].jobs[0].integration.referencedWorkflow, + target.path, + ); + } +}); + +test("missing, remote, cyclic and malformed reusable sources stay unresolved", () => { + const missing = workflow(".github/workflows/missing.yml", { + call: { uses: "./.github/workflows/absent.yml" }, + }); + const a = workflow(".github/workflows/a.yml", { + call: { uses: "./.github/workflows/b.yml" }, + }); + const b = workflow(".github/workflows/b.yml", { + call: { uses: "./.github/workflows/a.yml" }, + }); + const remote = workflow(".github/workflows/remote.yml", { + call: { uses: "example/shared/.github/workflows/ci.yml@main" }, + }); + const invalid = workflow(".github/workflows/invalid.yml", { + call: { uses: "./.github/workflows/leaf.yml", steps: [install] }, + }); + const leaf = workflow(".github/workflows/leaf.yml", { + build: { steps: [setup, install] }, + }); + for (const result of resolveLocalWorkflowCalls([ + missing, + a, + b, + remote, + invalid, + leaf, + ]).slice(0, -1)) + assert.equal(integrationDisposition([result]), "needs-review"); + const chain = Array.from({ length: 22 }, (_, n) => + workflow( + `.github/workflows/depth-${n}.yml`, + n === 21 + ? { build: { steps: [setup, install] } } + : { call: { uses: `./.github/workflows/depth-${n + 1}.yml` } }, + ), + ); + assert.equal( + integrationDisposition([resolveLocalWorkflowCalls(chain)[0]]), + "integrated", + ); +}); + +test("remote no-install calls resolve only against an exactly captured ref and source", () => { + const sha = "a".repeat(40); + const body = workflow(".github/workflows/guard.yml", { + guard: { steps: [{ run: "git diff --exit-code" }] }, + }); + const target = { + name: "shared", + defaultBranch: "main", + headSha: sha, + disposition: "no-js-ci", + workflows: [body], + }; + const caller = (ref, owner = "example") => ({ + name: "app", + disposition: "needs-review", + assuranceDisposition: "needs-review", + workflows: [ + workflow(".github/workflows/ci.yml", { + install: { steps: [setup, install] }, + guard: { uses: `${owner}/shared/.github/workflows/guard.yml@${ref}` }, + }), + ], + }); + for (const ref of [sha, "main"]) { + const result = resolveNoInstallWorkflowCalls( + [caller(ref), target], + "example", + )[0]; + assert.equal(result.disposition, "integrated"); + assert.equal(result.assuranceDisposition, "needs-review"); + assert.equal( + result.workflows[0].jobs.find((job) => job.job === "guard").integration + .referencedHeadSha, + sha, + ); + } + for (const input of [ + [caller("b".repeat(40)), target], + [caller("v1"), target], + [caller("main", "other"), target], + [caller("main")], + [caller("main"), { ...target, disposition: "audit-error" }], + [ + caller("main"), + { + ...target, + workflows: [ + workflow(body.path, { build: { steps: [setup, install] } }), + ], + }, + ], + ]) { + assert.equal( + resolveNoInstallWorkflowCalls(input, "example")[0].disposition, + "needs-review", + ); + } +}); + +test("whole-value composite inputs preserve actual caller configuration", () => { + const localActions = new Map([ + [ + "helper/action.yml", + JSON.stringify({ + inputs: { token: {} }, + runs: { + using: "composite", + steps: [ + { ...setup, with: { token: "${{ inputs.token }}" } }, + { ...install, shell: "bash" }, + ], + }, + }), + ], + ]); + const call = (token) => + inspect([{ uses: "./helper", with: { token } }], { localActions }); + assert.equal( + call("${{ secrets.SOCKET_FIREWALL_TOKEN }}").integration.disposition, + "integrated", + ); + assert.equal(call("wrong").integration.disposition, "needs-sfw"); + assert.equal( + call("${{ secrets[inputs.name] }}").integration.disposition, + "needs-review", + ); + assert.equal( + inspect([{ uses: "./helper" }], { localActions }).integration.disposition, + "needs-sfw", + ); +}); diff --git a/tools/rollout/configuration-regressions.test.mjs b/tools/rollout/configuration-regressions.test.mjs new file mode 100644 index 0000000..15edcf3 --- /dev/null +++ b/tools/rollout/configuration-regressions.test.mjs @@ -0,0 +1,348 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { classifyJob } from "./classify.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { + token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}", + "configure-bun": true, + }, +}; +const install = { run: "npm ci" }; +const inspect = (steps, extra = {}, job = {}) => + classifyJob( + "fixture", + { steps, ...job }, + { visibility: "private", ...extra }, + ["push"], + ); +const primary = (steps) => inspect(steps).integration.disposition; + +test("quoted assignments and config text never fabricate installs or mutations", () => { + for (const run of [ + 'NOTE="example npm ci "', + "NOTE='example npm ci '", + "NOTE=${VALUE:-example npm ci }", + "NOTE=$(printf '%s' 'npm ci')", + "VERSION=$(bun --version)", + "printf '%s' 'npm config set registry https://example.invalid'", + "printf '%s' 'HOME=/tmp > $GITHUB_ENV'", + "printf '%s' 'registry=https://example.invalid' '>' '.npmrc'", + "printf '%s' 'HOME=/tmp' '>' '$GITHUB_ENV'", + "echo 'HOME=/tmp' > '$GITHUB_ENV'", + ]) { + assert.equal(primary([{ run }]), "no-js-ci", run); + assert.equal(primary([setup, { run }, install]), "integrated", run); + } + assert.equal( + primary([ + setup, + { run: "npx tool 'npm config set registry https://example.invalid'" }, + install, + ]), + "integrated", + ); + assert.equal( + primary([ + setup, + { run: "RESULT=$(npx tool --registry=https://example.invalid)" }, + install, + ]), + "integrated", + ); +}); + +test("transient overrides on unparsed and nested JS installers remain findings", () => { + for (const run of [ + "npm --registry=https://example.invalid ci", + "pnpm --registry=https://example.invalid install", + "RESULT=$(npm --registry=https://example.invalid ci)", + "RESULT=$(env -i npm ci)", + "RESULT=$(HOME=/tmp npm ci)", + ]) { + const result = inspect([setup, install, { run }, install]); + assert.equal(result.integration.disposition, "needs-review", run); + assert.equal(result.integration.downloads.at(-1).status, "covered"); + } +}); + +test("standalone relevant assignments affect the same run, not later run blocks", () => { + for (const key of [ + "HOME", + "NPM_CONFIG_REGISTRY", + "NPM_CONFIG_USERCONFIG", + "PATH", + ]) { + assert.equal( + primary([setup, { run: `${key}=/tmp\nnpm ci` }]), + "needs-review", + key, + ); + assert.equal( + primary([setup, { run: `${key}=/tmp` }, install]), + "integrated", + key, + ); + } + assert.equal( + primary([setup, { run: 'NOTE="HOME=/tmp"\nnpm ci' }]), + "integrated", + ); + assert.equal( + primary([setup, { run: 'export NOTE="HOME=/tmp"\nnpm ci' }]), + "integrated", + ); +}); + +test("explicit persistent environment-file changes reach later installs", () => { + for (const run of [ + 'echo "HOME=/tmp" >> "$GITHUB_ENV"', + 'echo "NPM_CONFIG_USERCONFIG=/tmp/other" >> "${GITHUB_ENV:?}"', + 'echo "/tmp/bin" >> "$GITHUB_PATH"', + 'RESULT=$(echo "HOME=/tmp" >> "$GITHUB_ENV")', + ]) + assert.equal(primary([setup, { run }, install]), "needs-review", run); + assert.equal( + primary([ + setup, + { run: 'echo "APP_MODE=production" >> "$GITHUB_ENV"' }, + install, + ]), + "integrated", + ); + assert.equal( + primary([setup, { run: 'cat "$GITHUB_ENV"' }, install]), + "integrated", + ); + assert.equal( + inspect( + [setup, install], + {}, + { env: { GITHUB_ENV: "/tmp/not-the-runner-env" } }, + ).integration.disposition, + "needs-review", + ); +}); + +test("package-manager and file registry writes remain configuration conflicts", () => { + for (const run of [ + "pnpm config set @example:registry https://example.invalid", + "npm config set registry https://example.invalid", + ]) + assert.equal(primary([setup, { run }, install]), "needs-sfw", run); + for (const run of [ + "echo 'registry=https://example.invalid' | tee .npmrc", + 'tee "$NPM_CONFIG_USERCONFIG" < input.txt', + "RESULT=$(npm config set registry https://example.invalid)", + "npm config set registry $REGISTRY", + "npm config set $KEY $VALUE", + ]) + assert.equal(primary([setup, { run }, install]), "needs-review", run); + const bunWrite = { + run: "printf \"[install]\\nregistry='https://example.invalid'\\n\" > bunfig.toml", + }; + assert.notEqual( + primary([setup, bunWrite, { run: "bun install" }]), + "integrated", + ); + assert.equal(primary([setup, bunWrite, install]), "integrated"); +}); + +test("explicit config-file mutations are distinct from reading or copying a source", () => { + for (const run of [ + 'rm "$NPM_CONFIG_USERCONFIG"', + "mv .npmrc old-config", + "sed -i 's/registry/other/' .npmrc", + 'cp input.txt "$NPM_CONFIG_USERCONFIG"', + ]) + assert.equal(primary([setup, { run }, install]), "needs-review", run); + for (const run of [ + 'cat "$NPM_CONFIG_USERCONFIG"', + 'cp "$NPM_CONFIG_USERCONFIG" artifact.txt', + ]) + assert.equal(primary([setup, { run }, install]), "integrated", run); +}); + +test("composite condition references are not replaced with a different expression type", () => { + const localActions = new Map([ + [ + "helper/action.yml", + JSON.stringify({ + inputs: { enabled: { default: "false" } }, + runs: { + using: "composite", + steps: [ + { if: "${{ inputs.enabled }}", run: "npm ci", shell: "bash" }, + ], + }, + }), + ], + ]); + // Composite inputs are strings: the nonempty string "false" is not the + // literal boolean expression false. The install must not disappear. + const result = inspect([{ uses: "./helper" }], { localActions }); + assert.notEqual(result.integration.disposition, "no-js-ci"); + assert.equal(result.integration.downloads.length, 1); +}); + +test("the action-exported Bun config path is recognized without trusting overrides", () => { + for (const flag of [ + '--config="${SFW_BUN_CONFIG_PATH:?}"', + '--config "$SFW_BUN_CONFIG_PATH"', + "--config=$SFW_BUN_CONFIG_PATH", + ]) { + const run = `bun install --frozen-lockfile ${flag}`; + assert.equal(primary([setup, { run }]), "integrated"); + assert.equal( + primary([ + { ...setup, with: { ...setup.with, "configure-bun": false } }, + { run }, + ]), + "needs-sfw", + ); + assert.equal( + primary([setup, { run, env: { SFW_BUN_CONFIG_PATH: "/tmp/other" } }]), + "needs-review", + ); + } + for (const flag of [ + "--config='$SFW_BUN_CONFIG_PATH'", + '--config="${SFW_BUN_CONFIG_PATH:-other}"', + "--config=./custom.toml", + ]) + assert.equal( + primary([setup, { run: `bun install ${flag}` }]), + "needs-review", + ); +}); + +test("nested JS paths retain manager-specific configuration requirements", () => { + const npmOnly = { ...setup, with: { ...setup.with, "configure-bun": false } }; + for (const run of ["RESULT=$(bun install)", "RESULT=$(uv run bun install)"]) + assert.equal(primary([npmOnly, { run }]), "needs-review"); + assert.equal( + primary([npmOnly, { run: "RESULT=$(npx tool --argument bun)" }]), + "integrated", + ); + assert.equal( + primary([setup, { run: "RESULT=$(yarn install)" }]), + "needs-review", + ); + assert.equal( + primary([ + { run: "corepack enable" }, + setup, + { run: "RESULT=$(pnpm install)" }, + ]), + "needs-review", + ); + assert.equal( + primary([setup, { run: "RESULT=$(corepack prepare pnpm@10)" }]), + "needs-review", + ); + assert.equal( + primary([ + setup, + { run: "RESULT=$(corepack enable)" }, + { run: "pnpm install" }, + ]), + "needs-review", + ); +}); + +test("explicit nested installs survive outer script and mutation roles", () => { + for (const run of [ + 'npm run build "$(npm ci)"', + "RESULT=$(npm ci; npm config set registry https://example.invalid)", + ]) { + assert.equal(primary([{ run }]), "needs-review"); + assert.equal(primary([{ run }, setup, install]), "needs-review"); + } + assert.equal( + primary([setup, { run: 'npm run build "$(npm ci)"' }]), + "integrated", + ); + assert.equal( + primary([{ run: 'npm run build "$(printf npm)"' }, setup, install]), + "integrated", + ); +}); + +test("configuration-file mutations remain visible inside substitutions", () => { + for (const run of [ + 'RESULT=$(echo "@example:registry=https://example.invalid" > .npmrc)', + 'RESULT=$(rm "$NPM_CONFIG_USERCONFIG")', + "RESULT=$(tee .npmrc < input.txt)", + ]) + assert.equal(primary([setup, { run }, install]), "needs-review", run); + assert.equal( + primary([ + setup, + { run: "RESULT=$(printf '%s' 'registry=example' '>' '.npmrc')" }, + install, + ]), + "integrated", + ); +}); + +test("unmodeled direct JS launchers do not inherit guessed configuration", () => { + for (const run of [ + "uv run bun install", + "uv run npx --registry=https://example.invalid tool", + "uv run npm config set registry https://example.invalid", + 'bash -c "bun install"', + ]) { + assert.equal(primary([setup, { run }, install]), "needs-review", run); + assert.equal( + primary([setup, { run: `RESULT=$(${run})` }, install]), + "needs-review", + run, + ); + } +}); + +test("literal word quoting and escapes do not hide registry options", () => { + for (const run of [ + 'n"pm" ci --registry=https://example.invalid', + 'npm ci --reg""istry=https://example.invalid', + String.raw`npm ci \--registry=https://example.invalid`, + ]) + assert.equal(primary([setup, { run }]), "needs-sfw", run); + assert.equal( + primary([setup, { run: "npm ci --registration=unused" }]), + "needs-review", + ); + assert.equal( + primary([ + setup, + { run: 'npm run build -- --registry=example "$(npm ci)"' }, + ]), + "integrated", + ); +}); + +test("line continuations remove the newline without inventing a word boundary", () => { + assert.equal(primary([{ run: "NOTE=hello\\\nnpm ci" }]), "no-js-ci"); + assert.equal(primary([{ run: "n\\\npm ci" }]), "needs-sfw"); + assert.equal(primary([setup, { run: "n\\\npm ci" }]), "integrated"); + assert.equal( + primary([setup, { run: "npm ci --reg\\\nistry=https://example.invalid" }]), + "needs-sfw", + ); +}); + +test("unresolved lexical structure or a lexical limit cannot disappear after a covered install", () => { + for (const run of [ + "NOTE='unterminated", + `NOTE=${"$(".repeat(105)}echo ok${")".repeat(105)}`, + ]) { + const result = inspect([setup, install, { run }]); + assert.equal(result.integration.disposition, "needs-review"); + assert.ok( + result.operations.some( + (operation) => operation.sourceError === "unresolved-shell-source", + ), + ); + } +}); diff --git a/tools/rollout/integration.mjs b/tools/rollout/integration.mjs index ed495ea..8ef9ee3 100644 --- a/tools/rollout/integration.mjs +++ b/tools/rollout/integration.mjs @@ -2,7 +2,7 @@ import { APPROVED_RELEASE_SHA } from "./constants.mjs"; // Configuration/executable/startup controls, not application credentials. const RELEVANT_ENV = - /^(?:(?:npm|pnpm|bun|yarn|corepack)_|HOME$|USERPROFILE$|XDG_|APPDATA$|LOCALAPPDATA$|PATH$|NODE_OPTIONS$|NODE_PATH$|BASH_ENV$|ENV$|SHELL$|SHELLOPTS$|BASHOPTS$|CDPATH$|LD_|DYLD_|BASH_FUNC_|HTTP_PROXY$|HTTPS_PROXY$|ALL_PROXY$|NO_PROXY$|NODE_EXTRA_CA_CERTS$|NODE_TLS_REJECT_UNAUTHORIZED$|SSL_CERT_|CURL_CA_BUNDLE$)/i; + /^(?:(?:npm|pnpm|bun|yarn|corepack)_|HOME$|GITHUB_ENV$|GITHUB_PATH$|SFW_BUN_CONFIG_PATH$|USERPROFILE$|XDG_|APPDATA$|LOCALAPPDATA$|PATH$|NODE_OPTIONS$|NODE_PATH$|BASH_ENV$|ENV$|SHELL$|SHELLOPTS$|BASHOPTS$|CDPATH$|LD_|DYLD_|BASH_FUNC_|HTTP_PROXY$|HTTPS_PROXY$|ALL_PROXY$|NO_PROXY$|NODE_EXTRA_CA_CERTS$|NODE_TLS_REJECT_UNAUTHORIZED$|SSL_CERT_|CURL_CA_BUNDLE$)/i; // Workflow toolchain-version metadata does not select a registry/config path. const VERSION_ENV = new Set([ "NODE_VERSION", @@ -41,25 +41,6 @@ export function defaultsUncertain(defaults) { ); } -// Only the documented toolchain role is assumed here, not bootstrap traffic or -// arbitrary action implementation safety. Strict assurance keeps these opaque. -export function toolchainPreservesRouting(uses, inputs) { - if (uses.subpath !== "" || uses.kind !== "remote" || !uses.ref) return false; - const supported = - uses.repository === "pnpm/action-setup" - ? ["version", "package_json_file", "run_install"] - : uses.repository === "oven-sh/setup-bun" - ? ["bun-version", "bun-version-file"] - : undefined; - if (!supported || Object.keys(inputs).some((key) => !supported.includes(key))) - return false; - return ( - inputs.run_install === undefined || - inputs.run_install === false || - inputs.run_install === "false" - ); -} - export function certainTeardown(operation, setup) { if (operation.ref !== APPROVED_RELEASE_SHA) return false; if (!operation.uncertain) return true; @@ -79,53 +60,30 @@ export function certainTeardown(operation, setup) { ); } -function installationCapable(operation) { - if (operation.corepackControl) return false; - if (operation.kind === "unknown-local-action") return true; - if (operation.kind !== "unknown-wrapper") return false; - const program = - operation.program ?? - String(operation.command ?? "") - .trim() - .split(/\s+/)[0]; - return Boolean( - operation.manager || - operation.installationCapable || - program.includes("/") || - /['"$`=]/.test(program) || - /\.(?:[cm]?js|sh|py|rb)$/.test(program) || - [ - "env", - "npm", - "pnpm", - "bun", - "yarn", - "node", - "command", - "sudo", - "time", - "exec", - "eval", - "source", - ".", - "python", - "python3", - "ruby", - "perl", - "bash", - "sh", - "zsh", - "make", - "just", - "mise", - "rake", - "task", - "corepack", - "lerna", - "nx", - "rush", - "turbo", - ].includes(program), +// Unknown execution is not evidence of a JS dependency install. Retain review +// candidates only where source actually invokes a JS package tool, including +// unsupported wrappers/options. Do not guess from script names or executables. +export function unresolvedJsInvocation(operation) { + if (operation.explicitJsInvocation) return true; + if ( + operation.corepackControl || + operation.integrationLiteral || + operation.kind !== "unknown-wrapper" + ) + return false; + if (operation.manager) return true; + const command = operation.command ?? ""; + const install = + /(?:^|[\s/("'`])(?:(?:npm|pnpm|bun|yarn)\s+(?:ci|install|i|add|fetch|dlx|pack|update|upgrade|config\s+(?:set|delete|unset))\b|(?:npx|bunx)\s)/; + return ( + install.test(command) && + (/^(?:env|sudo|command|time|exec|bash|sh|zsh|uv|poetry|go)\b/.test( + command, + ) || + /^(?![A-Za-z_][A-Za-z0-9_]*=)[^\s=]+=/.test(command) || + /^(?:\.?\.?\/|\/)[^\s]*\/(?:npm|pnpm|npx|bun|bunx|yarn|corepack)\b/.test( + command, + )) ); } @@ -140,14 +98,15 @@ function contextUncertain(job, context) { export function observedIntegration(operations, job, context) { const downloads = []; + const additionalJsPaths = []; const notes = new Set(); const expectedToken = `\${{secrets.${context.visibility === "public" ? "PUBLIC_SOCKET_FIREWALL_TOKEN" : "SOCKET_FIREWALL_TOKEN"}}}`; let setup; let state; - let opaque = false; + let bunState; let corepack = "disabled"; - let potentialInstall = false; - let unresolvedEarlyInstall = false; + let environmentGroup; + let persistentEnvironmentUncertain = false; const uncertainContext = contextUncertain(job, context); const malformed = !Array.isArray(job.steps) || @@ -160,8 +119,24 @@ export function observedIntegration(operations, job, context) { if (malformed) notes.add("malformed-job-or-step"); for (const [index, operation] of operations.entries()) { - const uncertain = operation.integrationUncertain ?? operation.uncertain; - if (operation.corepackControl) corepack = "unresolved"; + const uncertain = + operation.integrationConfigurationUncertain ?? + operation.integrationUncertain ?? + operation.uncertain; + const currentState = + operation.manager === "bun" && bunState ? bunState : state; + const validInterval = ["covered", "fork-exception"].includes( + currentState?.status, + ); + if (operation.stepEnvironmentUncertain) + environmentGroup = operation.commandGroup; + if (operation.environmentPersisting) persistentEnvironmentUncertain = true; + const uncertainStepEnvironment = + persistentEnvironmentUncertain || + (environmentGroup !== undefined && + environmentGroup === operation.commandGroup); + if (operation.corepackControl || operation.integrationCorepackUncertain) + corepack = "unresolved"; if (["enable", "disable"].includes(operation.corepack)) corepack = uncertain ? "unresolved" @@ -170,7 +145,7 @@ export function observedIntegration(operations, job, context) { : "disabled"; if (operation.kind === "sfw-setup") { setup = operation; - opaque = false; + if (operation.configureBun === "true") bunState = undefined; if (uncertain || !["false", "true"].includes(operation.fallback)) { state = { status: "unresolved", @@ -200,7 +175,7 @@ export function observedIntegration(operations, job, context) { ? undefined : { status: "unresolved", reason: "conditional-or-uncertain-teardown" }; setup = undefined; - opaque = false; + bunState = undefined; continue; } if ( @@ -208,13 +183,17 @@ export function observedIntegration(operations, job, context) { operation.kind === "yarn-blocked" ) { let result; - if (uncertainContext || uncertain || opaque) { + if ( + uncertainContext || + uncertainStepEnvironment || + uncertain || + (operation.integrationSyntaxUncertain && !validInterval) + ) { result = { status: "unresolved", - reason: uncertainContext - ? "install-execution-context" - : opaque - ? "opaque-operation-before-install" + reason: + uncertainContext || uncertainStepEnvironment + ? "install-execution-context" : "uncertain-install-syntax-or-condition", }; } else if (operation.registryMutating) { @@ -231,10 +210,13 @@ export function observedIntegration(operations, job, context) { status: "gap", reason: "corepack-download-bypasses-registry", }; - } else if (!state) { + } else if (!currentState) { result = { status: "gap", reason: "no-active-setup" }; - } else if (state.status === "unresolved" || state.status === "gap") { - result = state; + } else if ( + currentState.status === "unresolved" || + currentState.status === "gap" + ) { + result = currentState; } else if ( operation.manager === "bun" && setup?.configureBun !== "true" @@ -244,7 +226,7 @@ export function observedIntegration(operations, job, context) { reason: "bun-configuration-missing-or-unresolved", }; } else { - result = state; + result = currentState; } downloads.push({ operation: index + 1, @@ -253,59 +235,179 @@ export function observedIntegration(operations, job, context) { ...result, }); } - if (operation.registryMutating) { - state = uncertain + if (operation.registryPersisting) { + const changed = uncertain ? { status: "unresolved", reason: "uncertain-registry-mutation" } : { status: "gap", reason: "registry-override" }; - setup = undefined; + if (operation.registryManager === "bun") bunState = changed; + else { + state = changed; + setup = undefined; + } } const unknown = operation.kind.startsWith("unknown"); if (unknown || uncertain) notes.add( unknown ? "opaque-execution" : "conditional-or-uncertain-operation", ); - const validInterval = ["covered", "fork-exception"].includes(state?.status); - const scriptRole = - operation.integrationScript && - validInterval && - !uncertainContext && - !uncertain && - !opaque; - if (installationCapable(operation)) { - potentialInstall = true; - if (!validInterval || uncertainContext || uncertain || opaque) - unresolvedEarlyInstall = true; + if (operation.integrationScript) { + notes.add("package-script-code-unverified"); } - if (scriptRole) notes.add("package-script-role-not-routing-invalidation"); - if (operation.integrationExecutor) notes.add("executor-code-unverified"); if ( - (unknown && - !operation.integrationToolchain && - !operation.integrationTransparent && - !scriptRole) || - (uncertain && operation.kind !== "js-public-download") - ) - opaque = true; + operation.explicitJsInvocation || + (!operation.integrationScript && + unresolvedJsInvocation(operation) && + !operation.registryPersisting) + ) { + const managers = new Set([ + ...(operation.integrationScript ? [] : [operation.manager]), + ...(operation.integrationManagers ?? []), + ]); + const managerUncertain = + !["npm", "pnpm", "bun", "yarn"].some((manager) => + managers.has(manager), + ) || + operation.integrationCorepackDownload || + managers.has("yarn") || + (managers.has("pnpm") && corepack !== "disabled") || + (managers.has("bun") && + (setup?.configureBun !== "true" || + (bunState && + !["covered", "fork-exception"].includes(bunState.status)))); + additionalJsPaths.push({ + operation: index + 1, + step: operation.step, + status: + validInterval && + !managerUncertain && + !operation.registryMutating && + !uncertainContext && + !uncertainStepEnvironment && + !uncertain + ? "setup-observed" + : "unresolved", + reason: "unparsed-js-invocation", + }); + } + // No opacity latch: arbitrary code may do anything at runtime, but that + // cannot erase or synthesize the configuration observed in this workflow. + if (operation.integrationExecutor) notes.add("executor-code-unverified"); } const statuses = new Set(downloads.map((download) => download.status)); - if (unresolvedEarlyInstall) notes.add("unresolved-install-path-before-setup"); const disposition = statuses.has("gap") ? "needs-sfw" - : statuses.has("unresolved") || unresolvedEarlyInstall || malformed + : statuses.has("unresolved") || + additionalJsPaths.some((path) => path.status === "unresolved") || + malformed ? "needs-review" - : downloads.length + : downloads.length || + additionalJsPaths.some((path) => path.status === "setup-observed") ? "integrated" - : potentialInstall - ? "needs-review" - : "no-js-ci"; + : "no-js-ci"; return { disposition, downloads, + additionalJsPaths, notes: [...notes].sort(), runtimeVerification: "not-performed", }; } +// Local reusable workflows are already read at this repository's captured SHA. +// Reuse their primary result rather than treating the call itself as opaque. +// Expressions inside the callee remain unresolved; this does not evaluate inputs +// or forward credentials. Missing files and cycles stay review. Acquisition +// bounds this graph to 200 workflow files; no new source is fetched here. +export function resolveLocalWorkflowCalls(workflows) { + const byPath = new Map( + workflows.map((workflow) => [workflow.path, workflow]), + ); + const resolved = new Map(); + function visit(path, stack = new Set()) { + if (stack.has(path) || !byPath.has(path)) return undefined; + if (resolved.has(path)) return resolved.get(path); + const workflow = byPath.get(path); + const result = { + ...workflow, + jobs: workflow.jobs.map((job) => { + const call = job.operations.find( + (operation) => operation.kind === "reusable-call", + ); + if (!call?.uses.startsWith("./")) return job; + const target = call.uses.slice(2); + const callee = visit(target, new Set([...stack, path])); + if (!callee) return job; + return { + ...job, + integration: { + ...job.integration, + disposition: integrationDisposition([callee]), + referencedWorkflow: target, + notes: ["local-workflow-analyzed-at-snapshot"], + }, + }; + }), + }; + resolved.set(path, result); + return result; + } + return workflows.map((workflow) => visit(workflow.path)); +} + +// Reuse already-captured remote source only when it has no observed JS install +// paths. A callee with installs needs caller-specific inputs/secrets/visibility; +// its coverage cannot simply be copied across repositories. Never substitute a +// default-branch body for a different pinned ref, tag, owner or unread source. +export function resolveNoInstallWorkflowCalls(repositories, organization) { + const byName = new Map( + repositories.map((repository) => [repository.name, repository]), + ); + return repositories.map((repository) => { + if (["audit-error", "empty", "no-ci"].includes(repository.disposition)) + return repository; + const workflows = repository.workflows.map((workflow) => ({ + ...workflow, + jobs: workflow.jobs.map((job) => { + const call = job.operations.find( + (operation) => operation.kind === "reusable-call", + ); + const match = call?.uses.match( + /^([^/]+)\/([^/]+)\/(\.github\/workflows\/[^/@]+\.ya?ml)@(.+)$/, + ); + if (!match || match[1] !== organization) return job; + const target = byName.get(match[2]); + if ( + !target?.headSha || + target.disposition === "audit-error" || + ![target.headSha, target.defaultBranch].includes(match[4]) + ) + return job; + const callee = target.workflows.find( + (candidate) => candidate.path === match[3], + ); + if (!callee || integrationDisposition([callee]) !== "no-js-ci") + return job; + return { + ...job, + integration: { + ...job.integration, + disposition: "no-js-ci", + referencedRepository: target.name, + referencedWorkflow: callee.path, + referencedHeadSha: target.headSha, + notes: ["referenced-snapshot-has-no-observed-js-install"], + }, + }; + }), + })); + return { + ...repository, + workflows, + disposition: integrationDisposition(workflows), + }; + }); +} + export function integrationDisposition(workflows) { const dispositions = new Set( workflows.flatMap((workflow) => [ diff --git a/tools/rollout/integration.test.mjs b/tools/rollout/integration.test.mjs index aa97ac2..1016d96 100644 --- a/tools/rollout/integration.test.mjs +++ b/tools/rollout/integration.test.mjs @@ -36,9 +36,9 @@ test("observed install coverage and opaque execution are separate", () => { assert.equal(result.integration.downloads[0].status, "covered"); assert.ok(result.integration.notes.includes("opaque-execution")); assert.equal(primary([install, build]), "needs-sfw"); - assert.equal(primary([setup, install, teardown, build]), "needs-review"); + assert.equal(primary([setup, install, teardown, build]), "integrated"); assert.equal(primary([setup, install, build, install]), "integrated"); - assert.equal(primary([build, setup, install]), "needs-review"); + assert.equal(primary([build, setup, install]), "integrated"); }); test("an integrated sibling cannot hide a missing or unresolved install job", () => { @@ -50,7 +50,7 @@ test("an integrated sibling cannot hide a missing or unresolved install job", () ); assert.equal( integrationDisposition([{ jobs: [job([setup, install]), job([build])] }]), - "needs-review", + "integrated", ); assert.equal(primary([{ uses: "./unresolved" }]), "needs-review"); assert.equal( @@ -78,10 +78,6 @@ test("supported toolchain roles preserve intervals without synthesizing setup", } for (const tool of [ { uses: "pnpm/action-setup@v4", with: { run_install: true } }, - { - uses: "pnpm/action-setup@v4", - with: { run_install: "${{ inputs.install }}" }, - }, { uses: "pnpm/action-setup@v4", with: { custom: "anything" } }, { uses: "pnpm/action-setup/other@v4" }, { uses: "untrusted/action-setup@v4" }, @@ -91,7 +87,18 @@ test("supported toolchain roles preserve intervals without synthesizing setup", }, { uses: "oven-sh/setup-bun/other@v2" }, ]) - assert.equal(primary([setup, tool, install]), "needs-review"); + assert.equal(primary([setup, tool, install]), "integrated"); + assert.equal( + primary([ + setup, + { + uses: "pnpm/action-setup@v4", + with: { run_install: "${{ inputs.install }}" }, + }, + install, + ]), + "needs-review", + ); }); test("job reachability and harmless context do not alter install ordering", () => { @@ -189,7 +196,6 @@ test("per-download negative controls remain actionable or unresolved", () => { [{ ...setup, "continue-on-error": true }, install], [setup, { run: "npm ci --future-option=unknown" }], [setup, { run: "npm --prefix help ci" }], - [setup, { run: "cat <<'EOF'\nnpm ci\nEOF" }], [setup, { uses: "./unresolved" }, install], [setup, { run: "corepack enable pnpm" }, { run: "pnpm install" }], ]) @@ -303,26 +309,36 @@ test("malformed steps cannot become no-js-ci or integrated", () => { ); }); -test("installation-capable ecosystem and script executors stay review candidates", () => { +test("explicit unparsed JS installers remain candidates, generic executors do not", () => { for (const run of [ "uv run npm ci", "poetry run npm ci", + "command npm ci", + "npm --prefix help ci", + ]) { + assert.equal(job([{ run }]).integration.disposition, "needs-review", run); + assert.equal( + integrationDisposition([ + { jobs: [job([setup, install]), job([{ run }])] }, + ]), + "needs-review", + run, + ); + } + for (const run of [ "go run bootstrap.go", "python scripts/bootstrap.py", "CI=1 make bootstrap", "CI=1 node scripts/bootstrap.mjs", "CI=1 ./scripts/bootstrap.sh", "${{ inputs.command }}", - "command npm ci", - "npm --prefix help ci", ]) { - assert.equal(job([{ run }]).integration.disposition, "needs-review", run); + assert.equal(primary([{ run }]), "no-js-ci"); assert.equal( integrationDisposition([ { jobs: [job([setup, install]), job([{ run }])] }, ]), - "needs-review", - run, + "integrated", ); } assert.equal( @@ -361,7 +377,7 @@ test("dynamic Bun configuration and uncertain local boundaries stay unresolved", assert.equal( job([setup, { uses: "pnpm/action-setup" }, install]).integration .disposition, - "needs-review", + "integrated", ); }); @@ -430,7 +446,7 @@ test("Corepack uncertainty survives setup and conditional or targeted controls", ); }); -test("scripts under absent, unresolved or invalid setup cannot be erased by later setup", () => { +test("script diagnostics survive later setup without becoming invented installs", () => { for (const before of [ [], [{ ...setup, if: "inputs.enable_sfw" }], @@ -443,10 +459,11 @@ test("scripts under absent, unresolved or invalid setup cannot be erased by late setup, install, ]); - assert.equal(result.integration.disposition, "needs-review"); + assert.equal(result.integration.disposition, "integrated"); assert.ok( - result.integration.notes.includes("unresolved-install-path-before-setup"), + result.integration.notes.includes("package-script-code-unverified"), ); + assert.equal(result.status, "unknown"); } }); @@ -477,7 +494,7 @@ test("inline environment and unsupported env wrappers retain installer candidate "NODE_OPTIONS=--require=./bootstrap.cjs npm ci", "env HOME=/tmp npm ci", "env -i npm ci", - "env --ignore-environment node scripts/bootstrap.mjs", + "env --ignore-environment npm ci", "env --unset=HOME npm ci", "1BAD=x npm ci", "sudo npm ci", @@ -520,13 +537,6 @@ test("only bounded literal logging and safe shell prologues preserve direct inst assert.notEqual(job([setup, { run }]).status, "protected"); } for (const run of [ - 'echo "$(node bootstrap.mjs)"\nnpm ci', - "source ./env.sh\nnpm ci", - "cat <<'EOF'\nnpm ci\nEOF", - "set +e\nnpm ci", - "set -o posix\nnpm ci", - 'echo "Installing" > .npmrc\nnpm ci', - "echo `node bootstrap.mjs`\nnpm ci", 'echo "literal source: npm ci"', 'echo "source: npm ci; npm ci"', "echo 'source: npm ci && npm ci'", @@ -553,22 +563,12 @@ test("ordinary package script roles preserve observed setup, never assure script assert.equal(result.integration.disposition, "integrated", run); assert.equal(result.status, "unknown", run); assert.ok( - result.integration.notes.includes( - "package-script-role-not-routing-invalidation", - ), + result.integration.notes.includes("package-script-code-unverified"), ); - assert.equal(primary([setup, { run }]), "needs-review", run); - assert.equal(primary([{ run }, setup, install]), "needs-review", run); + assert.equal(primary([setup, { run }]), "no-js-ci", run); + assert.equal(primary([{ run }, setup, install]), "integrated", run); } - for (const run of [ - "node scripts/build.mjs", - "npm --prefix scripts run build", - "source ./env.sh", - "npm run $TASK", - "npm run build --script-shell=./bootstrap.sh", - "npm run build --prefix=/tmp", - "npm run build --registry=https://example.invalid", - ]) { + for (const run of ["npm --prefix scripts run build"]) { assert.notEqual(primary([setup, { run }, install]), "integrated", run); } assert.equal(primary([setup, build, teardown, install]), "needs-sfw"); @@ -579,7 +579,7 @@ test("ordinary package script roles preserve observed setup, never assure script { run: "npm config set registry https://example.invalid" }, install, ]), - "needs-review", + "needs-sfw", ); }); @@ -589,6 +589,8 @@ test("direct executor target arguments are not installer configuration", () => { "npx -y wrangler deploy --env production", "bunx biome check --write", "npx tool --registry=https://example.invalid", + "npx tool --env $ENVIRONMENT", + "npx tool $(node bootstrap.mjs)", ]) { const result = job([setup, { run }]); assert.equal(result.integration.disposition, "integrated", run); @@ -607,12 +609,10 @@ test("direct executor target arguments are not installer configuration", () => { "npx --future-option tool", "npx --package=tool tool", "npx $TOOL --env production", - "npx tool $(node bootstrap.mjs)", "npx", "npm exec tool -- --env production", "npm exec tool", "env -i npx tool", - "npx tool --env $ENVIRONMENT", ]) assert.equal(primary([setup, { run }]), "needs-review", run); }); diff --git a/tools/rollout/primary-boundaries.test.mjs b/tools/rollout/primary-boundaries.test.mjs index 9a451c2..23723f6 100644 --- a/tools/rollout/primary-boundaries.test.mjs +++ b/tools/rollout/primary-boundaries.test.mjs @@ -87,16 +87,17 @@ test("toolchain version metadata is distinct from package-manager configuration" } }); -test("GitHub expressions are not literal shell logging", () => { +test("GitHub expression execution remains unverified without erasing separate install configuration", () => { for (const run of [ "echo '${{ inputs.payload }}'\nnpm ci", 'echo "${{ inputs.payload }}"\nnpm ci', ]) { assert.equal( inspect([setup, { run }]).integration.disposition, - "needs-review", + "integrated", ); - assert.equal(inspect([{ run }]).integration.disposition, "needs-review"); + assert.equal(inspect([setup, { run }]).status, "unknown"); + assert.equal(inspect([{ run }]).integration.disposition, "needs-sfw"); } assert.equal( inspect([setup, { run: "echo '$SHELL_LITERAL'\nnpm ci" }]).integration From 714c0c7d1c29f2c94f9e6cfa725adc8c007db829 Mon Sep 17 00:00:00 2001 From: "workos-tars[bot]" <269013284+workos-tars[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 12:01:46 +0000 Subject: [PATCH 08/19] Fix audited Socket Firewall configuration false positives --- README.md | 4 + tools/rollout/audit-cleanup.test.mjs | 230 +++++++++++++++++++++++++++ tools/rollout/audit.mjs | 7 +- tools/rollout/classify.mjs | 59 ++++++- tools/rollout/integration.mjs | 27 +++- 5 files changed, 317 insertions(+), 10 deletions(-) create mode 100644 tools/rollout/audit-cleanup.test.mjs diff --git a/README.md b/README.md index 97626d9..9d77f62 100644 --- a/README.md +++ b/README.md @@ -238,6 +238,10 @@ Configuration checks retain the approved action SHA, visibility-appropriate toke Known registry writes and teardown invalidate the relevant configuration; a command-local override does not contaminate later installs. npm/pnpm registry setters, configuration-file writes (including `tee`), standalone relevant assignments and GitHub environment-file writes remain findings rather than being treated as generic opacity. Reading a configuration variable is not a mutation. HOME/PATH/NODE_OPTIONS overrides, relevant exported variables within a run block, unsupported environment wrappers, dynamic mappings and job containers remain configuration questions. Ordinary application secrets/settings, version metadata and sibling service containers do not invalidate host setup ordering. Job-level guards affect shared reachability; explicit status predicates and conditional configuration transitions remain conservative. Exact npm `--replace-registry-host=always` preserves the configured registry; other modes require review. +Literal repository-relative default working directories and npm `--prefix` arguments after the install verb are treated like explicit run-step working directories. Supported install options include `--include=optional`, `--omit=dev`, `--package-lock=false` and Bun's literal `--os="*" --cpu="*"`; unknown options and dynamic or escaping directory paths remain reviewable. Root local actions (`uses: ./`) are read from `action.yml` or `action.yaml` at the captured SHA. Entirely commented-out workflow files have no active jobs; malformed or missing source still requires review. + +A setup-only `NPM_CONFIG_USERCONFIG` pointing to a named file under `${{ runner.temp }}` is recognized because the pinned action validates that path and configures both it and `HOME/.npmrc`. Install-step overrides remain unresolved. Public fallback expressions remain recorded as possible `fork-exception` paths: the pinned action validates the value and independently permits fallback only for public external-fork pull requests. Neither recognition is runtime proof or a repository allowlist. + Local reusable workflows are already read at the same repository SHA and contribute their actual primary result instead of an automatic unknown. Missing targets and cycles remain reviewable. A remote reusable call can be resolved as having no observed JS install only when that workflow was already captured in the same organization's inventory at the requested default branch or exact SHA. A different pin/tag is never replaced with the current body. Remote callees with installs require caller-specific context and remain unresolved; no additional sources are fetched by this cross-reference step. Only top-level `.github/workflows/*.yml|yaml`, `.depot/workflows/*.yml|yaml` and referenced local actions in the captured scope are inspected. The detector does not execute code, inspect arbitrary script/action implementations, validate credentials, prove traffic routing or certify bespoke controls. Security findings and unverified execution remain separate assurance diagnostics. No historical cohort, coverage percentage or repository exception allowlist is used. diff --git a/tools/rollout/audit-cleanup.test.mjs b/tools/rollout/audit-cleanup.test.mjs new file mode 100644 index 0000000..21a8404 --- /dev/null +++ b/tools/rollout/audit-cleanup.test.mjs @@ -0,0 +1,230 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { stringify } from "yaml"; +import { auditRepository } from "./audit.mjs"; +import { classifyJob, classifyWorkflow } from "./classify.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; + +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { + token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}", + "configure-bun": true, + }, +}; +const context = { visibility: "private", path: ".github/workflows/ci.yml" }; +const inspect = (steps, extra = {}) => + classifyJob("build", { steps, ...extra }, context, ["push"]); + +test("literal install options preserve configuration, not execution assurance", () => { + for (const run of [ + "npm ci --include=optional", + "npm install --omit=dev --package-lock=false", + "npm ci --prefix web", + "npm ci --prefix=packages/sdk", + 'bun install --frozen-lockfile --os="*" --cpu="*"', + ]) { + assert.equal( + inspect([setup, { run }]).integration.disposition, + "integrated", + run, + ); + assert.equal(inspect([{ run }]).integration.disposition, "needs-sfw", run); + assert.equal( + inspect([setup, { run: `${run} --registry=https://example.invalid` }]) + .integration.disposition, + "needs-sfw", + run, + ); + } + for (const run of [ + "npm ci --prefix $PROJECT", + "npm ci --prefix ../other", + "npm ci --prefix /tmp/project", + "npm ci --prefix --registry=https://example.invalid", + "npm ci --future-option web", + "npm --prefix help ci", + ]) { + assert.notEqual( + inspect([setup, { run }]).integration.disposition, + "integrated", + run, + ); + } +}); + +test("literal default working directories match explicit run-step directories", () => { + for (const directory of ["web", "./packages/chat", "."]) { + const defaults = { run: { shell: "bash", "working-directory": directory } }; + const result = inspect([setup, { run: "npm ci" }], { defaults }); + assert.equal(result.integration.disposition, "integrated"); + assert.equal(result.status, inspect([setup, { run: "npm ci" }]).status); + assert.equal( + inspect([{ run: "npm ci" }], { defaults }).integration.disposition, + "needs-sfw", + ); + const workflow = classifyWorkflow( + JSON.stringify({ + on: "push", + defaults, + jobs: { build: { steps: [setup, { run: "npm ci" }] } }, + }), + context, + ); + assert.equal(workflow.jobs[0].integration.disposition, "integrated"); + } + for (const directory of [ + "${{ inputs.path }}", + "$HOME", + "../other", + "/tmp", + null, + ]) { + assert.equal( + inspect([setup, { run: "npm ci" }], { + defaults: { run: { "working-directory": directory } }, + }).integration.disposition, + "needs-review", + ); + } + assert.equal( + inspect([setup, { run: "npm ci" }], { + defaults: { run: { "working-directory": "web", shell: "python" } }, + }).integration.disposition, + "needs-review", + ); +}); + +test("setup-owned temporary npm config does not invalidate HOME configuration", () => { + const env = { NPM_CONFIG_USERCONFIG: "${{ runner.temp }}/sfw.npmrc" }; + assert.equal( + inspect([{ ...setup, env }, { run: "npm ci" }]).integration.disposition, + "integrated", + ); + for (const steps of [ + [setup, { run: "npm ci", env }], + [{ ...setup, env: { ...env, HOME: "/other" } }, { run: "npm ci" }], + [ + { ...setup, env: { NPM_CONFIG_USERCONFIG: "${{ inputs.config }}" } }, + { run: "npm ci" }, + ], + ]) + assert.equal(inspect(steps).integration.disposition, "needs-review"); +}); + +test("public dynamic fallback stays bounded by the pinned action's fork checks", () => { + const publicSetup = { + ...setup, + with: { + ...setup.with, + token: "${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }}", + "allow-external-fork-fallback": + "${{ github.event_name == 'pull_request' }}", + }, + }; + const steps = [publicSetup, { run: "npm ci" }]; + const result = classifyJob( + "build", + { steps }, + { ...context, visibility: "public" }, + ["pull_request"], + ); + assert.equal(result.integration.disposition, "integrated"); + assert.equal(result.integration.downloads[0].status, "fork-exception"); + assert.equal(result.status, "unknown"); + assert.equal(inspect(steps).integration.disposition, "needs-review"); + assert.equal( + classifyJob( + "build", + { steps: [{ ...publicSetup, "continue-on-error": true }, steps[1]] }, + { ...context, visibility: "public" }, + ).integration.disposition, + "needs-review", + ); +}); + +test("commented-out workflows contain no active jobs; invalid YAML remains reviewable", () => { + const result = classifyWorkflow( + "# name: Disabled\n# on: push\n# jobs:\n# build: npm ci\n", + context, + ); + assert.equal(result.parseError, undefined); + assert.deepEqual(result.jobs, []); + for (const source of ["", "null", "jobs: {}", "# disabled\non: ["]) { + assert.ok(classifyWorkflow(source, context).parseError, source); + } +}); + +test("audit acquires root actions at the captured SHA, including nested composites", async () => { + const sha = "a".repeat(40); + for (const filename of ["action.yml", "action.yaml"]) { + const sources = new Map([ + [ + context.path, + stringify({ + on: "push", + jobs: { build: { steps: [{ uses: "./" }] } }, + }), + ], + [ + filename, + stringify({ + runs: { using: "composite", steps: [setup, { uses: "./nested" }] }, + }), + ], + [ + "nested/action.yml", + JSON.stringify({ + runs: { + using: "composite", + steps: [{ run: "npm ci", shell: "bash" }], + }, + }), + ], + ]); + const reads = []; + const result = await auditRepository( + { + async getRef() { + return { object: { sha } }; + }, + async getTree() { + return { + truncated: false, + tree: [...sources.keys()].map((path) => ({ + path, + type: "blob", + mode: "100644", + })), + }; + }, + async getText(repo, path, ref) { + assert.equal(ref, sha); + reads.push(path); + return sources.get(path); + }, + }, + { name: "fixture", defaultBranch: "main", visibility: "private" }, + ); + assert.deepEqual(reads.sort(), [...sources.keys()].sort()); + assert.equal(result.disposition, "integrated"); + } + const cyclic = new Map([ + [ + "action.yml", + JSON.stringify({ runs: { using: "composite", steps: [{ uses: "./" }] } }), + ], + ]); + assert.equal( + classifyJob( + "cycle", + { steps: [{ uses: "./" }] }, + { ...context, localActions: cyclic }, + ).integration.disposition, + "needs-review", + ); + assert.equal( + inspect([{ uses: "./" }]).integration.disposition, + "needs-review", + ); +}); diff --git a/tools/rollout/audit.mjs b/tools/rollout/audit.mjs index f0b1794..f4244fb 100644 --- a/tools/rollout/audit.mjs +++ b/tools/rollout/audit.mjs @@ -13,7 +13,7 @@ import { const WORKFLOW_PATH_PATTERN = /^\.(?:github|depot)\/workflows\/[^/]+\.ya?ml$/; const LOCKFILE_PATTERN = /(?:^|\/)(?:package-lock\.json|npm-shrinkwrap\.json|pnpm-lock\.yaml|yarn\.lock|bun\.lockb?)$/; -const LOCAL_ACTION_PATTERN = /uses:\s*['"]?\.\/([^\s'"#]+)/g; +const LOCAL_ACTION_PATTERN = /uses:\s*['"]?\.\/([^\s'"#]*)/g; const AUDIT_CONCURRENCY = 5; const MAX_WORKFLOWS_PER_REPOSITORY = 200; const MAX_LOCAL_ACTIONS_PER_REPOSITORY = 200; @@ -182,9 +182,10 @@ export async function auditRepository(client, repository) { for (const text of pendingTexts.splice(0)) { for (const match of text.matchAll(LOCAL_ACTION_PATTERN)) { const base = match[1].replace(/\/+$/, ""); + const prefix = base ? `${base}/` : ""; for (const candidate of [ - `${base}/action.yml`, - `${base}/action.yaml`, + `${prefix}action.yml`, + `${prefix}action.yaml`, ]) { if (blobPaths.has(candidate) && !localActionPaths.has(candidate)) { localActionPaths.add(candidate); diff --git a/tools/rollout/classify.mjs b/tools/rollout/classify.mjs index 50ae574..789c746 100644 --- a/tools/rollout/classify.mjs +++ b/tools/rollout/classify.mjs @@ -6,6 +6,7 @@ import { certainTeardown, defaultsUncertain, environmentUncertain, + literalWorkingDirectory, observedIntegration, unresolvedJsInvocation, } from "./integration.mjs"; @@ -89,6 +90,11 @@ const TRANSPARENT_INSTALL_FLAGS = new Set([ "--prefer-offline", "--no-progress", "--no-save", + "--package-lock=false", + "--omit=dev", + "--include=optional", + "--os=*", + "--cpu=*", "--save-exact", "--save-dev", "--global", @@ -524,9 +530,10 @@ function classifyUsesStep(step, context) { }, ]; } + const prefix = uses.path ? `${uses.path}/` : ""; const actionText = - context.localActions?.get(`${uses.path}/action.yml`) ?? - context.localActions?.get(`${uses.path}/action.yaml`); + context.localActions?.get(`${prefix}action.yml`) ?? + context.localActions?.get(`${prefix}action.yaml`); if (actionText === undefined) { return [ { @@ -696,9 +703,23 @@ export function classifyStep(step, context = {}) { return [{ kind: "unknown", sourceError: "malformed-action-input" }]; let operations; let uncertain = boundaryUncertain(step); + let integrationEnv = step.env; + if ( + typeof step.uses === "string" && + step.uses.split("@")[0] === ACTION_REPOSITORY && + isMapping(step.env) && + /^\$\{\{runner\.temp\}\}\/[\w.-]+$/.test( + normalizeExpression(step.env.NPM_CONFIG_USERCONFIG), + ) + ) { + // Setup validates this path and configures both it and HOME/.npmrc. + // An override on an install step still requires separate review. + integrationEnv = { ...step.env }; + delete integrationEnv.NPM_CONFIG_USERCONFIG; + } let integrationUncertain = boundaryUncertain({ ...step, env: undefined, if: primaryRunGuard(step) }) || - environmentUncertain(step.env); + environmentUncertain(integrationEnv); const configurationBoundaryUncertain = integrationUncertain; if ( step.uses !== undefined && @@ -793,6 +814,25 @@ export function classifyStep(step, context = {}) { Object.keys(environment).some((key) => /^(?:npm|pnpm|bun)_CONFIG_/i.test(key), ); + // A literal npm prefix selects the project directory, just like a + // run-step working-directory. Do not accept dynamic or escaping paths. + const directoryArguments = new Set(); + if (words[0] === "npm" && !words[1]?.startsWith("-")) { + words.forEach((word, index) => { + if ( + word === "--prefix" && + literalWorkingDirectory(words[index + 1]) + ) { + directoryArguments.add(index); + directoryArguments.add(index + 1); + } else if ( + word.startsWith("--prefix=") && + literalWorkingDirectory(word.slice(9)) + ) { + directoryArguments.add(index); + } + }); + } const unsupportedFlags = operation.kind === "js-public-download" && operation.corepack === undefined && @@ -800,6 +840,7 @@ export function classifyStep(step, context = {}) { .filter( (word, index) => !bunArguments.has(index) && + !directoryArguments.has(index) && !( supportedReplacement && word === "--replace-registry-host=always" @@ -1304,6 +1345,18 @@ export function classifyWorkflow(text, context) { triggers: [], }; } + if ( + workflow == null && + text.trim().startsWith("#") && + text.split("\n").every((line) => /^\s*(?:#.*)?$/.test(line)) + ) { + return { + jobs: [], + path: context.path, + triggers: [], + status: "no-in-scope-download", + }; + } if ( !isMapping(workflow) || !isMapping(workflow.jobs) || diff --git a/tools/rollout/integration.mjs b/tools/rollout/integration.mjs index 8ef9ee3..2b35240 100644 --- a/tools/rollout/integration.mjs +++ b/tools/rollout/integration.mjs @@ -25,6 +25,14 @@ export function environmentUncertain(env) { ); } +export function literalWorkingDirectory(value) { + return ( + typeof value === "string" && + /^(?:\.\/)?[\w.-]+(?:\/[\w.-]+)*\/?$/.test(value) && + !value.split("/").includes("..") + ); +} + export function defaultsUncertain(defaults) { if (defaults === undefined) return false; return ( @@ -35,8 +43,10 @@ export function defaultsUncertain(defaults) { !defaults.run || typeof defaults.run !== "object" || Array.isArray(defaults.run) || - Object.entries(defaults.run).some( - ([key, value]) => key !== "shell" || !["bash", "sh"].includes(value), + Object.entries(defaults.run).some(([key, value]) => + key === "working-directory" + ? !literalWorkingDirectory(value) + : key !== "shell" || !["bash", "sh"].includes(value), ) ); } @@ -146,7 +156,16 @@ export function observedIntegration(operations, job, context) { if (operation.kind === "sfw-setup") { setup = operation; if (operation.configureBun === "true") bunState = undefined; - if (uncertain || !["false", "true"].includes(operation.fallback)) { + // The pinned action validates the value and independently restricts + // fallback to public external-fork PRs. An expression cannot widen that. + const publicForkFallback = + context.visibility === "public" && + (operation.fallback === "true" || + /^\$\{\{.+\}\}$/.test(operation.fallback)); + if ( + uncertain || + (!publicForkFallback && !["false", "true"].includes(operation.fallback)) + ) { state = { status: "unresolved", reason: "conditional-or-uncertain-setup", @@ -163,7 +182,7 @@ export function observedIntegration(operations, job, context) { state = { status: "unresolved", reason: "unresolved-setup-token" }; } else { state = { - status: operation.fallback === "true" ? "fork-exception" : "covered", + status: publicForkFallback ? "fork-exception" : "covered", reason: "approved-setup-interval", }; } From 89d8dd3b082600783fc3ac43b8732f384ded9a0d Mon Sep 17 00:00:00 2001 From: "workos-tars[bot]" <269013284+workos-tars[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 13:44:34 +0000 Subject: [PATCH 09/19] Recognize guarded installs and npm workspace executor boundaries --- README.md | 4 +- tools/rollout/audit-cleanup.test.mjs | 167 +++++++++++++++++++++++++++ tools/rollout/classify.mjs | 53 +++++++-- tools/rollout/integration.mjs | 45 +++++++- tools/rollout/integration.test.mjs | 3 +- 5 files changed, 261 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 9d77f62..4e252bc 100644 --- a/README.md +++ b/README.md @@ -234,12 +234,14 @@ The primary question is **whether SFW is configured for identified dependency-in The small grammar recognizes npm/pnpm/Bun installs, download-capable executors, Yarn blockers and setup/teardown ordering. A lexical boundary scanner keeps quoted data and command substitutions inside their containing command rather than inventing installs from text. Pipelines and executor payload arguments do not invalidate configuration simply because their execution is opaque. For `npx`/`bunx`, a literal target after optional `-y`/`--yes` separates installer options from program arguments, including quoted or variable-valued payloads. Pre-target registry overrides remain gaps; unsupported installer options, dynamic installer targets and explicit context changes can still require review. Uninterpreted shell structure can establish configuration presence without proving whether a particular command executes; it cannot establish a definite missing install when the command may only be data. -Configuration checks retain the approved action SHA, visibility-appropriate token, Bun configuration and public-fork exception rules. Disabled, conditional or continued-on-error setup cannot silently cover an install. Corepack uncertainty survives conditional or targeted controls. The action-exported `SFW_BUN_CONFIG_PATH` is recognized in Bun's `--config` argument when Bun setup is enabled; explicit overrides of that variable remain configuration questions. Whole-value composite input references in `with`/`env` are bound to caller values/defaults without evaluating conditions or interpolating shell text. Missing/malformed local source, cycles and expansion limits remain review findings. +Configuration checks retain the approved action SHA, visibility-appropriate token, Bun configuration and public-fork exception rules. Disabled or continued-on-error setup cannot silently cover an install. Conditional setup covers only an install with the same recognized boolean-string output comparison: a dependency job output, or an output from a uniquely identified earlier step. Future/duplicate step IDs, mutable environment guards, status functions and unmatched conditions remain unresolved; strict execution assurance is unchanged. Corepack uncertainty survives conditional or targeted controls. The action-exported `SFW_BUN_CONFIG_PATH` is recognized in Bun's `--config` argument when Bun setup is enabled; explicit overrides of that variable remain configuration questions. Whole-value composite input references in `with`/`env` are bound to caller values/defaults without evaluating conditions or interpolating shell text. Missing/malformed local source, cycles and expansion limits remain review findings. Known registry writes and teardown invalidate the relevant configuration; a command-local override does not contaminate later installs. npm/pnpm registry setters, configuration-file writes (including `tee`), standalone relevant assignments and GitHub environment-file writes remain findings rather than being treated as generic opacity. Reading a configuration variable is not a mutation. HOME/PATH/NODE_OPTIONS overrides, relevant exported variables within a run block, unsupported environment wrappers, dynamic mappings and job containers remain configuration questions. Ordinary application secrets/settings, version metadata and sibling service containers do not invalidate host setup ordering. Job-level guards affect shared reachability; explicit status predicates and conditional configuration transitions remain conservative. Exact npm `--replace-registry-host=always` preserves the configured registry; other modes require review. Literal repository-relative default working directories and npm `--prefix` arguments after the install verb are treated like explicit run-step working directories. Supported install options include `--include=optional`, `--omit=dev`, `--package-lock=false` and Bun's literal `--os="*" --cpu="*"`; unknown options and dynamic or escaping directory paths remain reviewable. Root local actions (`uses: ./`) are read from `action.yml` or `action.yaml` at the captured SHA. Entirely commented-out workflow files have no active jobs; malformed or missing source still requires review. +Literal leading npm workspace selectors (`-w`, `--workspace`, and `--workspace=`) retain the underlying command classification: scripts stay unverified script execution, while installs and explicit `npm exec -- ` remain download candidates. Literal npm package selectors in `npx --package`/`-p` are recognized without treating executable payload flags as installer configuration. Dynamic selectors, unsupported options, and post-teardown executor downloads remain findings. + A setup-only `NPM_CONFIG_USERCONFIG` pointing to a named file under `${{ runner.temp }}` is recognized because the pinned action validates that path and configures both it and `HOME/.npmrc`. Install-step overrides remain unresolved. Public fallback expressions remain recorded as possible `fork-exception` paths: the pinned action validates the value and independently permits fallback only for public external-fork pull requests. Neither recognition is runtime proof or a repository allowlist. Local reusable workflows are already read at the same repository SHA and contribute their actual primary result instead of an automatic unknown. Missing targets and cycles remain reviewable. A remote reusable call can be resolved as having no observed JS install only when that workflow was already captured in the same organization's inventory at the requested default branch or exact SHA. A different pin/tag is never replaced with the current body. Remote callees with installs require caller-specific context and remain unresolved; no additional sources are fetched by this cross-reference step. diff --git a/tools/rollout/audit-cleanup.test.mjs b/tools/rollout/audit-cleanup.test.mjs index 21a8404..c181b01 100644 --- a/tools/rollout/audit-cleanup.test.mjs +++ b/tools/rollout/audit-cleanup.test.mjs @@ -16,6 +16,173 @@ const context = { visibility: "private", path: ".github/workflows/ci.yml" }; const inspect = (steps, extra = {}) => classifyJob("build", { steps, ...extra }, context, ["push"]); +test("matching immutable output guards cover only the same guarded installs", () => { + const ready = { id: "ready", run: 'echo enabled=true >> "$GITHUB_OUTPUT"' }; + for (const guard of [ + "steps.ready.outputs.enabled == 'true'", + "needs.prepare.outputs.enabled == 'true'", + ]) { + const guardedSetup = { ...setup, if: guard }; + const install = { run: "npm ci", if: `\${{ ${guard} }}` }; + const result = inspect([ready, guardedSetup, install]); + assert.equal(result.integration.disposition, "integrated"); + assert.equal(result.status, "unknown"); + assert.equal( + inspect([ready, guardedSetup, { run: "npm ci" }]).integration.disposition, + "needs-review", + ); + assert.equal( + inspect([ + ready, + guardedSetup, + { ...install, if: guard.replace("'true'", "'false'") }, + ]).integration.disposition, + "needs-review", + ); + assert.equal( + inspect([ready, { ...guardedSetup, "continue-on-error": true }, install]) + .integration.disposition, + "needs-review", + ); + assert.equal( + inspect([ready, { ...guardedSetup, env: { HOME: "/other" } }, install]) + .integration.disposition, + "needs-review", + ); + assert.equal( + inspect([ + ready, + guardedSetup, + { + uses: `workos/setup-socket-firewall/teardown@${APPROVED_RELEASE_SHA}`, + }, + install, + ]).integration.disposition, + "needs-sfw", + ); + } + const guardedSetup = { + ...setup, + if: "steps.ready.outputs.enabled == 'true'", + }; + const install = { run: "npm ci", if: guardedSetup.if }; + for (const steps of [ + [guardedSetup, ready, install], + [ready, guardedSetup, ready, install], + [ + ready, + guardedSetup, + { ...install, if: "steps.ready.outputs.enabled == 't r u e'" }, + ], + ]) + assert.equal(inspect(steps).integration.disposition, "needs-review"); + for (const guard of [ + "env.ENABLED == 'true'", + "always()", + "steps.ready.outputs.enabled == 't r u e'", + ]) { + assert.equal( + inspect([ready, { ...setup, if: guard }, { ...install, if: guard }]) + .integration.disposition, + "needs-review", + ); + } +}); + +test("literal npm workspaces preserve command classification and teardown gaps", () => { + const teardown = { + uses: `${setup.uses.split("@")[0]}/teardown@${APPROVED_RELEASE_SHA}`, + }; + for (const selector of [ + "-w apps/research", + "--workspace apps/research", + "--workspace=apps/research", + ]) { + const script = { run: `npm ${selector} run build` }; + assert.equal(inspect([script]).integration.disposition, "no-js-ci"); + assert.equal(inspect([script]).status, "unknown"); + for (const verb of ["ci", "exec -- wrangler --env production"]) { + const install = { run: `npm ${selector} ${verb}` }; + assert.equal( + inspect([setup, install]).integration.disposition, + "integrated", + ); + assert.equal(inspect([install]).integration.disposition, "needs-sfw"); + assert.equal( + inspect([setup, teardown, install]).integration.disposition, + "needs-sfw", + ); + } + } + for (const run of [ + "npm -w $APP run build", + "npm -w apps/${{ matrix.app }} run build", + "npm -w ../outside ci", + "npm -w --userconfig ci", + "npm -w apps/research --userconfig other ci", + "npm -w apps/research exec wrangler", + ]) + assert.notEqual( + inspect([setup, { run }]).integration.disposition, + "integrated", + run, + ); + assert.equal( + inspect([ + setup, + { run: "npm -w apps/research ci --registry=https://example.invalid" }, + ]).integration.disposition, + "needs-sfw", + ); +}); + +test("literal npx package selectors retain all download and configuration boundaries", () => { + for (const run of [ + "npx --yes --package renovate@43.257.6 renovate-config-validator --strict default.json", + "npx -p @scope/tool@1.2.3 tool --env $ENVIRONMENT", + "npx --package=tool --package=other tool --registry=https://payload.invalid", + "npm exec -- wrangler --env $ENVIRONMENT", + ]) { + const result = inspect([setup, { run }]); + assert.equal(result.integration.disposition, "integrated", run); + assert.equal(result.status, "unknown", run); + assert.equal(inspect([{ run }]).integration.disposition, "needs-sfw", run); + } + for (const run of [ + "npx --package=$PACKAGE tool", + "npx --package $RUNNER_TEMP/tool.tgz tool", + "npx --package=https://example.invalid/tool.tgz tool", + "npx --package tool", + "bunx --package tool tool", + "npm exec -- $TOOL", + "npm exec -- https://example.invalid/tool.tgz", + ]) + assert.equal( + inspect([setup, { run }]).integration.disposition, + "needs-review", + run, + ); + assert.equal( + inspect([setup, { run: "npx --package --userconfig tool" }]).integration + .disposition, + "needs-sfw", + ); + assert.equal( + inspect([ + setup, + { run: "npx --package tool --registry=https://example.invalid tool" }, + ]).integration.disposition, + "needs-sfw", + ); + assert.equal( + inspect([ + setup, + { run: "npm exec --registry=https://example.invalid -- tool" }, + ]).integration.disposition, + "needs-sfw", + ); +}); + test("literal install options preserve configuration, not execution assurance", () => { for (const run of [ "npm ci --include=optional", diff --git a/tools/rollout/classify.mjs b/tools/rollout/classify.mjs index 789c746..ef92d2d 100644 --- a/tools/rollout/classify.mjs +++ b/tools/rollout/classify.mjs @@ -174,11 +174,23 @@ function commandParts(command) { start += 1; assignments(true); } - return { - words: words.slice(start), - rawWords: tokens.commands.slice(start), - environment, - }; + const args = words.slice(start); + const rawArgs = tokens.commands.slice(start); + // A literal workspace selector changes the package, not the npm command. + // Leave dynamic/escaping selectors and other leading options unresolved. + while (args[0] === "npm") { + const separate = ["-w", "--workspace"].includes(args[1]); + const value = separate + ? args[2] + : args[1]?.startsWith("--workspace=") + ? args[1].slice(12) + : undefined; + if (!literalWorkingDirectory(value) || value.startsWith("-")) break; + const count = separate ? 2 : 1; + args.splice(1, count); + rawArgs.splice(1, count); + } + return { words: args, rawWords: rawArgs, environment }; } function commandWords(command) { @@ -307,9 +319,33 @@ function approvedBunArguments(words, rawWords) { } function directExecutor(words) { + const packageName = /^(?:@[\w.-]+\/)?[A-Za-z0-9_][\w.-]*(?:@[\w.^~*+-]+)?$/; + if (words[0] === "npm") { + // npm parses options after the command too, unless an explicit -- ends them. + return ["exec", "x"].includes(words[1]) && + words[2] === "--" && + packageName.test(words[3] ?? "") + ? words.slice(0, 3).join(" ") + : undefined; + } if (!["npx", "bunx"].includes(words[0])) return undefined; let target = 1; - while (["-y", "--yes"].includes(words[target])) target += 1; + while (target < words.length) { + if (["-y", "--yes"].includes(words[target])) target += 1; + else if ( + words[0] === "npx" && + ["-p", "--package"].includes(words[target]) && + packageName.test(words[target + 1] ?? "") + ) + target += 2; + else if ( + words[0] === "npx" && + words[target].startsWith("--package=") && + packageName.test(words[target].slice(10)) + ) + target += 1; + else break; + } // Installer options end at the literal package/binary target. Arguments to // that program are payload, not npx/bunx registry configuration. if (!/^[A-Za-z0-9_@][A-Za-z0-9_@./:+-]*$/.test(words[target] ?? "")) @@ -874,7 +910,10 @@ export function classifyStep(step, context = {}) { (configFileWrite && !knownRegistryContent) || nestedOverride || dynamicSetter; - if (["npx", "bunx"].includes(words[0])) { + if ( + ["npx", "bunx"].includes(words[0]) || + (words[0] === "npm" && ["exec", "x"].includes(words[1])) + ) { // Strict assurance still sees unknown executor code/configuration flags. operation.uncertain = true; operation.integrationExecutor = executorPrefix !== undefined; diff --git a/tools/rollout/integration.mjs b/tools/rollout/integration.mjs index 2b35240..6ddae5e 100644 --- a/tools/rollout/integration.mjs +++ b/tools/rollout/integration.mjs @@ -106,6 +106,32 @@ function contextUncertain(job, context) { ); } +// Only boolean-string output comparisons are modeled. Step outputs must belong +// to one completed earlier step, never a future/duplicate ID or mutable env. +function stableOutputGuard(value, job, step) { + if (typeof value !== "string") return undefined; + const text = value + .trim() + .replace(/^\$\{\{([\s\S]*)\}\}$/, "$1") + .trim(); + const match = text.match( + /^(steps|needs)\.([\w-]+)\.outputs\.([\w-]+)\s*==\s*(['"])(true|false)\4$/, + ); + if (!match) return undefined; + if (match[1] === "steps") { + const indexes = job.steps.flatMap((item, index) => + item?.id === match[2] ? [index] : [], + ); + if ( + indexes.length !== 1 || + !Number.isInteger(step) || + indexes[0] >= step - 1 + ) + return undefined; + } + return `${match[1]}.${match[2]}.outputs.${match[3]}==${match[5]}`; +} + export function observedIntegration(operations, job, context) { const downloads = []; const additionalJsPaths = []; @@ -133,8 +159,18 @@ export function observedIntegration(operations, job, context) { operation.integrationConfigurationUncertain ?? operation.integrationUncertain ?? operation.uncertain; - const currentState = + const observedState = operation.manager === "bun" && bunState ? bunState : state; + const currentState = + observedState?.guard && + observedState.guard !== + stableOutputGuard( + job.steps?.[operation.step - 1]?.if, + job, + operation.step, + ) + ? { status: "unresolved", reason: "unmatched-setup-condition" } + : observedState; const validInterval = ["covered", "fork-exception"].includes( currentState?.status, ); @@ -162,8 +198,12 @@ export function observedIntegration(operations, job, context) { context.visibility === "public" && (operation.fallback === "true" || /^\$\{\{.+\}\}$/.test(operation.fallback)); + const guard = + !operation.via && !operation.boundaryUncertainWithoutCondition + ? stableOutputGuard(operation.condition, job, operation.step) + : undefined; if ( - uncertain || + (uncertain && !guard) || (!publicForkFallback && !["false", "true"].includes(operation.fallback)) ) { state = { @@ -184,6 +224,7 @@ export function observedIntegration(operations, job, context) { state = { status: publicForkFallback ? "fork-exception" : "covered", reason: "approved-setup-interval", + ...(guard ? { guard } : {}), }; } continue; diff --git a/tools/rollout/integration.test.mjs b/tools/rollout/integration.test.mjs index 1016d96..dffb172 100644 --- a/tools/rollout/integration.test.mjs +++ b/tools/rollout/integration.test.mjs @@ -587,6 +587,7 @@ test("direct executor target arguments are not installer configuration", () => { for (const run of [ "npx wrangler deploy --env production", "npx -y wrangler deploy --env production", + "npx --package=tool tool", "bunx biome check --write", "npx tool --registry=https://example.invalid", "npx tool --env $ENVIRONMENT", @@ -607,7 +608,7 @@ test("direct executor target arguments are not installer configuration", () => { } for (const run of [ "npx --future-option tool", - "npx --package=tool tool", + "npx --package=$PACKAGE tool", "npx $TOOL --env production", "npx", "npm exec tool -- --env production", From 23594323d1f2bb6354d7e5ea174ebb74c275d016 Mon Sep 17 00:00:00 2001 From: "workos-tars[bot]" <269013284+workos-tars[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:04:25 +0000 Subject: [PATCH 10/19] feat(audit): track pinned exclusions and bound remaining CI forms --- README.md | 19 ++- tools/rollout/audit-cleanup.test.mjs | 140 ++++++++++++++++++- tools/rollout/audit.mjs | 14 +- tools/rollout/classify.mjs | 177 +++++++++++++++++++++--- tools/rollout/exclusions.mjs | 104 ++++++++++++++ tools/rollout/exclusions.test.mjs | 200 +++++++++++++++++++++++++++ tools/rollout/integration.mjs | 40 +++++- tools/rollout/integration.test.mjs | 4 +- 8 files changed, 660 insertions(+), 38 deletions(-) create mode 100644 tools/rollout/exclusions.mjs create mode 100644 tools/rollout/exclusions.test.mjs diff --git a/README.md b/README.md index 4e252bc..a2ad902 100644 --- a/README.md +++ b/README.md @@ -214,19 +214,26 @@ npm run audit:live # read-only, default-branch workflow candidate scan Terminal output is sanitized JSON counts, not repository names or workflow source. Full inventory and audit JSON go to `reports/inventory.json` and `reports/live-audit.json`, respectively. These paths are ignored; reports are atomically replaced with owner-only (`0600`) permissions. Keep them private: never force-add, publish, or upload reports as CI artifacts. Both commands take no CLI flags; a later run replaces the previous report. -Audit JSON schema **2** separates primary `dispositions` (observed dependency-install integration) from `assuranceDispositions` (strict execution/security review). Successfully inspected repositories retain both dispositions, their `headSha`, workflow/job operations and violations; read errors remain explicit error rows. Each job's `integration` includes per-download `covered`, `fork-exception`, `gap` or `unresolved` configuration evidence, reason codes and diagnostic notes. `additionalJsPaths` records explicit JS invocations outside the direct-install grammar, distinguishing `setup-observed` from unresolved configuration. `runtimeVerification` is always `not-performed`. `scanStatus: complete` means the token-visible scan completed, **not** that all jobs are protected. `partial` means one or more repositories have an `audit-error` row. CLI exit **0** means the scan completed, even when gaps/review candidates were found; exit **1** means an operational failure (including partial scans). Consumers should inspect dispositions separately from the exit code. Fatal inventory/API/command errors emit a sanitized error and exit 1; they do not refresh the report, so check its timestamp before use. +Audit JSON schema **3** separates primary `dispositions` (observed dependency-install integration) from `assuranceDispositions` (strict execution/security review). Successfully inspected repositories retain both dispositions, their `headSha`, workflow/job operations and violations; read errors remain explicit error rows. Each job's `integration` includes per-download `covered`, `covered-with-exclusion`, `fork-exception`, `gap` or `unresolved` configuration evidence, reason codes and diagnostic notes. `additionalJsPaths` records explicit JS invocations outside the direct-install grammar, distinguishing `setup-observed` from unresolved configuration. `runtimeVerification` is always `not-performed`. `scanStatus: complete` means the token-visible scan completed, **not** that all jobs are protected. `partial` means one or more repositories have an `audit-error` row. CLI exit **0** means the scan completed, even when gaps/review candidates were found; exit **1** means an operational failure (including partial scans). Consumers should inspect dispositions separately from the exit code. Fatal inventory/API/command errors emit a sanitized error and exit 1; they do not refresh the report, so check its timestamp before use. | Repository disposition | Meaning | | --- | --- | | `needs-sfw` | A recognized download has a definite missing/unsupported setup interval in its own job. | | `needs-review` | Install configuration, an explicit JS invocation, or relevant workflow/local-action source could not be resolved. Ordinary opaque scripts alone do not cause this result. | | `integrated` | SFW configuration is observed for the identified install paths, possibly with an explicit public-fork exception. This is not execution or traffic assurance. | +| `integrated-with-exclusions` | Observed integration has an approved, source-pinned exclusion. Inspect the repository's `exclusions`; the excluded source is not inspected by SFW. | | `audit-error` | A repository read failed or was incomplete; never a clean result. | | `no-ci`, `no-js-ci`, `empty` | No observed in-scope download candidate; not proof that remote actions or arbitrary code cannot download packages. | Primary integration is not a rollout percentage or a certification of every command. For example, setup → `npm ci` → `npm test` can be `integrated` while strict assurance remains `needs-review`. Without setup, the same direct install remains a `needs-sfw` candidate even though the later test is opaque. A new uncovered install job cannot be masked by an integrated sibling. An ordinary script-only sibling does not invent another install path: its code remains unverified in assurance diagnostics. -Strict job `status` and `assuranceDisposition` retain opaque execution and security findings (`blocked-trust`, `unsafe-publish`, `blocked-yarn`, etc.). **Inspect those findings separately; `integrated` does not dismiss them.** A strict `protected` or `safe-publish` status remains static evidence, not runtime or lifecycle-hook proof. Schema1 consumers must not interpret schema2 primary dispositions as the old assurance verdicts. +Strict job `status` and `assuranceDisposition` retain opaque execution and security findings (`blocked-trust`, `unsafe-publish`, `blocked-yarn`, etc.). **Inspect those findings separately; `integrated` does not dismiss them.** A strict `protected` or `safe-publish` status remains static evidence, not runtime or lifecycle-hook proof. Schema1 consumers must not interpret newer primary dispositions as the old assurance verdicts. Schema2 consumers must handle schema3's `integrated-with-exclusions` disposition and `covered-with-exclusion` download status explicitly, not count them as fully covered. + +### Approved source exclusions + +`tools/rollout/exclusions.mjs` records the approved `tree-sitter-kotlin` GitHub archive in `oagen` and `oagen-emitters`, including its exact path, version, URL, integrity pin and approval provenance. This archive intentionally downloads outside SFW; npm-registry dependencies still use the firewall. The existing `NPM_CONFIG_REPLACE_REGISTRY_HOST=npmjs` setting is retained because rewriting GitHub archives to the npm proxy breaks installation. + +Each scan reads the manifest and version-3 lockfile at the same captured repository SHA. Manifest dependency drift, workspace/override configuration, a changed pin or any additional non-npm source makes the exclusion `stale` and keeps the repository in review; an unreadable lockfile is an audit error. Only direct, repository-root `npm install`/`npm ci` steps with that exact environment setting in the recorded workflows receive the exclusion, after one unconditional default-source checkout. Alternate checkout refs/repositories/paths, nested working directories and composite-action installs are not excused. Missing setup, additional gaps, unknown execution contexts and other configuration overrides remain findings. Qualified results use `integrated-with-exclusions`, never an unqualified protection claim. Strict assurance remains unchanged. ### Static limits @@ -236,17 +243,19 @@ The small grammar recognizes npm/pnpm/Bun installs, download-capable executors, Configuration checks retain the approved action SHA, visibility-appropriate token, Bun configuration and public-fork exception rules. Disabled or continued-on-error setup cannot silently cover an install. Conditional setup covers only an install with the same recognized boolean-string output comparison: a dependency job output, or an output from a uniquely identified earlier step. Future/duplicate step IDs, mutable environment guards, status functions and unmatched conditions remain unresolved; strict execution assurance is unchanged. Corepack uncertainty survives conditional or targeted controls. The action-exported `SFW_BUN_CONFIG_PATH` is recognized in Bun's `--config` argument when Bun setup is enabled; explicit overrides of that variable remain configuration questions. Whole-value composite input references in `with`/`env` are bound to caller values/defaults without evaluating conditions or interpolating shell text. Missing/malformed local source, cycles and expansion limits remain review findings. -Known registry writes and teardown invalidate the relevant configuration; a command-local override does not contaminate later installs. npm/pnpm registry setters, configuration-file writes (including `tee`), standalone relevant assignments and GitHub environment-file writes remain findings rather than being treated as generic opacity. Reading a configuration variable is not a mutation. HOME/PATH/NODE_OPTIONS overrides, relevant exported variables within a run block, unsupported environment wrappers, dynamic mappings and job containers remain configuration questions. Ordinary application secrets/settings, version metadata and sibling service containers do not invalidate host setup ordering. Job-level guards affect shared reachability; explicit status predicates and conditional configuration transitions remain conservative. Exact npm `--replace-registry-host=always` preserves the configured registry; other modes require review. +Known registry writes and teardown invalidate the relevant configuration; a command-local override does not contaminate later installs. npm/pnpm registry setters, configuration-file writes (including `tee`), standalone relevant assignments and GitHub environment-file writes remain findings rather than being treated as generic opacity. Reading a configuration variable is not a mutation. HOME/PATH/NODE_OPTIONS overrides, relevant exported variables within a run block, unsupported environment wrappers, dynamic mappings and job containers remain configuration questions. Ordinary application secrets/settings, version metadata and sibling service containers do not invalidate host setup ordering. Job-level guards affect shared reachability; explicit status predicates and conditional configuration transitions remain conservative. Exact npm `--replace-registry-host=always` preserves the configured registry; other modes require review unless covered by an approved source exclusion. Literal repository-relative default working directories and npm `--prefix` arguments after the install verb are treated like explicit run-step working directories. Supported install options include `--include=optional`, `--omit=dev`, `--package-lock=false` and Bun's literal `--os="*" --cpu="*"`; unknown options and dynamic or escaping directory paths remain reviewable. Root local actions (`uses: ./`) are read from `action.yml` or `action.yaml` at the captured SHA. Entirely commented-out workflow files have no active jobs; malformed or missing source still requires review. -Literal leading npm workspace selectors (`-w`, `--workspace`, and `--workspace=`) retain the underlying command classification: scripts stay unverified script execution, while installs and explicit `npm exec -- ` remain download candidates. Literal npm package selectors in `npx --package`/`-p` are recognized without treating executable payload flags as installer configuration. Dynamic selectors, unsupported options, and post-teardown executor downloads remain findings. +Literal leading npm workspace selectors (`-w`, `--workspace`, and `--workspace=`) retain the underlying command classification: scripts stay unverified script execution, while installs and explicit `npm exec -- ` remain download candidates. Literal npm package selectors in `npx --package`/`-p` are recognized without treating executable payload flags as installer configuration. A workspace selector may contain a matrix component only when every declared alternative is a literal, safe single path component and the matrix has no `include` overrides; this does not verify any script body. Runner-temp tarball arguments such as `"$RUNNER_TEMP"/package-*.tgz` are local inputs to npm/npx, not alternate registry URLs; overriding `RUNNER_TEMP` invalidates that recognition's configuration evidence. Other dynamic selectors, unsupported options, and post-teardown executor downloads remain findings. + +The exact standard shell template `bash --noprofile --norc -euo pipefail {0}` is recognized alongside `bash`/`sh`. A clean-room `env -i` npm invocation is recognized only when it retains the exact inherited HOME, PATH, action registry and npm-config path; missing, changed, duplicated or additional configuration variables remain unresolved. Local `npm version`, argument-free/format-only `npm pack`, and literal Bun script filenames remain unverified execution, not invented download operations. Remote or dynamic `npm pack` targets remain review candidates. A setup-only `NPM_CONFIG_USERCONFIG` pointing to a named file under `${{ runner.temp }}` is recognized because the pinned action validates that path and configures both it and `HOME/.npmrc`. Install-step overrides remain unresolved. Public fallback expressions remain recorded as possible `fork-exception` paths: the pinned action validates the value and independently permits fallback only for public external-fork pull requests. Neither recognition is runtime proof or a repository allowlist. Local reusable workflows are already read at the same repository SHA and contribute their actual primary result instead of an automatic unknown. Missing targets and cycles remain reviewable. A remote reusable call can be resolved as having no observed JS install only when that workflow was already captured in the same organization's inventory at the requested default branch or exact SHA. A different pin/tag is never replaced with the current body. Remote callees with installs require caller-specific context and remain unresolved; no additional sources are fetched by this cross-reference step. -Only top-level `.github/workflows/*.yml|yaml`, `.depot/workflows/*.yml|yaml` and referenced local actions in the captured scope are inspected. The detector does not execute code, inspect arbitrary script/action implementations, validate credentials, prove traffic routing or certify bespoke controls. Security findings and unverified execution remain separate assurance diagnostics. No historical cohort, coverage percentage or repository exception allowlist is used. +Only top-level `.github/workflows/*.yml|yaml`, `.depot/workflows/*.yml|yaml` and referenced local actions in the captured scope are inspected. The detector does not execute code, inspect arbitrary script/action implementations, validate credentials, prove traffic routing or certify bespoke controls. Security findings and unverified execution remain separate assurance diagnostics. No historical cohort, coverage percentage or blanket repository allowlist is used. ### Manual release snapshot verification diff --git a/tools/rollout/audit-cleanup.test.mjs b/tools/rollout/audit-cleanup.test.mjs index c181b01..dd176f4 100644 --- a/tools/rollout/audit-cleanup.test.mjs +++ b/tools/rollout/audit-cleanup.test.mjs @@ -16,6 +16,144 @@ const context = { visibility: "private", path: ".github/workflows/ci.yml" }; const inspect = (steps, extra = {}) => classifyJob("build", { steps, ...extra }, context, ["push"]); +test("workspace matrix selectors require bounded literal values", () => { + const strategy = { matrix: { app: ["research", "incident-watch"] } }; + for (const run of [ + "npm -w apps/${{ matrix.app }} run build", + 'npm --workspace="apps/${{ matrix.app }}" run typecheck', + ]) { + assert.equal( + inspect([setup, { run: "npm ci" }, { run }], { strategy }).integration + .disposition, + "integrated", + ); + assert.equal(inspect([{ run }], { strategy }).status, "unknown"); + for (const matrix of [ + { app: ["research; npm install evil"] }, + { app: ["../elsewhere"] }, + { app: ["--registry=elsewhere"] }, + { app: [] }, + { app: ["research"], include: [{ app: "unsafe value" }] }, + "${{ fromJSON(needs.matrix.outputs.apps) }}", + ]) + assert.equal( + inspect([{ run }], { strategy: { matrix } }).integration.disposition, + "needs-review", + ); + } + const run = "npm -w apps/${{ matrix.app }} install"; + assert.equal( + inspect([{ run }], { strategy }).integration.disposition, + "needs-sfw", + ); + assert.equal( + inspect([setup, { run }], { strategy }).integration.disposition, + "integrated", + ); +}); + +test("runner-temp tarballs stay local, without excusing runner-temp overrides", () => { + for (const run of [ + 'npm install --no-save --ignore-scripts "$RUNNER_TEMP"/example-*.tgz', + 'npx --yes --package "$RUNNER_TEMP"/example-*.tgz example --list', + ]) { + assert.equal( + inspect([setup, { run }]).integration.disposition, + "integrated", + ); + assert.equal(inspect([{ run }]).integration.disposition, "needs-sfw"); + assert.equal( + inspect([setup, { run }], { + env: { RUNNER_TEMP: "https://example.invalid" }, + }).integration.disposition, + "needs-review", + ); + assert.equal( + inspect([setup, { run: `RUNNER_TEMP=https://example.invalid\n${run}` }]) + .integration.disposition, + "needs-review", + ); + } + for (const target of [ + "$ARTIFACTS/example.tgz", + "$RUNNER_TEMP/../example.tgz", + "$RUNNER_TEMP/$PACKAGE.tgz", + ]) + assert.equal( + inspect([setup, { run: `npm install "${target}"` }]).integration + .disposition, + "needs-review", + ); +}); + +test("clean-room npm preserves only the exact inherited registry and config", () => { + const prefix = + 'env -i CI=true HOME="$HOME" PATH="$PATH" NPM_CONFIG_REGISTRY="${NPM_CONFIG_REGISTRY:?}" NPM_CONFIG_USERCONFIG="${NPM_CONFIG_USERCONFIG:-$HOME/.npmrc}"'; + for (const command of ["npm ci", "npm ci --prefix web"]) { + const run = `${prefix} ${command}`; + assert.equal( + inspect([setup, { run }]).integration.disposition, + "integrated", + ); + assert.equal(inspect([{ run }]).integration.disposition, "needs-sfw"); + assert.equal(inspect([setup, { run }]).status, "unknown"); + } + for (const run of [ + `${prefix.replace('PATH="$PATH"', 'PATH="/other"')} npm ci`, + `${prefix.replace('HOME="$HOME"', 'HOME="/other"')} npm ci`, + `${prefix.replace('"${NPM_CONFIG_REGISTRY:?}"', '"https://example.invalid"')} npm ci`, + `${prefix.replace('"${NPM_CONFIG_USERCONFIG:-$HOME/.npmrc}"', '"/other/.npmrc"')} npm ci`, + `${prefix} NODE_OPTIONS=--require=evil.cjs npm ci`, + `${prefix} HOME="$HOME" npm ci`, + "env -i npm ci", + ]) + assert.notEqual( + inspect([setup, { run }]).integration.disposition, + "integrated", + run, + ); +}); + +test("standard explicit Bash and local script commands are not unknown installer options", () => { + const shell = "bash --noprofile --norc -euo pipefail {0}"; + assert.equal( + inspect([setup, { run: "npm ci", shell }]).integration.disposition, + "integrated", + ); + assert.equal( + inspect([{ run: "npm ci", shell }]).integration.disposition, + "needs-sfw", + ); + for (const custom of ["bash --rcfile /other {0}", "python {0}"]) { + assert.equal( + inspect([setup, { run: "npm ci", shell: custom }]).integration + .disposition, + "needs-review", + ); + } + for (const run of [ + 'npm version "$VERSION" --no-git-tag-version', + "npm pack --json", + "packed=$(npm pack --json)", + "bun scripts/build-binary.ts --target $TARGET", + ]) { + const result = inspect([{ run }]); + assert.equal(result.integration.disposition, "no-js-ci", run); + assert.equal(result.status, "unknown", run); + } + for (const run of [ + "npm pack @scope/remote", + "npm pack $PACKAGE", + "npm pack --pack-destination $DIR", + "bun $SCRIPT", + ]) + assert.equal( + inspect([{ run }]).integration.disposition, + "needs-review", + run, + ); +}); + test("matching immutable output guards cover only the same guarded installs", () => { const ready = { id: "ready", run: 'echo enabled=true >> "$GITHUB_OUTPUT"' }; for (const guard of [ @@ -150,7 +288,7 @@ test("literal npx package selectors retain all download and configuration bounda } for (const run of [ "npx --package=$PACKAGE tool", - "npx --package $RUNNER_TEMP/tool.tgz tool", + "npx --package $ARTIFACTS/tool.tgz tool", "npx --package=https://example.invalid/tool.tgz tool", "npx --package tool", "bunx --package tool tool", diff --git a/tools/rollout/audit.mjs b/tools/rollout/audit.mjs index f4244fb..756f8e6 100644 --- a/tools/rollout/audit.mjs +++ b/tools/rollout/audit.mjs @@ -4,6 +4,7 @@ import { dirname, join } from "node:path"; import { classifyWorkflow, repositoryDisposition } from "./classify.mjs"; import { captureRepositoryInventory } from "./inventory.mjs"; import { ORGANIZATION } from "./constants.mjs"; +import { readRegistryExclusions } from "./exclusions.mjs"; import { integrationDisposition, resolveLocalWorkflowCalls, @@ -212,11 +213,19 @@ export async function auditRepository(client, repository) { ); } + const exclusions = await readRegistryExclusions( + repository.name, + readSource, + ); const workflows = resolveLocalWorkflowCalls( workflowPaths.map((path, index) => classifyWorkflow(workflowTexts[index], { localActions, path, + registryExclusion: exclusions.find( + (entry) => + entry.status === "matched" && entry.workflows.includes(path), + ), visibility: repository.visibility, }), ), @@ -238,8 +247,9 @@ export async function auditRepository(client, repository) { return { defaultBranch: repository.defaultBranch, - disposition: integrationDisposition(workflows), + disposition: integrationDisposition(workflows, exclusions), assuranceDisposition: repositoryDisposition(workflows), + exclusions, headSha, jobs: { byStatus: Object.fromEntries( @@ -322,7 +332,7 @@ export async function runAudit(client, options = {}) { coverage: "token-visible repositories only; organization-wide access is an operator prerequisite", repositories: rows, - schemaVersion: 2, + schemaVersion: 3, }; } diff --git a/tools/rollout/classify.mjs b/tools/rollout/classify.mjs index ef92d2d..849c69f 100644 --- a/tools/rollout/classify.mjs +++ b/tools/rollout/classify.mjs @@ -9,9 +9,11 @@ import { literalWorkingDirectory, observedIntegration, unresolvedJsInvocation, + SUPPORTED_SHELLS, } from "./integration.mjs"; const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/; +const LOCAL_TARBALL = /^\$(?:RUNNER_TEMP|\{RUNNER_TEMP\})\/[\w.*-]+\.tgz$/; const UNSAFE_PUBLIC_TRIGGERS = new Set([ "issue_comment", "pull_request_target", @@ -152,7 +154,7 @@ function expectedTokenExpression(visibility) { return `\${{secrets.${secret}}}`; } -function commandParts(command) { +function commandParts(command, context = {}) { const tokens = shellTokens(command); const words = tokens.words; const environment = {}; @@ -170,6 +172,41 @@ function commandParts(command) { } }; assignments(); + let preservedEnvironment = false; + if (words[start] === "env" && words[start + 1] === "-i") { + // This clean-room npm invocation retains the exact action-configured paths + // and registry. Every other env -i shape remains unresolved. + const expected = { + HOME: '"$HOME"', + PATH: '"$PATH"', + NPM_CONFIG_REGISTRY: '"${NPM_CONFIG_REGISTRY:?}"', + NPM_CONFIG_USERCONFIG: '"${NPM_CONFIG_USERCONFIG:-$HOME/.npmrc}"', + }; + const values = new Map(); + let end = start + 2; + let duplicate = false; + while (/^[A-Za-z_][A-Za-z0-9_]*=/.test(words[end] ?? "")) { + const raw = tokens.commands[end]; + const key = raw.slice(0, raw.indexOf("=")); + duplicate ||= values.has(key); + values.set(key, raw.slice(key.length + 1)); + end += 1; + } + if ( + !duplicate && + words[end] === "npm" && + Object.entries(expected).every( + ([key, value]) => values.get(key) === value, + ) && + [...values].every( + ([key, value]) => + Object.hasOwn(expected, key) || (key === "CI" && value === "true"), + ) + ) { + start = end; + preservedEnvironment = true; + } + } if (words[start] === "env" && !words[start + 1]?.startsWith("-")) { start += 1; assignments(true); @@ -185,16 +222,21 @@ function commandParts(command) { : args[1]?.startsWith("--workspace=") ? args[1].slice(12) : undefined; - if (!literalWorkingDirectory(value) || value.startsWith("-")) break; + const bounded = value?.replace( + /\$\{\{\s*matrix\.([\w-]+)\s*\}\}/g, + (match, key) => + context.literalMatrixKeys?.has(key) ? "matrix-value" : match, + ); + if (!literalWorkingDirectory(bounded) || bounded.startsWith("-")) break; const count = separate ? 2 : 1; args.splice(1, count); rawArgs.splice(1, count); } - return { words: args, rawWords: rawArgs, environment }; + return { words: args, rawWords: rawArgs, environment, preservedEnvironment }; } -function commandWords(command) { - return commandParts(command).words; +function commandWords(command, context) { + return commandParts(command, context).words; } // Literal logging only: quoted source, substitution and shell controls are not @@ -324,7 +366,7 @@ function directExecutor(words) { // npm parses options after the command too, unless an explicit -- ends them. return ["exec", "x"].includes(words[1]) && words[2] === "--" && - packageName.test(words[3] ?? "") + (packageName.test(words[3] ?? "") || LOCAL_TARBALL.test(words[3] ?? "")) ? words.slice(0, 3).join(" ") : undefined; } @@ -335,20 +377,25 @@ function directExecutor(words) { else if ( words[0] === "npx" && ["-p", "--package"].includes(words[target]) && - packageName.test(words[target + 1] ?? "") + (packageName.test(words[target + 1] ?? "") || + LOCAL_TARBALL.test(words[target + 1] ?? "")) ) target += 2; else if ( words[0] === "npx" && words[target].startsWith("--package=") && - packageName.test(words[target].slice(10)) + (packageName.test(words[target].slice(10)) || + LOCAL_TARBALL.test(words[target].slice(10))) ) target += 1; else break; } // Installer options end at the literal package/binary target. Arguments to // that program are payload, not npx/bunx registry configuration. - if (!/^[A-Za-z0-9_@][A-Za-z0-9_@./:+-]*$/.test(words[target] ?? "")) + if ( + !/^[A-Za-z0-9_@][A-Za-z0-9_@./:+-]*$/.test(words[target] ?? "") && + !LOCAL_TARBALL.test(words[target] ?? "") + ) return undefined; return words.slice(0, target).join(" "); } @@ -366,8 +413,8 @@ function firstSubcommand(words) { return words.slice(1).find((word) => !word.startsWith("-")); } -export function classifyCommand(command) { - const words = commandWords(command); +export function classifyCommand(command, context) { + const words = commandWords(command, context); const program = words[0]; if (!program) { return { command, kind: "no-network" }; @@ -753,6 +800,19 @@ export function classifyStep(step, context = {}) { integrationEnv = { ...step.env }; delete integrationEnv.NPM_CONFIG_USERCONFIG; } + const registryExclusion = + context.registryExclusion && + !context.actionStack?.length && + [undefined, ".", "./"].includes(step["working-directory"]) && + isMapping(integrationEnv) && + integrationEnv.NPM_CONFIG_REPLACE_REGISTRY_HOST === "npmjs" && + /^\s*npm\s+(?:ci|install)\s*$/.test(step.run ?? "") + ? context.registryExclusion.id + : undefined; + if (registryExclusion) { + integrationEnv = { ...integrationEnv }; + delete integrationEnv.NPM_CONFIG_REPLACE_REGISTRY_HOST; + } let integrationUncertain = boundaryUncertain({ ...step, env: undefined, if: primaryRunGuard(step) }) || environmentUncertain(integrationEnv); @@ -771,20 +831,21 @@ export function classifyStep(step, context = {}) { /[;'"$`|<>(){}\\]|(?:^|\s)(?:if|then|else|fi|for|while|case|eval|source|sudo)(?:\s|$)|(? /^(?:if|case|for|while|until|select)\b/.test(command), ); operations = shell.commands.map((command) => { - const operation = classifyCommand(command); - const { words, rawWords, environment } = commandParts(command); + const operation = classifyCommand(command, context); + const { words, rawWords, environment, preservedEnvironment } = + commandParts(command, context); const bunArguments = approvedBunArguments(words, rawWords); const tokens = shellTokens(command); const nestedCommands = tokens.substitutions.flatMap( @@ -905,7 +966,13 @@ export function classifyStep(step, context = {}) { (unsupportedFlags && !registryMutating) || (operation.kind === "js-public-download" && /[$`]/.test( - words.filter((_, index) => !bunArguments.has(index)).join(" "), + words + .filter( + (word, index) => + !bunArguments.has(index) && + !(words[0] === "npm" && LOCAL_TARBALL.test(word)), + ) + .join(" "), )) || (configFileWrite && !knownRegistryContent) || nestedOverride || @@ -995,7 +1062,7 @@ export function classifyStep(step, context = {}) { operation.integrationUncertain || (operation.explicitJsInvocation && nestedEnvironment) || (directInvocation && !operation.manager) || - (step.shell !== undefined && !["bash", "sh"].includes(step.shell)) || + (step.shell !== undefined && !SUPPORTED_SHELLS.has(step.shell)) || (operation.kind === "unknown-wrapper" && operation.manager !== undefined && (words[1]?.startsWith("-") || /[$`]/.test(words[1] ?? ""))) || @@ -1003,7 +1070,8 @@ export function classifyStep(step, context = {}) { /^(?:\.?\.?\/|\/)[^\s]*\/(?:npm|pnpm|npx|bun|bunx|yarn|corepack)\b/.test( command, ) || - (/^(?:env|sudo|command|time|exec)$/.test(words[0] ?? "") && + (!preservedEnvironment && + /^(?:env|sudo|command|time|exec)$/.test(words[0] ?? "") && /(?:^|\s)(?:npm|pnpm|npx|bun|bunx|yarn|corepack)\b/.test(command)) || (operation.registryMutating && shell.ambiguous); operation.integrationSyntaxUncertain = shell.ambiguous; @@ -1024,6 +1092,7 @@ export function classifyStep(step, context = {}) { } return operations.map((operation) => ({ ...operation, + ...(registryExclusion ? { registryExclusion } : {}), ...(uncertain || operation.uncertain ? { uncertain: true } : {}), integrationUncertain: integrationUncertain || @@ -1042,7 +1111,18 @@ function scriptInvocation(words, kind) { kind === "unknown-wrapper" && ["npm", "pnpm", "bun", "yarn"].includes(words[0]) && (/^(?:run|test|start|stop|restart)$/.test(words[1] ?? "") || - (words[0] === "bun" && words[1] === "build") || + (words[0] === "bun" && + (words[1] === "build" || + (literalWorkingDirectory(words[1]) && + /\.[cm]?[jt]sx?$/.test(words[1])))) || + (words[0] === "npm" && + (words[1] === "version" || + (words[1] === "pack" && + words + .slice(2) + .every((arg) => + ["--json", "--ignore-scripts", "--dry-run"].includes(arg), + )))) || (["pnpm", "yarn"].includes(words[0]) && /^[A-Za-z0-9_:.+-]+$/.test(words[1] ?? "") && !/^(?:config|env|setup|set|create|self-update|plugin|policies|patch|patch-commit)$/.test( @@ -1278,12 +1358,62 @@ export function classifyJob(jobName, job, context = {}, triggers = []) { } const steps = Array.isArray(job.steps) ? job.steps : []; - const stepContext = { ...context, stepBudget: { remaining: 1000 } }; + const matrix = job.strategy?.matrix; + const literalMatrixKeys = new Set( + matrix && + typeof matrix === "object" && + !Array.isArray(matrix) && + matrix.include === undefined + ? Object.entries(matrix) + .filter( + ([, values]) => + Array.isArray(values) && + values.length > 0 && + values.every( + (value) => + typeof value === "string" && + /^[A-Za-z0-9_][\w.-]*$/.test(value), + ), + ) + .map(([key]) => key) + : [], + ); + const checkouts = steps + .map((step, index) => ({ step, index })) + .filter( + ({ step }) => + typeof step?.uses === "string" && + step.uses.startsWith("actions/checkout@"), + ); + const checkout = checkouts[0]; + const defaultCheckout = + checkouts.length === 1 && + condition(checkout.step.if) === "always" && + condition(checkout.step["continue-on-error"] ?? false) === "never" && + ["ref", "repository", "path", "sparse-checkout"].every( + (key) => checkout.step.with?.[key] === undefined, + ); + const stepContext = { + ...context, + registryExclusion: + defaultCheckout && + context.exclusionRootDirectory !== false && + [undefined, ".", "./"].includes(job.defaults?.run?.["working-directory"]) + ? context.registryExclusion + : undefined, + literalMatrixKeys, + stepBudget: { remaining: 1000 }, + }; const operations = condition(job.if) === "never" ? [] : steps.flatMap((step, index) => - classifyStep(step, stepContext).map((operation) => ({ + classifyStep( + step, + index > (checkout?.index ?? -1) + ? stepContext + : { ...stepContext, registryExclusion: undefined }, + ).map((operation) => ({ ...operation, step: index + 1, })), @@ -1296,7 +1426,7 @@ export function classifyJob(jobName, job, context = {}, triggers = []) { boundaryUncertain(job) || context.workflowUncertain || (job.defaults?.run?.shell !== undefined && - !["bash", "sh"].includes(job.defaults.run.shell)) + !SUPPORTED_SHELLS.has(job.defaults.run.shell)) ) { operations.push({ kind: "unknown", @@ -1429,6 +1559,9 @@ export function classifyWorkflow(text, context) { job, { ...context, + exclusionRootDirectory: [undefined, ".", "./"].includes( + workflow.defaults?.run?.["working-directory"], + ), workflowUncertain: workflow.env !== undefined || workflow.defaults !== undefined, integrationContextUncertain: diff --git a/tools/rollout/exclusions.mjs b/tools/rollout/exclusions.mjs new file mode 100644 index 0000000..1755121 --- /dev/null +++ b/tools/rollout/exclusions.mjs @@ -0,0 +1,104 @@ +import { isDeepStrictEqual } from "node:util"; + +// Approved source exclusions, not repository exemptions. A changed archive, +// integrity pin, or additional non-npm source must be reviewed again. +export const REGISTRY_EXCLUSIONS = ["oagen", "oagen-emitters"].map( + (repository) => ({ + id: `${repository}-tree-sitter-kotlin`, + repository, + lockfile: "package-lock.json", + packagePath: "node_modules/tree-sitter-kotlin", + version: "0.4.0", + resolved: + "https://github.com/fwcd/tree-sitter-kotlin/archive/f66d2908542e93c0204c6c241f794afe4e9cd5d1.tar.gz", + integrity: + "sha512-7pk1Tg/gXh+6hM4E0F2rPKeLyA/bNlYyqVu6T1Md/AV/vloakbvEZG0R4CYwUfVtgFAMZRbOtA8JBzX1SFatBA==", + workflows: [ + ".github/workflows/ci.yml", + ".github/workflows/lint.yml", + ".github/workflows/release.yml", + ], + reason: + "The pinned tree-sitter-kotlin GitHub archive intentionally downloads outside Socket Firewall; npm-registry dependencies still use SFW.", + approvedBy: "matt.peake@workos.com", + approvalRequestId: "24cb3145-70c0-471a-86b6-3f5d7b5860a0", + approvalUrl: + "https://tars.workos.tools/conversations/pi_ae1489a3e5ee424ebd2cac9260cbad13", + }), +); + +export async function readRegistryExclusions(repository, readSource) { + const rule = REGISTRY_EXCLUSIONS.find( + (entry) => entry.repository === repository, + ); + if (!rule) return []; + // Read errors are audit errors, not permission to apply an exclusion. + const lock = JSON.parse(await readSource(rule.lockfile)); + const manifest = JSON.parse(await readSource("package.json")); + const packages = lock?.packages; + // npm install can change the lock: do not excuse an unrecorded manifest edit. + const manifestMatches = + manifest && + typeof manifest === "object" && + !Array.isArray(manifest) && + !manifest.workspaces && + !manifest.overrides && + [ + "dependencies", + "devDependencies", + "optionalDependencies", + "peerDependencies", + "peerDependenciesMeta", + "bundledDependencies", + "bundleDependencies", + ].every((key) => + isDeepStrictEqual(manifest[key] ?? {}, packages?.[""]?.[key] ?? {}), + ); + const valid = + lock?.lockfileVersion === 3 && + manifestMatches && + packages && + typeof packages === "object" && + !Array.isArray(packages) && + Object.values(packages).every( + (entry) => + entry && + typeof entry === "object" && + !Array.isArray(entry) && + (entry.resolved === undefined || typeof entry.resolved === "string") && + [ + "dependencies", + "devDependencies", + "optionalDependencies", + "peerDependencies", + ].every( + (key) => + entry[key] === undefined || + (entry[key] && + typeof entry[key] === "object" && + !Array.isArray(entry[key]) && + Object.values(entry[key]).every( + (spec) => + typeof spec === "string" && + (spec === rule.resolved || + !/[:/#]/.test(spec) || + /^npm:(?:@[\w.-]+\/)?[\w.-]+@[\w.*^~|<>= +-]+$/.test(spec)), + )), + ), + ); + const external = valid + ? Object.entries(packages).filter( + ([, entry]) => + entry.resolved !== undefined && + !entry.resolved.startsWith("https://registry.npmjs.org/"), + ) + : []; + const match = + valid && + external.length === 1 && + external[0][0] === rule.packagePath && + ["version", "resolved", "integrity"].every( + (key) => external[0][1][key] === rule[key], + ); + return [{ ...rule, status: match ? "matched" : "stale" }]; +} diff --git a/tools/rollout/exclusions.test.mjs b/tools/rollout/exclusions.test.mjs new file mode 100644 index 0000000..3d69379 --- /dev/null +++ b/tools/rollout/exclusions.test.mjs @@ -0,0 +1,200 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { stringify } from "yaml"; +import { auditRepository } from "./audit.mjs"; +import { REGISTRY_EXCLUSIONS } from "./exclusions.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +import { resolveNoInstallWorkflowCalls } from "./integration.mjs"; + +const rule = REGISTRY_EXCLUSIONS[0]; +const sha = "a".repeat(40); +const checkout = { uses: "actions/checkout@v7" }; +const dependencies = { ordinary: "1.0.0", "tree-sitter-kotlin": rule.resolved }; +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { token: "${{ secrets.PUBLIC_SOCKET_FIREWALL_TOKEN }}" }, +}; +const install = { + run: "npm install", + env: { NPM_CONFIG_REPLACE_REGISTRY_HOST: "npmjs" }, +}; +async function scan({ + repository = "oagen", + entry = {}, + extraPackages = {}, + jobs, + defaults, + path = rule.workflows[0], + failRead = false, + manifest = { dependencies }, +} = {}) { + const lock = { + lockfileVersion: 3, + packages: { + "": { dependencies }, + "node_modules/ordinary": { + version: "1.0.0", + resolved: "https://registry.npmjs.org/ordinary/-/ordinary-1.0.0.tgz", + }, + [rule.packagePath]: { + version: rule.version, + resolved: rule.resolved, + integrity: rule.integrity, + ...entry, + }, + ...extraPackages, + }, + }; + const sources = new Map([ + [ + path, + stringify({ + on: "push", + defaults, + jobs: jobs ?? { check: { steps: [checkout, setup, install] } }, + }), + ], + [rule.lockfile, JSON.stringify(lock)], + ["package.json", JSON.stringify(manifest)], + ]); + return auditRepository( + { + async getRef() { + return { object: { sha } }; + }, + async getTree() { + return { + truncated: false, + tree: [...sources.keys()].map((path) => ({ + path, + type: "blob", + mode: "100644", + })), + }; + }, + async getText(repo, path, ref) { + assert.equal(ref, sha); + assert.equal(repo, `workos/${repository}`); + if (failRead && path === rule.lockfile) + throw new Error("lockfile unavailable"); + return sources.get(path); + }, + }, + { name: repository, defaultBranch: "main", visibility: "public" }, + ); +} + +test("approved archives are reported separately from fully integrated repositories", async () => { + for (const repository of ["oagen", "oagen-emitters"]) { + const result = await scan({ repository }); + assert.equal(result.disposition, "integrated-with-exclusions"); + assert.equal(result.assuranceDisposition, "needs-review"); + assert.equal(result.exclusions[0].status, "matched"); + assert.equal( + result.exclusions[0].approvalRequestId, + "24cb3145-70c0-471a-86b6-3f5d7b5860a0", + ); + const download = result.workflows[0].jobs[0].integration.downloads[0]; + assert.equal(download.status, "covered-with-exclusion"); + assert.equal(download.exclusionId, `${repository}-tree-sitter-kotlin`); + assert.equal( + resolveNoInstallWorkflowCalls([result], "workos")[0].disposition, + result.disposition, + ); + } +}); + +test("changed pins and additional external sources invalidate the exclusion", async () => { + for (const entry of [ + { version: "0.5.0" }, + { resolved: rule.resolved.replace("f66d290", "aaaaaaa") }, + { integrity: "sha512-other" }, + ]) { + const result = await scan({ entry }); + assert.equal(result.exclusions[0].status, "stale"); + assert.equal(result.disposition, "needs-review"); + } + const result = await scan({ + extraPackages: { + "node_modules/another": { resolved: "https://example.invalid/other.tgz" }, + }, + }); + assert.equal(result.exclusions[0].status, "stale"); + assert.equal(result.disposition, "needs-review"); + assert.equal((await scan({ failRead: true })).disposition, "audit-error"); + for (const manifest of [ + { dependencies: { extra: "https://example.invalid/extra.tgz" } }, + { overrides: { extra: "https://example.invalid/extra.tgz" } }, + { workspaces: ["packages/*"] }, + ]) + assert.equal((await scan({ manifest })).exclusions[0].status, "stale"); + assert.equal( + ( + await scan({ + entry: { dependencies: { extra: "github:other/unapproved" } }, + }) + ).exclusions[0].status, + "stale", + ); +}); + +test("source exclusions cannot hide other gaps, configuration changes, or repositories", async () => { + const missing = await scan({ + jobs: { check: { steps: [checkout, install] } }, + }); + assert.equal(missing.disposition, "needs-sfw"); + const sibling = await scan({ + jobs: { + check: { steps: [checkout, setup, install] }, + uncovered: { steps: [{ run: "npm ci" }] }, + }, + }); + assert.equal(sibling.disposition, "needs-sfw"); + for (const changed of [ + { ...install, env: { ...install.env, HOME: "/elsewhere" } }, + { ...install, "working-directory": "other" }, + { ...install, run: "npm install --registry=https://example.invalid" }, + { ...install, env: { NPM_CONFIG_REPLACE_REGISTRY_HOST: "never" } }, + ]) + assert.equal( + (await scan({ jobs: { check: { steps: [checkout, setup, changed] } } })) + .disposition, + "needs-review", + ); + assert.equal( + (await scan({ path: ".github/workflows/new.yml" })).disposition, + "needs-review", + ); + for (const changedCheckout of [ + { ...checkout, with: { ref: "other" } }, + { ...checkout, with: { repository: "other/repo" } }, + { ...checkout, with: { path: "other" } }, + { ...checkout, if: false }, + ]) + assert.equal( + ( + await scan({ + jobs: { check: { steps: [changedCheckout, setup, install] } }, + }) + ).disposition, + "needs-review", + ); + const defaults = { run: { "working-directory": "other" } }; + assert.equal((await scan({ defaults })).disposition, "needs-review"); + assert.equal( + ( + await scan({ + jobs: { check: { defaults, steps: [checkout, setup, install] } }, + }) + ).disposition, + "needs-review", + ); + assert.equal( + (await scan({ jobs: { check: { steps: [setup, install, checkout] } } })) + .disposition, + "needs-review", + ); + const unrelated = await scan({ repository: "unapproved" }); + assert.deepEqual(unrelated.exclusions, []); + assert.equal(unrelated.disposition, "needs-review"); +}); diff --git a/tools/rollout/integration.mjs b/tools/rollout/integration.mjs index 6ddae5e..8ef789c 100644 --- a/tools/rollout/integration.mjs +++ b/tools/rollout/integration.mjs @@ -2,7 +2,7 @@ import { APPROVED_RELEASE_SHA } from "./constants.mjs"; // Configuration/executable/startup controls, not application credentials. const RELEVANT_ENV = - /^(?:(?:npm|pnpm|bun|yarn|corepack)_|HOME$|GITHUB_ENV$|GITHUB_PATH$|SFW_BUN_CONFIG_PATH$|USERPROFILE$|XDG_|APPDATA$|LOCALAPPDATA$|PATH$|NODE_OPTIONS$|NODE_PATH$|BASH_ENV$|ENV$|SHELL$|SHELLOPTS$|BASHOPTS$|CDPATH$|LD_|DYLD_|BASH_FUNC_|HTTP_PROXY$|HTTPS_PROXY$|ALL_PROXY$|NO_PROXY$|NODE_EXTRA_CA_CERTS$|NODE_TLS_REJECT_UNAUTHORIZED$|SSL_CERT_|CURL_CA_BUNDLE$)/i; + /^(?:(?:npm|pnpm|bun|yarn|corepack)_|HOME$|RUNNER_TEMP$|GITHUB_ENV$|GITHUB_PATH$|SFW_BUN_CONFIG_PATH$|USERPROFILE$|XDG_|APPDATA$|LOCALAPPDATA$|PATH$|NODE_OPTIONS$|NODE_PATH$|BASH_ENV$|ENV$|SHELL$|SHELLOPTS$|BASHOPTS$|CDPATH$|LD_|DYLD_|BASH_FUNC_|HTTP_PROXY$|HTTPS_PROXY$|ALL_PROXY$|NO_PROXY$|NODE_EXTRA_CA_CERTS$|NODE_TLS_REJECT_UNAUTHORIZED$|SSL_CERT_|CURL_CA_BUNDLE$)/i; // Workflow toolchain-version metadata does not select a registry/config path. const VERSION_ENV = new Set([ "NODE_VERSION", @@ -10,6 +10,11 @@ const VERSION_ENV = new Set([ "BUN_VERSION", "XCODE_VERSION", ]); +export const SUPPORTED_SHELLS = new Set([ + "bash", + "sh", + "bash --noprofile --norc -euo pipefail {0}", +]); const normalize = (value) => String(value ?? "").replaceAll(/\s+/g, ""); const expression = (value) => normalize(value).replace(/^\$\{\{(.*)\}\}$/, "$1"); @@ -46,7 +51,7 @@ export function defaultsUncertain(defaults) { Object.entries(defaults.run).some(([key, value]) => key === "working-directory" ? !literalWorkingDirectory(value) - : key !== "shell" || !["bash", "sh"].includes(value), + : key !== "shell" || !SUPPORTED_SHELLS.has(value), ) ); } @@ -293,6 +298,15 @@ export function observedIntegration(operations, job, context) { step: operation.step, manager: operation.manager, ...result, + ...(operation.registryExclusion + ? { + exclusionId: operation.registryExclusion, + status: + result.status === "covered" + ? "covered-with-exclusion" + : result.status, + } + : {}), }); } if (operation.registryPersisting) { @@ -362,7 +376,9 @@ export function observedIntegration(operations, job, context) { ? "needs-review" : downloads.length || additionalJsPaths.some((path) => path.status === "setup-observed") - ? "integrated" + ? downloads.some((entry) => entry.exclusionId) + ? "integrated-with-exclusions" + : "integrated" : "no-js-ci"; return { disposition, @@ -463,12 +479,12 @@ export function resolveNoInstallWorkflowCalls(repositories, organization) { return { ...repository, workflows, - disposition: integrationDisposition(workflows), + disposition: integrationDisposition(workflows, repository.exclusions), }; }); } -export function integrationDisposition(workflows) { +export function integrationDisposition(workflows, exclusions = []) { const dispositions = new Set( workflows.flatMap((workflow) => [ ...(workflow.parseError !== undefined ? ["needs-review"] : []), @@ -477,7 +493,19 @@ export function integrationDisposition(workflows) { ), ]), ); - for (const disposition of ["needs-sfw", "needs-review", "integrated"]) { + if (exclusions.some((entry) => entry.status === "stale")) + dispositions.add("needs-review"); + if ( + dispositions.has("integrated") && + exclusions.some((entry) => entry.status === "matched") + ) + dispositions.add("integrated-with-exclusions"); + for (const disposition of [ + "needs-sfw", + "needs-review", + "integrated-with-exclusions", + "integrated", + ]) { if (dispositions.has(disposition)) return disposition; } return workflows.length ? "no-js-ci" : "no-ci"; diff --git a/tools/rollout/integration.test.mjs b/tools/rollout/integration.test.mjs index dffb172..f2756cf 100644 --- a/tools/rollout/integration.test.mjs +++ b/tools/rollout/integration.test.mjs @@ -245,7 +245,7 @@ test("teardown guards establish only the appropriate success-path boundary", () } }); -test("audit v2 separates primary integration counts from assurance and errors", async () => { +test("audit v3 separates primary integration counts from assurance and errors", async () => { const workflow = `on: push\njobs:\n build:\n steps:\n - uses: ${setup.uses}\n with: { token: '${setup.with.token}' }\n - run: npm ci\n - run: npm test\n`; const client = { listRestRepositories: async () => @@ -271,7 +271,7 @@ test("audit v2 separates primary integration counts from assurance and errors", getText: async () => workflow, }; const report = await runAudit(client); - assert.equal(report.schemaVersion, 2); + assert.equal(report.schemaVersion, 3); assert.deepEqual(report.dispositions, { "audit-error": 1, integrated: 1 }); assert.deepEqual(report.assuranceDispositions, { "audit-error": 1, From d3eca12d25f15d952454c4dc3a8840f13f1956d1 Mon Sep 17 00:00:00 2001 From: "workos-tars[bot]" <269013284+workos-tars[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 15:28:56 +0000 Subject: [PATCH 11/19] feat(audit): record OpenAPI Kotlin archive exclusion --- CONTRIBUTING.md | 2 +- README.md | 6 +- tools/rollout/exclusions.mjs | 75 ++++++++++++++++--------- tools/rollout/exclusions.test.mjs | 91 ++++++++++++++++++++++++++++++- 4 files changed, 144 insertions(+), 30 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index daf54da..74ea349 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,7 +42,7 @@ The minimum test-coverage policy is one shell test suite for every executable sh `npm run inventory` and `npm run audit:live` are read-only manual commands using an existing authorized `gh` session. Confirm organization-wide read access separately before making organization-wide claims. No scheduling or remediation is implicit. See README for the scan scope and limitations. -Audit schema2 separates primary observed configuration from strict `assuranceDispositions`. Acceptance tests must show that arbitrary execution before or after setup, pipelines, package scripts and script-only siblings do not invent missing installs or erase configuration. An actual uncovered install, explicit configuration conflict or unresolved relevant source must remain visible. Cover conditional Corepack state, scoped environment/registry changes, quoted data versus executable commands, executor payload boundaries, composite input binding and snapshot-bound reusable-workflow references. Preserve strict security/opacity diagnostics and source acquisition limits. `integrated` is configuration evidence, not runtime proof. `no-js-ci` is absence of an observed install path, not assurance about script bodies. Do not introduce a historical cohort, coverage percentage or repository allowlist. +Audit schema3 separates primary observed configuration from strict `assuranceDispositions`. Acceptance tests must show that arbitrary execution before or after setup, pipelines, package scripts and script-only siblings do not invent missing installs or erase configuration. An actual uncovered install, explicit configuration conflict or unresolved relevant source must remain visible. Cover conditional Corepack state, scoped environment/registry changes, quoted data versus executable commands, executor payload boundaries, composite input binding and snapshot-bound reusable-workflow references. Preserve strict security/opacity diagnostics and source acquisition limits. `integrated` is configuration evidence, not runtime proof. `no-js-ci` is absence of an observed install path, not assurance about script bodies. Do not introduce a historical cohort, coverage percentage or blanket repository allowlist. Approved source exclusions must remain explicitly recorded, pin-checked and separate from protection claims. Full results are owner-only, ignored `reports/inventory.json` and `reports/live-audit.json`; terminal JSON contains sanitized counts. Audit `scanErrors`/`scanStatus` distinguish operational failures from discovered gaps. Exit 1 includes partial scans; exit 0 can still include `needs-sfw` or `needs-review`. Do not force-add reports, expose inventory/source in logs, or upload full reports as artifacts. diff --git a/README.md b/README.md index a2ad902..90a4792 100644 --- a/README.md +++ b/README.md @@ -231,9 +231,11 @@ Strict job `status` and `assuranceDisposition` retain opaque execution and secur ### Approved source exclusions -`tools/rollout/exclusions.mjs` records the approved `tree-sitter-kotlin` GitHub archive in `oagen` and `oagen-emitters`, including its exact path, version, URL, integrity pin and approval provenance. This archive intentionally downloads outside SFW; npm-registry dependencies still use the firewall. The existing `NPM_CONFIG_REPLACE_REGISTRY_HOST=npmjs` setting is retained because rewriting GitHub archives to the npm proxy breaks installation. +`tools/rollout/exclusions.mjs` records the approved `tree-sitter-kotlin` GitHub archive in `oagen`, `oagen-emitters` and `openapi-spec`, including its exact path, version, URL, integrity pin and approval provenance. This archive intentionally downloads outside SFW; npm-registry dependencies still use the firewall. The existing `NPM_CONFIG_REPLACE_REGISTRY_HOST=npmjs` setting is retained because rewriting GitHub archives to the npm proxy breaks installation. -Each scan reads the manifest and version-3 lockfile at the same captured repository SHA. Manifest dependency drift, workspace/override configuration, a changed pin or any additional non-npm source makes the exclusion `stale` and keeps the repository in review; an unreadable lockfile is an audit error. Only direct, repository-root `npm install`/`npm ci` steps with that exact environment setting in the recorded workflows receive the exclusion, after one unconditional default-source checkout. Alternate checkout refs/repositories/paths, nested working directories and composite-action installs are not excused. Missing setup, additional gaps, unknown execution contexts and other configuration overrides remain findings. Qualified results use `integrated-with-exclusions`, never an unqualified protection claim. Strict assurance remains unchanged. +Each scan reads the manifest and version-3 lockfile at the same captured repository SHA. Manifest dependency drift, workspace or unrecorded override configuration, a changed pin or any additional non-npm source makes the exclusion `stale` and keeps the repository in review; an unreadable lockfile is an audit error. For step-level environment exceptions, only direct, repository-root `npm install`/`npm ci` steps with that exact setting in the recorded workflows receive the exclusion, after one unconditional default-source checkout. Alternate checkout refs/repositories/paths, nested working directories and composite-action installs are not excused. Missing setup, additional gaps, unknown execution contexts and other configuration overrides remain findings. Qualified results use `integrated-with-exclusions`, never an unqualified protection claim. Strict assurance remains unchanged. + +OpenAPI's exception is recorded from its project `.npmrc`, not used to waive any step-level environment override. Its effective config lines and existing `js-yaml`/`lodash` override snapshot must match the record; changes return it to review. Unresolved checkout/action provenance and conditional execution remain findings even when the archive pin matches. ### Static limits diff --git a/tools/rollout/exclusions.mjs b/tools/rollout/exclusions.mjs index 1755121..62d3ccf 100644 --- a/tools/rollout/exclusions.mjs +++ b/tools/rollout/exclusions.mjs @@ -2,30 +2,44 @@ import { isDeepStrictEqual } from "node:util"; // Approved source exclusions, not repository exemptions. A changed archive, // integrity pin, or additional non-npm source must be reviewed again. -export const REGISTRY_EXCLUSIONS = ["oagen", "oagen-emitters"].map( - (repository) => ({ - id: `${repository}-tree-sitter-kotlin`, - repository, - lockfile: "package-lock.json", - packagePath: "node_modules/tree-sitter-kotlin", - version: "0.4.0", - resolved: - "https://github.com/fwcd/tree-sitter-kotlin/archive/f66d2908542e93c0204c6c241f794afe4e9cd5d1.tar.gz", - integrity: - "sha512-7pk1Tg/gXh+6hM4E0F2rPKeLyA/bNlYyqVu6T1Md/AV/vloakbvEZG0R4CYwUfVtgFAMZRbOtA8JBzX1SFatBA==", - workflows: [ - ".github/workflows/ci.yml", - ".github/workflows/lint.yml", - ".github/workflows/release.yml", - ], - reason: - "The pinned tree-sitter-kotlin GitHub archive intentionally downloads outside Socket Firewall; npm-registry dependencies still use SFW.", - approvedBy: "matt.peake@workos.com", - approvalRequestId: "24cb3145-70c0-471a-86b6-3f5d7b5860a0", - approvalUrl: - "https://tars.workos.tools/conversations/pi_ae1489a3e5ee424ebd2cac9260cbad13", - }), -); +export const REGISTRY_EXCLUSIONS = [ + "oagen", + "oagen-emitters", + "openapi-spec", +].map((repository) => ({ + id: `${repository}-tree-sitter-kotlin`, + repository, + lockfile: "package-lock.json", + packagePath: "node_modules/tree-sitter-kotlin", + version: "0.4.0", + resolved: + "https://github.com/fwcd/tree-sitter-kotlin/archive/f66d2908542e93c0204c6c241f794afe4e9cd5d1.tar.gz", + integrity: + "sha512-7pk1Tg/gXh+6hM4E0F2rPKeLyA/bNlYyqVu6T1Md/AV/vloakbvEZG0R4CYwUfVtgFAMZRbOtA8JBzX1SFatBA==", + workflows: [ + ".github/workflows/ci.yml", + ".github/workflows/lint.yml", + ".github/workflows/release.yml", + ], + reason: + "The pinned tree-sitter-kotlin GitHub archive intentionally downloads outside Socket Firewall; npm-registry dependencies still use SFW.", + approvedBy: "matt.peake@workos.com", + approvalRequestId: "24cb3145-70c0-471a-86b6-3f5d7b5860a0", + approvalUrl: + "https://tars.workos.tools/conversations/pi_ae1489a3e5ee424ebd2cac9260cbad13", + ...(repository === "openapi-spec" + ? { + approvalRequestId: "bf1c082e-4112-484a-8683-f36854138690", + // Record the project-config exception; do not waive any step overrides. + workflows: [], + projectNpmrc: [ + "omit-lockfile-registry-resolved=true", + "replace-registry-host=npmjs", + ], + manifestOverrides: { "js-yaml": "^5.4.1", lodash: "^4.17.23" }, + } + : {}), +})); export async function readRegistryExclusions(repository, readSource) { const rule = REGISTRY_EXCLUSIONS.find( @@ -36,13 +50,23 @@ export async function readRegistryExclusions(repository, readSource) { const lock = JSON.parse(await readSource(rule.lockfile)); const manifest = JSON.parse(await readSource("package.json")); const packages = lock?.packages; + const configMatches = + !rule.projectNpmrc || + isDeepStrictEqual( + (await readSource(".npmrc")) + .split(/\r?\n/) + .map((line) => line.trim()) + .filter((line) => line && !/^[#;]/.test(line)) + .toSorted(), + rule.projectNpmrc.toSorted(), + ); // npm install can change the lock: do not excuse an unrecorded manifest edit. const manifestMatches = manifest && typeof manifest === "object" && !Array.isArray(manifest) && !manifest.workspaces && - !manifest.overrides && + isDeepStrictEqual(manifest.overrides, rule.manifestOverrides) && [ "dependencies", "devDependencies", @@ -57,6 +81,7 @@ export async function readRegistryExclusions(repository, readSource) { const valid = lock?.lockfileVersion === 3 && manifestMatches && + configMatches && packages && typeof packages === "object" && !Array.isArray(packages) && diff --git a/tools/rollout/exclusions.test.mjs b/tools/rollout/exclusions.test.mjs index 3d69379..919c775 100644 --- a/tools/rollout/exclusions.test.mjs +++ b/tools/rollout/exclusions.test.mjs @@ -27,6 +27,7 @@ async function scan({ path = rule.workflows[0], failRead = false, manifest = { dependencies }, + npmrc, } = {}) { const lock = { lockfileVersion: 3, @@ -56,6 +57,7 @@ async function scan({ ], [rule.lockfile, JSON.stringify(lock)], ["package.json", JSON.stringify(manifest)], + ...(npmrc === undefined ? [] : [[".npmrc", npmrc]]), ]); return auditRepository( { @@ -75,8 +77,8 @@ async function scan({ async getText(repo, path, ref) { assert.equal(ref, sha); assert.equal(repo, `workos/${repository}`); - if (failRead && path === rule.lockfile) - throw new Error("lockfile unavailable"); + if (!sources.has(path) || (failRead && path === rule.lockfile)) + throw new Error("source unavailable"); return sources.get(path); }, }, @@ -104,6 +106,91 @@ test("approved archives are reported separately from fully integrated repositori } }); +test("OpenAPI records the approved archive without waiving other findings", async () => { + const overrides = { "js-yaml": "^5.4.1", lodash: "^4.17.23" }; + const npmrc = + "# Preserve the archive\nreplace-registry-host=npmjs\nomit-lockfile-registry-resolved=true\n"; + const steps = [checkout, setup, { run: "npm ci" }, { run: "npm test" }]; + const options = { + repository: "openapi-spec", + manifest: { dependencies, overrides }, + npmrc, + jobs: { check: { steps } }, + }; + const result = await scan(options); + assert.equal(result.exclusions[0].status, "matched"); + assert.equal( + result.exclusions[0].approvalRequestId, + "bf1c082e-4112-484a-8683-f36854138690", + ); + assert.deepEqual(result.exclusions[0].workflows, []); + assert.equal(result.disposition, "integrated-with-exclusions"); + assert.equal(result.assuranceDisposition, "needs-review"); + assert.equal( + ( + await scan({ + ...options, + jobs: { check: { steps: [...steps, { uses: "./missing" }] } }, + }) + ).disposition, + "needs-review", + ); + assert.equal( + ( + await scan({ + ...options, + jobs: { check: { steps }, uncovered: { steps: [{ run: "npm ci" }] } }, + }) + ).disposition, + "needs-sfw", + ); + assert.equal( + ( + await scan({ + ...options, + jobs: { check: { steps: [checkout, setup, install] } }, + }) + ).disposition, + "needs-review", + ); + for (const changes of [ + { npmrc: npmrc.replace("=npmjs", "=never") }, + { npmrc: `${npmrc}@other:registry=https://example.invalid/\n` }, + { + manifest: { dependencies, overrides: { ...overrides, lodash: "^5.0.0" } }, + }, + { + manifest: { + dependencies, + overrides: { ...overrides, extra: "https://example.invalid/other.tgz" }, + }, + }, + { entry: { integrity: "sha512-changed" } }, + { entry: { resolved: "https://example.invalid/other.tgz" } }, + { + extraPackages: { + "node_modules/another": { + resolved: "https://example.invalid/other.tgz", + }, + }, + }, + ]) { + const changed = await scan({ ...options, ...changes }); + assert.equal(changed.exclusions[0].status, "stale"); + assert.equal(changed.disposition, "needs-review"); + } + assert.equal( + (await scan({ ...options, npmrc: undefined })).disposition, + "audit-error", + ); + // The OpenAPI override snapshot never authorizes overrides in the other repos. + assert.equal( + (await scan({ manifest: { dependencies, overrides } })).exclusions[0] + .status, + "stale", + ); +}); + test("changed pins and additional external sources invalidate the exclusion", async () => { for (const entry of [ { version: "0.5.0" }, From cc0e8c5275aa7836673a248ea697ae6c16f30ee3 Mon Sep 17 00:00:00 2001 From: "workos-tars[bot]" <269013284+workos-tars[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 16:14:02 +0000 Subject: [PATCH 12/19] fix(audit): recognize stricter stable install conditions --- README.md | 2 +- tools/rollout/audit-cleanup.test.mjs | 47 ++++++++++++++++++++++++++++ tools/rollout/integration.mjs | 34 ++++++++++++++------ 3 files changed, 72 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 90a4792..156de21 100644 --- a/README.md +++ b/README.md @@ -243,7 +243,7 @@ The primary question is **whether SFW is configured for identified dependency-in The small grammar recognizes npm/pnpm/Bun installs, download-capable executors, Yarn blockers and setup/teardown ordering. A lexical boundary scanner keeps quoted data and command substitutions inside their containing command rather than inventing installs from text. Pipelines and executor payload arguments do not invalidate configuration simply because their execution is opaque. For `npx`/`bunx`, a literal target after optional `-y`/`--yes` separates installer options from program arguments, including quoted or variable-valued payloads. Pre-target registry overrides remain gaps; unsupported installer options, dynamic installer targets and explicit context changes can still require review. Uninterpreted shell structure can establish configuration presence without proving whether a particular command executes; it cannot establish a definite missing install when the command may only be data. -Configuration checks retain the approved action SHA, visibility-appropriate token, Bun configuration and public-fork exception rules. Disabled or continued-on-error setup cannot silently cover an install. Conditional setup covers only an install with the same recognized boolean-string output comparison: a dependency job output, or an output from a uniquely identified earlier step. Future/duplicate step IDs, mutable environment guards, status functions and unmatched conditions remain unresolved; strict execution assurance is unchanged. Corepack uncertainty survives conditional or targeted controls. The action-exported `SFW_BUN_CONFIG_PATH` is recognized in Bun's `--config` argument when Bun setup is enabled; explicit overrides of that variable remain configuration questions. Whole-value composite input references in `with`/`env` are bound to caller values/defaults without evaluating conditions or interpolating shell text. Missing/malformed local source, cycles and expansion limits remain review findings. +Configuration checks retain the approved action SHA, visibility-appropriate token, Bun configuration and public-fork exception rules. Disabled or continued-on-error setup cannot silently cover an install. Conditional setup covers an install only when the install requires every recognized setup predicate. The bounded grammar supports boolean-string output comparisons (dependency jobs or uniquely identified earlier steps) and literal `github.event_name` equalities, with at most two predicates joined by `&&`. A stricter install condition can imply setup, never the reverse. Future/duplicate step IDs, mutable environment guards, status functions, partial expression interpolation, unsupported expressions and unmatched conditions remain unresolved; strict execution assurance is unchanged. Corepack uncertainty survives conditional or targeted controls. The action-exported `SFW_BUN_CONFIG_PATH` is recognized in Bun's `--config` argument when Bun setup is enabled; explicit overrides of that variable remain configuration questions. Whole-value composite input references in `with`/`env` are bound to caller values/defaults without evaluating conditions or interpolating shell text. Missing/malformed local source, cycles and expansion limits remain review findings. Known registry writes and teardown invalidate the relevant configuration; a command-local override does not contaminate later installs. npm/pnpm registry setters, configuration-file writes (including `tee`), standalone relevant assignments and GitHub environment-file writes remain findings rather than being treated as generic opacity. Reading a configuration variable is not a mutation. HOME/PATH/NODE_OPTIONS overrides, relevant exported variables within a run block, unsupported environment wrappers, dynamic mappings and job containers remain configuration questions. Ordinary application secrets/settings, version metadata and sibling service containers do not invalidate host setup ordering. Job-level guards affect shared reachability; explicit status predicates and conditional configuration transitions remain conservative. Exact npm `--replace-registry-host=always` preserves the configured registry; other modes require review unless covered by an approved source exclusion. diff --git a/tools/rollout/audit-cleanup.test.mjs b/tools/rollout/audit-cleanup.test.mjs index dd176f4..eb87944 100644 --- a/tools/rollout/audit-cleanup.test.mjs +++ b/tools/rollout/audit-cleanup.test.mjs @@ -227,6 +227,53 @@ test("matching immutable output guards cover only the same guarded installs", () } }); +test("a stricter stable conjunction implies setup, never the reverse", () => { + const ready = { id: "ready", run: 'echo enabled=true >> "$GITHUB_OUTPUT"' }; + const output = "steps.ready.outputs.enabled == 'true'"; + const event = "github.event_name == 'pull_request'"; + const check = (setupGuard, installGuard, steps = [ready]) => + inspect([ + ...steps, + { ...setup, if: setupGuard }, + { run: "npx tool", if: installGuard }, + ]); + for (const guard of [`${output} && ${event}`, `${event} && ${output}`]) { + const result = check(output, `\${{ ${guard} }}`); + assert.equal(result.integration.disposition, "integrated"); + assert.equal(result.status, "unknown"); + assert.equal(check(guard, output).integration.disposition, "needs-review"); + assert.equal(check(guard, guard).integration.disposition, "integrated"); + assert.equal( + check(output, guard, [ready, ready]).integration.disposition, + "needs-review", + ); + assert.equal( + check(output, guard, []).integration.disposition, + "needs-review", + ); + } + for (const guard of [ + `${output} || ${event}`, + `\${{ ${output} }} && ${event}`, + `${event} && \${{ ${output} }}`, + `${output} && always()`, + `${output} && env.ENABLED == 'true'`, + `${output} && (${event})`, + `${output} && ${event} && needs.other.outputs.ok == 'true'`, + `${output.replace("'true'", "'false'")} && ${event}`, + ]) + assert.equal( + check(output, guard).integration.disposition, + "needs-review", + guard, + ); + assert.equal( + check(`${output} && ${event}`, `${output} && github.event_name == 'push'`) + .integration.disposition, + "needs-review", + ); +}); + test("literal npm workspaces preserve command classification and teardown gaps", () => { const teardown = { uses: `${setup.uses.split("@")[0]}/teardown@${APPROVED_RELEASE_SHA}`, diff --git a/tools/rollout/integration.mjs b/tools/rollout/integration.mjs index 8ef789c..1f12100 100644 --- a/tools/rollout/integration.mjs +++ b/tools/rollout/integration.mjs @@ -111,14 +111,24 @@ function contextUncertain(job, context) { ); } -// Only boolean-string output comparisons are modeled. Step outputs must belong -// to one completed earlier step, never a future/duplicate ID or mutable env. -function stableOutputGuard(value, job, step) { +// Stable output/event comparisons only. Step outputs must belong to one +// completed earlier step, never a future/duplicate ID or mutable env. +function stableGuard(value, job, step) { if (typeof value !== "string") return undefined; const text = value .trim() .replace(/^\$\{\{([\s\S]*)\}\}$/, "$1") .trim(); + if (/\$\{\{|\}\}/.test(text)) return undefined; + const terms = text.split(/\s*&&\s*/); + // ponytail: two conjuncts cover observed workflows; no general expression evaluator. + if (terms.length > 2) return undefined; + if (terms.length === 2) { + const guards = terms.map((term) => stableGuard(term, job, step)); + return guards.every(Boolean) ? guards.sort().join("&&") : undefined; + } + const event = text.match(/^github\.event_name\s*==\s*(['"])([a-z_]+)\1$/); + if (event) return `github.event_name==${event[2]}`; const match = text.match( /^(steps|needs)\.([\w-]+)\.outputs\.([\w-]+)\s*==\s*(['"])(true|false)\4$/, ); @@ -166,14 +176,18 @@ export function observedIntegration(operations, job, context) { operation.uncertain; const observedState = operation.manager === "bun" && bunState ? bunState : state; + const operationGuard = + observedState?.guard && + stableGuard( + job.steps?.[operation.step - 1]?.if, + job, + operation.step, + )?.split("&&"); const currentState = observedState?.guard && - observedState.guard !== - stableOutputGuard( - job.steps?.[operation.step - 1]?.if, - job, - operation.step, - ) + !observedState.guard + .split("&&") + .every((term) => operationGuard?.includes(term)) ? { status: "unresolved", reason: "unmatched-setup-condition" } : observedState; const validInterval = ["covered", "fork-exception"].includes( @@ -205,7 +219,7 @@ export function observedIntegration(operations, job, context) { /^\$\{\{.+\}\}$/.test(operation.fallback)); const guard = !operation.via && !operation.boundaryUncertainWithoutCondition - ? stableOutputGuard(operation.condition, job, operation.step) + ? stableGuard(operation.condition, job, operation.step) : undefined; if ( (uncertain && !guard) || From f9eae987db0b873ab9921423c37c12244ba37df4 Mon Sep 17 00:00:00 2001 From: "workos-tars[bot]" <269013284+workos-tars[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 17:10:11 +0000 Subject: [PATCH 13/19] fix(audit): resolve snapshot-pinned pnpm bootstrap --- README.md | 2 + tools/rollout/audit.mjs | 15 ++- tools/rollout/classify.mjs | 61 +++++++++-- tools/rollout/package-manager.test.mjs | 140 +++++++++++++++++++++++++ 4 files changed, 207 insertions(+), 11 deletions(-) create mode 100644 tools/rollout/package-manager.test.mjs diff --git a/README.md b/README.md index 156de21..ba32c6a 100644 --- a/README.md +++ b/README.md @@ -251,6 +251,8 @@ Literal repository-relative default working directories and npm `--prefix` argum Literal leading npm workspace selectors (`-w`, `--workspace`, and `--workspace=`) retain the underlying command classification: scripts stay unverified script execution, while installs and explicit `npm exec -- ` remain download candidates. Literal npm package selectors in `npx --package`/`-p` are recognized without treating executable payload flags as installer configuration. A workspace selector may contain a matrix component only when every declared alternative is a literal, safe single path component and the matrix has no `include` overrides; this does not verify any script body. Runner-temp tarball arguments such as `"$RUNNER_TEMP"/package-*.tgz` are local inputs to npm/npx, not alternate registry URLs; overriding `RUNNER_TEMP` invalidates that recognition's configuration evidence. Other dynamic selectors, unsupported options, and post-teardown executor downloads remain findings. +The exact three-line `set -euo pipefail` → `pnpm_package="$(node --print 'require("./package.json").packageManager')"` → `npm install --global "$pnpm_package" --ignore-scripts --no-audit --no-fund` bootstrap can resolve a plain `pnpm@major.minor.patch` from the captured root manifest. It requires one default-source checkout, root working directories, no project `.npmrc`, and no preceding shell/local/mutable action steps. Unsupported pins, source changes, extra commands and configuration overrides remain unresolved. This is snapshot-based configuration evidence, not runtime verification of the JSON reader or preceding pinned actions; strict assurance stays unverified. + The exact standard shell template `bash --noprofile --norc -euo pipefail {0}` is recognized alongside `bash`/`sh`. A clean-room `env -i` npm invocation is recognized only when it retains the exact inherited HOME, PATH, action registry and npm-config path; missing, changed, duplicated or additional configuration variables remain unresolved. Local `npm version`, argument-free/format-only `npm pack`, and literal Bun script filenames remain unverified execution, not invented download operations. Remote or dynamic `npm pack` targets remain review candidates. A setup-only `NPM_CONFIG_USERCONFIG` pointing to a named file under `${{ runner.temp }}` is recognized because the pinned action validates that path and configures both it and `HOME/.npmrc`. Install-step overrides remain unresolved. Public fallback expressions remain recorded as possible `fork-exception` paths: the pinned action validates the value and independently permits fallback only for public external-fork pull requests. Neither recognition is runtime proof or a repository allowlist. diff --git a/tools/rollout/audit.mjs b/tools/rollout/audit.mjs index 756f8e6..f19d2ca 100644 --- a/tools/rollout/audit.mjs +++ b/tools/rollout/audit.mjs @@ -1,7 +1,11 @@ import { mkdtemp, rename, rm, writeFile } from "node:fs/promises"; import { dirname, join } from "node:path"; -import { classifyWorkflow, repositoryDisposition } from "./classify.mjs"; +import { + classifyWorkflow, + PACKAGE_MANAGER_READ, + repositoryDisposition, +} from "./classify.mjs"; import { captureRepositoryInventory } from "./inventory.mjs"; import { ORGANIZATION } from "./constants.mjs"; import { readRegistryExclusions } from "./exclusions.mjs"; @@ -213,6 +217,14 @@ export async function auditRepository(client, repository) { ); } + // Read the value at the captured SHA only when this exact idiom occurs. + // A project npmrc or an absent manifest leaves the dynamic install unresolved. + const packageManager = + blobPaths.has("package.json") && + !blobPaths.has(".npmrc") && + workflowTexts.some((text) => text.includes(PACKAGE_MANAGER_READ)) + ? JSON.parse(await readSource("package.json"))?.packageManager + : undefined; const exclusions = await readRegistryExclusions( repository.name, readSource, @@ -221,6 +233,7 @@ export async function auditRepository(client, repository) { workflowPaths.map((path, index) => classifyWorkflow(workflowTexts[index], { localActions, + packageManager, path, registryExclusion: exclusions.find( (entry) => diff --git a/tools/rollout/classify.mjs b/tools/rollout/classify.mjs index 849c69f..54f44fb 100644 --- a/tools/rollout/classify.mjs +++ b/tools/rollout/classify.mjs @@ -12,6 +12,9 @@ import { SUPPORTED_SHELLS, } from "./integration.mjs"; +export const PACKAGE_MANAGER_READ = 'require("./package.json").packageManager'; +const PNPM_BOOTSTRAP = `set -euo pipefail\npnpm_package="$(node --print '${PACKAGE_MANAGER_READ}')"\nnpm install --global "$pnpm_package" --ignore-scripts --no-audit --no-fund`; + const FULL_SHA_PATTERN = /^[0-9a-f]{40}$/; const LOCAL_TARBALL = /^\$(?:RUNNER_TEMP|\{RUNNER_TEMP\})\/[\w.*-]+\.tgz$/; const UNSAFE_PUBLIC_TRIGGERS = new Set([ @@ -784,8 +787,29 @@ export function classifyStep(step, context = {}) { ) ) return [{ kind: "unknown", sourceError: "malformed-action-input" }]; + // Resolve only this root-manifest bootstrap, not arbitrary shell variables. + // Snapshot evidence does not certify execution of the JSON reader at runtime. + const pinnedBootstrap = + !context.actionStack?.length && + typeof context.packageManager === "string" && + context.packageManager === context.packageManager.trim() && + /^pnpm@(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$/.test( + context.packageManager ?? "", + ) && + [undefined, ".", "./"].includes(step["working-directory"]) && + typeof step.run === "string" && + step.run + .trim() + .split("\n") + .map((line) => line.trim()) + .join("\n") === PNPM_BOOTSTRAP; + if (pinnedBootstrap) + step = { + ...step, + run: `npm install --global ${context.packageManager} --ignore-scripts --no-audit --no-fund`, + }; let operations; - let uncertain = boundaryUncertain(step); + let uncertain = boundaryUncertain(step) || pinnedBootstrap; let integrationEnv = step.env; if ( typeof step.uses === "string" && @@ -1404,20 +1428,37 @@ export function classifyJob(jobName, job, context = {}, triggers = []) { literalMatrixKeys, stepBudget: { remaining: 1000 }, }; + let packageManagerSourceUnchanged = + defaultCheckout && + [undefined, ".", "./"].includes(job.defaults?.run?.["working-directory"]) && + context.exclusionRootDirectory !== false; const operations = condition(job.if) === "never" ? [] - : steps.flatMap((step, index) => - classifyStep( - step, - index > (checkout?.index ?? -1) - ? stepContext - : { ...stepContext, registryExclusion: undefined }, - ).map((operation) => ({ + : steps.flatMap((step, index) => { + const afterCheckout = index > (checkout?.index ?? -1); + const currentContext = { + ...stepContext, + registryExclusion: afterCheckout + ? stepContext.registryExclusion + : undefined, + packageManager: + afterCheckout && packageManagerSourceUnchanged + ? context.packageManager + : undefined, + }; + // Earlier shell/local or mutable action execution can replace the file. + if ( + typeof step?.uses !== "string" || + step.uses.startsWith("./") || + !FULL_SHA_PATTERN.test(step.uses.split("@")[1] ?? "") + ) + packageManagerSourceUnchanged = false; + return classifyStep(step, currentContext).map((operation) => ({ ...operation, step: index + 1, - })), - ); + })); + }); if ( !Array.isArray(job.steps) || steps.length === 0 || diff --git a/tools/rollout/package-manager.test.mjs b/tools/rollout/package-manager.test.mjs new file mode 100644 index 0000000..896cf22 --- /dev/null +++ b/tools/rollout/package-manager.test.mjs @@ -0,0 +1,140 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { stringify } from "yaml"; +import { auditRepository } from "./audit.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; + +const sha = "a".repeat(40); +const checkout = { uses: `actions/checkout@${"b".repeat(40)}` }; +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}" }, +}; +const run = `set -euo pipefail +pnpm_package="$(node --print 'require("./package.json").packageManager')" +npm install --global "$pnpm_package" --ignore-scripts --no-audit --no-fund`; + +async function inspect({ + packageManager = "pnpm@11.20.0", + steps = [checkout, setup, { run }], + job = {}, + files = {}, + manifestMode = "100644", +} = {}) { + const sources = { + ".github/workflows/ci.yml": stringify({ + on: "push", + jobs: { check: { ...job, steps } }, + }), + "package.json": JSON.stringify({ packageManager }), + ...files, + }; + const reads = []; + const result = await auditRepository( + { + getRef: async () => ({ object: { sha } }), + getTree: async () => ({ + truncated: false, + tree: Object.keys(sources).map((path) => ({ + path, + type: "blob", + mode: path === "package.json" ? manifestMode : "100644", + })), + }), + getText: async (repository, path, ref) => { + assert.equal(ref, sha); + reads.push(path); + if (!Object.hasOwn(sources, path)) throw new Error("missing source"); + return sources[path]; + }, + }, + { name: "fixture", defaultBranch: "main", visibility: "private" }, + ); + return { ...result, reads }; +} + +test("root packageManager bootstrap resolves the captured pin, not an arbitrary variable", async () => { + const result = await inspect(); + assert.equal(result.disposition, "integrated"); + assert.equal(result.assuranceDisposition, "needs-review"); + assert.ok(result.reads.includes("package.json")); + assert.equal( + (await inspect({ steps: [checkout, { run }] })).disposition, + "needs-sfw", + ); + assert.equal( + ( + await inspect({ steps: [checkout, setup, { run: "npm ci" }] }) + ).reads.includes("package.json"), + false, + ); +}); + +test("unresolved or changed packageManager source never clears the finding", async () => { + for (const packageManager of [ + null, + ["pnpm@11.20.0"], + { packageManager: "pnpm@11.20.0" }, + "pnpm@latest", + "pnpm@11.20.0\n", + "pnpm@^11.20.0", + "pnpm@https://example.com/pnpm.tgz", + "--registry=https://example.com", + "pnpm@11.20.0\nnpm install evil", + "npm@11.0.0", + ]) + assert.equal( + (await inspect({ packageManager })).disposition, + "needs-review", + String(packageManager), + ); + for (const changed of [ + { steps: [setup, { run }] }, + { steps: [checkout, setup, { uses: 42 }, { run }] }, + { steps: [{ ...checkout, with: { ref: "other" } }, setup, { run }] }, + { steps: [{ ...checkout, with: { path: "nested" } }, setup, { run }] }, + { steps: [checkout, setup, { run: "node rewrite-package.mjs" }, { run }] }, + { + steps: [checkout, setup, { uses: "./rewrite" }, { run }], + files: { + "rewrite/action.yml": stringify({ + runs: { + using: "composite", + steps: [{ shell: "bash", run: "node rewrite-package.mjs" }], + }, + }), + }, + }, + { steps: [checkout, setup, { run, "working-directory": "nested" }] }, + { job: { defaults: { run: { "working-directory": "nested" } } } }, + { job: { env: { NODE_OPTIONS: "--require ./rewrite.cjs" } } }, + { files: { ".npmrc": "registry=https://other.invalid/" } }, + { + steps: [ + checkout, + setup, + { run: `${run}\nnpm config set registry https://other.invalid/` }, + ], + }, + { + steps: [ + checkout, + setup, + { run: run.replace("--no-fund", "--registry=https://other.invalid/") }, + ], + }, + ]) + assert.notEqual( + (await inspect(changed)).disposition, + "integrated", + JSON.stringify(changed), + ); + assert.equal( + (await inspect({ manifestMode: "120000" })).disposition, + "audit-error", + ); + assert.equal( + (await inspect({ files: { "package.json": "{broken" } })).disposition, + "audit-error", + ); +}); From c56c877ff3c5ef3f75dcc5138ce9f6d0944dad5f Mon Sep 17 00:00:00 2001 From: "workos-tars[bot]" <269013284+workos-tars[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:07:22 +0000 Subject: [PATCH 14/19] fix(audit): finish bounded source and offline classification --- README.md | 10 +- tools/rollout/audit.mjs | 60 ++++- tools/rollout/classify.mjs | 133 ++++++++++- tools/rollout/commands.mjs | 49 +++- tools/rollout/configuration-presence.test.mjs | 2 +- tools/rollout/constants.mjs | 8 + tools/rollout/final-detector.test.mjs | 115 +++++++++ tools/rollout/fixtures/approved-release.json | 100 ++++++++ tools/rollout/integration.mjs | 11 +- tools/rollout/release-snapshot.test.mjs | 90 +++++++ tools/rollout/source-resolution.test.mjs | 220 ++++++++++++++++++ 11 files changed, 777 insertions(+), 21 deletions(-) create mode 100644 tools/rollout/final-detector.test.mjs create mode 100644 tools/rollout/fixtures/approved-release.json create mode 100644 tools/rollout/release-snapshot.test.mjs create mode 100644 tools/rollout/source-resolution.test.mjs diff --git a/README.md b/README.md index ba32c6a..0fee6a5 100644 --- a/README.md +++ b/README.md @@ -259,6 +259,14 @@ A setup-only `NPM_CONFIG_USERCONFIG` pointing to a named file under `${{ runner. Local reusable workflows are already read at the same repository SHA and contribute their actual primary result instead of an automatic unknown. Missing targets and cycles remain reviewable. A remote reusable call can be resolved as having no observed JS install only when that workflow was already captured in the same organization's inventory at the requested default branch or exact SHA. A different pin/tag is never replaced with the current body. Remote callees with installs require caller-specific context and remain unresolved; no additional sources are fetched by this cross-reference step. +Local action lookup understands literal checkout mount paths only when the preceding checkout selects the captured repository and source (default selection, `github.sha`, or the exact captured SHA). A different repository/ref, conditional or sparse checkout, or unknown potentially overlapping destination remains an explicit checkout-provenance finding. The detector does not evaluate environment assignments or reusable-workflow input expressions to guess a destination/ref. In particular, a caller-selected checker revision cannot be replaced with the checker's current default-branch implementation. + +A local setup/teardown entrypoint can supply configuration evidence when all four runtime blobs match the immutable approved release, with a preceding resolved checkout. Changed/missing blobs or different source selections do not inherit this recognition. This is a source-equivalence check, not a self-test exemption or proof that earlier runtime code did not modify the files. Matrix-dependent Bun configuration, intentional direct-public fixture installs and expected Yarn failures remain reviewable; no unapproved test or archive exception is added. + +An adjacent fail-closed `bun install --help` check for `--offline` permits the exact non-composite run-step canonicalization command `bun install --lockfile-only --offline --ignore-scripts --registry=https://registry.npmjs.org/` to be recorded as capability-guarded offline validation, not a registry download. The guard must exit 1 when support is absent; intervening commands, custom shells, relevant environment overrides, composite expansion, missing flags and unguarded/later invocations do not inherit the recognition. A bare `--offline` flag is insufficient because older Bun versions can ignore it. Strict runtime assurance remains unverified. + +Literal quoted `cat` heredoc bodies are data rather than shell commands, including inside command substitutions. Shell-fed/piped heredocs, missing delimiters and GitHub expressions in the body are not granted that recognition. The lexer bounds substitutions per command (20), nesting (100), and emitted commands/words (1,000), so many independent metadata reads do not exhaust a whole job's substitution budget. Exhausted or malformed source always retains a review finding. + Only top-level `.github/workflows/*.yml|yaml`, `.depot/workflows/*.yml|yaml` and referenced local actions in the captured scope are inspected. The detector does not execute code, inspect arbitrary script/action implementations, validate credentials, prove traffic routing or certify bespoke controls. Security findings and unverified execution remain separate assurance diagnostics. No historical cohort, coverage percentage or blanket repository allowlist is used. ### Manual release snapshot verification @@ -267,7 +275,7 @@ Only top-level `.github/workflows/*.yml|yaml`, `.depot/workflows/*.yml|yaml` and npm run verify-action ``` -This separate **live, strict snapshot check** validates discovery refs, the approved signed action-only SHA, exact allowlisted tree, and runtime entrypoints against `tools/rollout/constants.mjs` and the local release manifest. It is not part of `npm run check` or ordinary source tests. It is expected to fail when `v1` advances or the source manifest no longer matches that historical snapshot; this alone is not evidence the current release is unsafe. Review/update the snapshot deliberately for a new release, rather than weakening integrity checks. Offline unit tests use an explicit historical manifest fixture so future manifest additions cannot break unrelated source CI. +This separate **live, strict snapshot check** validates discovery refs, the approved signed action-only SHA, exact allowlisted tree, and runtime entrypoints against `tools/rollout/constants.mjs` and the local release manifest. It is not part of `npm run check` or ordinary source tests. It is expected to fail when `v1` advances or the source manifest no longer matches that historical snapshot; this alone is not evidence the current release is unsafe. Review/update the snapshot deliberately for a new release, rather than weakening integrity checks. Offline unit tests use an explicit historical manifest fixture so future manifest additions cannot break unrelated source CI. `tools/rollout/fixtures/approved-release.json` also records curated public API responses for the approved immutable commit/tree/action contents and the newer discovery refs observed at capture time. Positive historical tests explicitly synthesize only the old ref targets; separate tests assert that the captured moved refs are rejected. These fixtures are independent of production constants and contain no credential, inventory, or private repository source. ## Contributing diff --git a/tools/rollout/audit.mjs b/tools/rollout/audit.mjs index f19d2ca..6133eac 100644 --- a/tools/rollout/audit.mjs +++ b/tools/rollout/audit.mjs @@ -5,9 +5,11 @@ import { classifyWorkflow, PACKAGE_MANAGER_READ, repositoryDisposition, + resolveLocalActionSource, } from "./classify.mjs"; +import { parseYamlSource } from "./yaml.mjs"; import { captureRepositoryInventory } from "./inventory.mjs"; -import { ORGANIZATION } from "./constants.mjs"; +import { APPROVED_RUNTIME_BLOBS, ORGANIZATION } from "./constants.mjs"; import { readRegistryExclusions } from "./exclusions.mjs"; import { integrationDisposition, @@ -180,6 +182,50 @@ export async function auditRepository(client, repository) { // Fetch only referenced local actions, including nested composites. The set // bounds cycles in fetching; the classifier separately bounds expansion. const localActions = new Map(); + const sourceContext = { + repository: fullName, + headSha, + localSfwRelease: Object.entries(APPROVED_RUNTIME_BLOBS).every( + ([path, sha]) => + tree.tree.some( + (entry) => + entry.path === path && + entry.sha === sha && + entry.type === "blob" && + ["100644", "100755"].includes(entry.mode), + ), + ), + }; + const mounts = new Set([""]); + for (const text of workflowTexts) { + let parsed; + try { + parsed = parseYamlSource(text); + } catch { + continue; + } // Classification reports malformed YAML. + for (const job of Object.values(parsed?.jobs ?? {})) { + for (const step of Array.isArray(job?.steps) ? job.steps : []) { + if ( + !step?.uses?.startsWith?.("actions/checkout@") || + typeof step.with?.path !== "string" + ) + continue; + const prefix = step.with.path + .replace(/^\.\//, "") + .replace(/\/+$/, ""); + const resolved = resolveLocalActionSource( + `${prefix}/action-placeholder`, + { ...sourceContext, checkouts: [step] }, + ); + if (resolved.path === "action-placeholder") mounts.add(`${prefix}/`); + } + } + } + if (mounts.size > MAX_LOCAL_ACTIONS_PER_REPOSITORY) + throw new Error( + "too many checkout mounts; refusing unbounded source lookup", + ); const pendingTexts = [...workflowTexts]; const localActionPaths = new Set(); while (pendingTexts.length > 0) { @@ -188,10 +234,13 @@ export async function auditRepository(client, repository) { for (const match of text.matchAll(LOCAL_ACTION_PATTERN)) { const base = match[1].replace(/\/+$/, ""); const prefix = base ? `${base}/` : ""; - for (const candidate of [ - `${prefix}action.yml`, - `${prefix}action.yaml`, - ]) { + const candidates = [...mounts] + .filter((mount) => prefix.startsWith(mount)) + .flatMap((mount) => [ + `${prefix.slice(mount.length)}action.yml`, + `${prefix.slice(mount.length)}action.yaml`, + ]); + for (const candidate of candidates) { if (blobPaths.has(candidate) && !localActionPaths.has(candidate)) { localActionPaths.add(candidate); paths.push(candidate); @@ -232,6 +281,7 @@ export async function auditRepository(client, repository) { const workflows = resolveLocalWorkflowCalls( workflowPaths.map((path, index) => classifyWorkflow(workflowTexts[index], { + ...sourceContext, localActions, packageManager, path, diff --git a/tools/rollout/classify.mjs b/tools/rollout/classify.mjs index 54f44fb..534e833 100644 --- a/tools/rollout/classify.mjs +++ b/tools/rollout/classify.mjs @@ -571,6 +571,51 @@ function parseUses(uses) { }; } +// Local uses paths are workspace-relative, not relative to the action file. +// Only a checkout selecting this captured source may remap a literal mount. +export function resolveLocalActionSource(path, context = {}) { + for (const step of [...(context.checkouts ?? [])].reverse()) { + if (condition(step.if) === "never") continue; + const input = step.with ?? {}; + if (!isMapping(input)) return { sourceError: "unresolved-checkout-source" }; + const mount = input.path ?? "."; + if (!literalWorkingDirectory(mount)) + return { sourceError: "unresolved-checkout-path" }; + const prefix = mount.replace(/^\.\//, "").replace(/\/$/, ""); + const root = prefix === "." || prefix === ""; + if (!root && path !== prefix && !path.startsWith(`${prefix}/`)) continue; + const repository = input.repository; + const ref = input.ref; + const sameRepository = + repository === undefined || + repository === context.repository || + normalizeExpression(repository) === "${{github.repository}}"; + const sameRef = + ref === undefined || + ref === "" || + (context.headSha !== undefined && ref === context.headSha) || + normalizeExpression(ref) === "${{github.sha}}"; + if ( + !sameRepository || + !sameRef || + condition(step.if) !== "always" || + condition(step["continue-on-error"] ?? false) !== "never" || + input["sparse-checkout"] !== undefined || + step.env !== undefined + ) + return { + sourceError: "unresolved-checkout-source", + checkoutRepository: repository ?? context.repository, + checkoutRef: ref, + }; + return { + path: root ? path : path.slice(prefix.length).replace(/^\//, ""), + checkedOut: true, + }; + } + return { path }; +} + // Bind only whole-value composite input references; never evaluate expressions // or interpolate shell text. This preserves caller token names through helpers. function bindInputs(value, inputs) { @@ -605,6 +650,28 @@ function classifyUsesStep(step, context) { const withInput = step.with ?? {}; if (uses.kind === "local") { + const source = resolveLocalActionSource(uses.path, context); + if (source.sourceError) + return [{ kind: "unknown-local-action", uses: step.uses, ...source }]; + if ( + context.localSfwRelease && + source.checkedOut && + ["", "teardown"].includes(source.path) + ) { + return classifyUsesStep( + { + ...step, + uses: `${ACTION_REPOSITORY}${source.path ? "/teardown" : ""}@${APPROVED_RELEASE_SHA}`, + }, + context, + ).map((operation) => ({ + ...operation, + uses: step.uses, + localRuntimeVerified: true, + uncertain: true, + integrationConfigurationUncertain: false, + })); + } const stack = context.actionStack ?? []; if (stack.includes(uses.path) || stack.length >= 20) { return [ @@ -616,7 +683,7 @@ function classifyUsesStep(step, context) { }, ]; } - const prefix = uses.path ? `${uses.path}/` : ""; + const prefix = source.path ? `${source.path}/` : ""; const actionText = context.localActions?.get(`${prefix}action.yml`) ?? context.localActions?.get(`${prefix}action.yaml`); @@ -808,6 +875,25 @@ export function classifyStep(step, context = {}) { ...step, run: `npm install --global ${context.packageManager} --ignore-scripts --no-audit --no-fund`, }; + const lines = + typeof step.run === "string" + ? step.run + .trim() + .split("\n") + .map((line) => line.trim()) + : []; + const offlineBun = + !context.actionStack?.length && + context.offlineBunAllowed !== false && + !environmentUncertain(step.env) && + (step.shell === undefined || SUPPORTED_SHELLS.has(step.shell)) && + lines[0] === + "if ! bun install --help | grep -F -- '--offline ' >/dev/null; then" && + /^echo '[^'\r\n$`\\]*' >&2$/.test(lines[1] ?? "") && + lines[2] === "exit 1" && + lines[3] === "fi" && + lines[4] === + "bun install --lockfile-only --offline --ignore-scripts --registry=https://registry.npmjs.org/"; let operations; let uncertain = boundaryUncertain(step) || pinnedBootstrap; let integrationEnv = step.env; @@ -866,12 +952,24 @@ export function classifyStep(step, context = {}) { shell.commands.some((command) => /^(?:if|case|for|while|until|select)\b/.test(command), ); + let offlineCommandPending = offlineBun; operations = shell.commands.map((command) => { - const operation = classifyCommand(command, context); + const offlineValidation = offlineCommandPending && command === lines[4]; + if (offlineValidation) offlineCommandPending = false; + const operation = offlineValidation + ? { + command, + kind: "no-network", + offlineValidation: true, + uncertain: true, + } + : classifyCommand(command, context); const { words, rawWords, environment, preservedEnvironment } = commandParts(command, context); const bunArguments = approvedBunArguments(words, rawWords); const tokens = shellTokens(command); + if (tokens.limit || tokens.lexError) + operation.sourceError = "unresolved-shell-source"; const nestedCommands = tokens.substitutions.flatMap( (body) => shellCommands(body).commands, ); @@ -926,15 +1024,16 @@ export function classifyStep(step, context = {}) { installerArguments(args).some((word) => REGISTRY_FLAG.test(word)), ); const registryMutating = - setter || - dynamicSetter || - configFileWrite || - nestedOverride || - (["npm", "pnpm", "bun", "yarn", "npx", "bunx"].includes(words[0]) && - configurationWords.some((word) => REGISTRY_FLAG.test(word))) || - Object.keys(environment).some((key) => - /^(?:npm|pnpm|bun)_CONFIG_/i.test(key), - ); + !offlineValidation && + (setter || + dynamicSetter || + configFileWrite || + nestedOverride || + (["npm", "pnpm", "bun", "yarn", "npx", "bunx"].includes(words[0]) && + configurationWords.some((word) => REGISTRY_FLAG.test(word))) || + Object.keys(environment).some((key) => + /^(?:npm|pnpm|bun)_CONFIG_/i.test(key), + )); // A literal npm prefix selects the project directory, just like a // run-step working-directory. Do not accept dynamic or escaping paths. const directoryArguments = new Set(); @@ -1419,6 +1518,11 @@ export function classifyJob(jobName, job, context = {}, triggers = []) { ); const stepContext = { ...context, + offlineBunAllowed: + !environmentUncertain(job.env) && + !defaultsUncertain(job.defaults) && + !context.integrationContextUncertain && + job.container === undefined, registryExclusion: defaultCheckout && context.exclusionRootDirectory !== false && @@ -1432,13 +1536,20 @@ export function classifyJob(jobName, job, context = {}, triggers = []) { defaultCheckout && [undefined, ".", "./"].includes(job.defaults?.run?.["working-directory"]) && context.exclusionRootDirectory !== false; + const priorCheckouts = []; const operations = condition(job.if) === "never" ? [] : steps.flatMap((step, index) => { const afterCheckout = index > (checkout?.index ?? -1); + if ( + typeof step?.uses === "string" && + step.uses.startsWith("actions/checkout@") + ) + priorCheckouts.push(step); const currentContext = { ...stepContext, + checkouts: priorCheckouts, registryExclusion: afterCheckout ? stepContext.registryExclusion : undefined, diff --git a/tools/rollout/commands.mjs b/tools/rollout/commands.mjs index 6c9ec8f..8033ffb 100644 --- a/tools/rollout/commands.mjs +++ b/tools/rollout/commands.mjs @@ -5,6 +5,8 @@ function lex(source, words = false) { const substitutions = []; const groups = []; const backticks = []; + const heredocs = []; + let heredocError = false; let command = ""; let quote; let escaped = false; @@ -12,10 +14,15 @@ function lex(source, words = false) { let limit = false; let ambiguous = /<<|^\s*(?:function\s|[\w-]+\s*\(\s*\)\s*\{)/m.test(source); const flush = () => { - if (command.trim()) commands.push(command.trim()); + if (command.trim()) { + if (commands.length < 1000) commands.push(command.trim()); + else limit = true; + } command = ""; }; const capture = (text) => { + // Word analysis bounds substitutions per command, not across unrelated reads. + if (!words) return; if (substitutions.length < 20) substitutions.push(text); else limit = true; }; @@ -59,6 +66,42 @@ function lex(source, words = false) { index += 1; continue; } + if (char === "\n" && !quote && heredocs.length) { + let cursor = index + 1; + for (const { delimiter, tabs } of heredocs.splice(0)) { + const start = cursor; + let found = false; + while (cursor < source.length) { + const end = source.indexOf("\n", cursor); + const lineEnd = end === -1 ? source.length : end; + const line = source.slice(cursor, lineEnd); + if ((tabs ? line.replace(/^\t+/, "") : line) === delimiter) { + heredocError ||= source.slice(start, cursor).includes("${{"); + cursor = lineEnd; + found = true; + break; + } + cursor = lineEnd + 1; + } + heredocError ||= !found; + if (!words || groups.length) command += `\n${delimiter}`; + } + index = cursor - 1; + continue; + } + if (!quote && char === "<" && source[index - 1] !== "<") { + // Only literal cat data, not a heredoc executed by bash/sh or a pipeline. + const match = source + .slice(index) + .match(/^<<(-?)[ \t]*(['"])([A-Za-z_][A-Za-z0-9_]*)\2[ \t]*(?=\n|$)/); + if ( + match && + /(?:^|\$\()cat(?:[ \t]+>{1,2}[ \t]+[\w./-]+)?[ \t]*$/.test( + words ? source.slice(0, index) : command, + ) + ) + heredocs.push({ delimiter: match[3], tabs: match[1] === "-" }); + } if (quote) { command += char; if (char === quote) quote = undefined; @@ -110,7 +153,9 @@ function lex(source, words = false) { } if (escaped) command += "\\"; flush(); - const lexError = Boolean(quote || groups.length || escaped); + const lexError = Boolean( + quote || groups.length || escaped || heredocError || heredocs.length, + ); return { commands, substitutions, diff --git a/tools/rollout/configuration-presence.test.mjs b/tools/rollout/configuration-presence.test.mjs index 471d8d5..93216c1 100644 --- a/tools/rollout/configuration-presence.test.mjs +++ b/tools/rollout/configuration-presence.test.mjs @@ -164,7 +164,7 @@ test("lexical boundaries do not turn quoted data into installs", () => { ['VALUE="$(node --print \'require("./package.json").name\')"', "npm ci"], ); assert.equal(shellCommands("cat <<'EOF'\nnpm ci\nEOF").ambiguous, true); - assert.equal(primary([{ run: "cat <<'EOF'\nnpm ci\nEOF" }]), "needs-review"); + assert.equal(primary([{ run: "cat <<'EOF'\nnpm ci\nEOF" }]), "no-js-ci"); }); test("unparsed explicit JS invocation retains setup evidence, not execution assurance", () => { diff --git a/tools/rollout/constants.mjs b/tools/rollout/constants.mjs index 11a9a1c..8eb3d87 100644 --- a/tools/rollout/constants.mjs +++ b/tools/rollout/constants.mjs @@ -4,6 +4,14 @@ export const RELEASE_CHANNEL = "v1"; export const RELEASE_BRANCH = `action-release/${RELEASE_CHANNEL}`; export const APPROVED_RELEASE_SHA = "ca93dd8aa351f54f4729fe3377a9be23c631c25d"; +// Runtime blobs read from the immutable approved release, not mutable discovery refs. +export const APPROVED_RUNTIME_BLOBS = Object.freeze({ + "action.yml": "156de46f25b2facc56cd7a5b2ea9e78b3de4ea1d", + "scripts/configure.sh": "155d6aab883d58211a4bb3fbfa70b27a2d96dae2", + "scripts/teardown.sh": "a1ddc2f06a65c607f2af2ae0395bada78e236913", + "teardown/action.yml": "d47a0abeaaa0d6717e5ce9cdbc45c3356ea3726c", +}); + export const EXPECTED_RELEASE_TREE = Object.freeze([ Object.freeze({ mode: "100644", path: "LICENSE", type: "blob" }), Object.freeze({ mode: "100644", path: "action.yml", type: "blob" }), diff --git a/tools/rollout/final-detector.test.mjs b/tools/rollout/final-detector.test.mjs new file mode 100644 index 0000000..f7ac332 --- /dev/null +++ b/tools/rollout/final-detector.test.mjs @@ -0,0 +1,115 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { classifyJob } from "./classify.mjs"; +import { shellCommands, shellTokens } from "./commands.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; + +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { + token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}", + "configure-bun": true, + }, +}; +const inspect = (steps, extra = {}) => + classifyJob("test", { steps, ...extra }, { visibility: "private" }); +const offline = `if ! bun install --help | grep -F -- '--offline ' >/dev/null; then + echo 'Offline support is required.' >&2 + exit 1 +fi +bun install --lockfile-only --offline --ignore-scripts --registry=https://registry.npmjs.org/`; + +test("Bun offline canonicalization requires its adjacent fail-closed capability guard", () => { + const result = inspect([ + setup, + { run: "bun install --frozen-lockfile" }, + { run: offline }, + ]); + assert.equal(result.integration.disposition, "integrated"); + assert.equal(result.integration.downloads.length, 1); + assert.equal(result.status, "unknown"); + assert.equal(inspect([{ run: offline }]).integration.disposition, "no-js-ci"); + for (const run of [ + offline.split("\n").at(-1), + offline.replace("exit 1", "exit 0"), + offline.replace("--offline --ignore-scripts", "--offline"), + offline.replace("fi\nbun", "fi\nexport PATH=/elsewhere\nbun"), + offline.replace("--offline '", "--offline'"), + offline.replace("'Offline support is required.'", '"$(npm ci)"'), + ]) { + assert.notEqual( + inspect([setup, { run }]).integration.disposition, + "integrated", + run, + ); + } + assert.notEqual( + inspect([setup, { run: offline, env: { PATH: "/elsewhere" } }]).integration + .disposition, + "integrated", + ); + assert.notEqual( + inspect([setup, { run: offline }], { env: { PATH: "/elsewhere" } }) + .integration.disposition, + "integrated", + ); + assert.notEqual( + inspect([setup, { run: offline, shell: "python" }]).integration.disposition, + "integrated", + ); + assert.equal( + inspect([ + setup, + { run: offline + "\nexport PATH=/other\n" + offline.split("\n").at(-1) }, + ]).integration.downloads.length, + 1, + ); +}); + +test("substitution budget is per command, not every independent read in a job", () => { + const run = Array.from( + { length: 30 }, + (_, i) => `value${i}=$(printf value)`, + ).join("\n"); + assert.equal(shellCommands(run).limit, false); + assert.equal(inspect([{ run }]).integration.disposition, "no-js-ci"); + // A single command exceeding the bound must not hide its last substitution. + const many = `echo ${"$(printf value) ".repeat(20)}$(npm ci)`; + assert.equal(shellTokens(many).limit, true); + assert.equal( + inspect([setup, { run: many }]).integration.disposition, + "needs-review", + ); +}); + +test("quoted heredoc bodies are data, including quotes and apparent installers", () => { + for (const delimiter of ["'EOF'", '"EOF"']) { + const run = `prompt=$(cat <<${delimiter}\nIt's a prompt: npm ci\n$(npm install fake)\n\`bun install\`\n((((\nEOF\n)\necho "$prompt"`; + assert.equal(shellCommands(run).lexError, false); + assert.equal(inspect([{ run }]).integration.disposition, "no-js-ci"); + assert.equal( + inspect([{ run: run + "\nnpm ci" }]).integration.disposition, + "needs-review", + ); + assert.equal( + inspect([setup, { run: run + "\nnpm ci" }]).integration.downloads.length, + 1, + ); + } + assert.equal( + inspect([{ run: "cat <<-'EOF'\n\tnpm ci\n\tEOF\n" }]).integration + .disposition, + "no-js-ci", + ); + for (const run of [ + "cat <<'EOF'\nunclosed", + "cat <<'EOF'\n${{ inputs.untrusted }}\nEOF", + "cat < [ + `${prefix}/contents/${path}?ref=${data.commit.sha}`, + body, + ]), + ...Object.values(data.discovery).map((body) => [ + `${prefix}/git/ref/${body.ref.slice(5)}`, + historicalRefs + ? { ...body, object: { type: "commit", sha: data.commit.sha } } + : body, + ]), + ]); + return new GitHubClient({ + execute: async (args) => { + assert.deepEqual(args.slice(0, 3), ["api", "--method", "GET"]); + assert.ok(replies.has(args[3]), args[3]); + return { stdout: JSON.stringify(replies.get(args[3])) }; + }, + }); +} +const verify = (data, historicalRefs = false) => + verifyActionRelease({ + client: client(data, historicalRefs), + manifestUrl: new URL("./fixtures/release-manifest.txt", import.meta.url), + }); + +test("captured immutable API shapes verify without manufacturing a tree from production constants", async () => { + // Only discovery refs are synthesized to the historical commit. The capture + // records the newer live refs separately; it does not claim they still match. + const result = await verify(snapshot, true); + assert.equal(result.sha, snapshot.commit.sha); + assert.equal(result.treeSha, snapshot.tree.sha); + for (const [path, sha] of Object.entries(APPROVED_RUNTIME_BLOBS)) + assert.equal( + snapshot.tree.tree.find((entry) => entry.path === path)?.sha, + sha, + ); +}); + +test("captured moved discovery refs fail the historical verifier without entering normal source CI", async () => { + assert.notEqual(snapshot.discovery.branch.object.sha, snapshot.commit.sha); + await assert.rejects(verify(snapshot), /unapproved SHA/); +}); + +test("captured responses retain signature, tree and content integrity failures", async () => { + for (const [mutate, message] of [ + [ + (data) => { + data.commit.verification.verified = false; + }, + /not GitHub-verified/, + ], + [ + (data) => { + data.tree.truncated = true; + }, + /truncated/, + ], + [ + (data) => { + data.tree.tree[0].mode = "120000"; + }, + /tree differs/, + ], + [ + (data) => { + data.contents["action.yml"].type = "symlink"; + }, + /not a complete base64/, + ], + ]) { + const data = structuredClone(snapshot); + mutate(data); + await assert.rejects(verify(data, true), message); + } +}); diff --git a/tools/rollout/source-resolution.test.mjs b/tools/rollout/source-resolution.test.mjs new file mode 100644 index 0000000..b8e5b78 --- /dev/null +++ b/tools/rollout/source-resolution.test.mjs @@ -0,0 +1,220 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import test from "node:test"; +import { stringify } from "yaml"; +import { auditRepository } from "./audit.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; + +const sha = "c".repeat(40); +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}" }, +}; +const action = stringify({ + runs: { using: "composite", steps: [{ shell: "bash", run: "npm ci" }] }, +}); +const captured = JSON.parse( + readFileSync(new URL("./fixtures/approved-release.json", import.meta.url)), +); +async function audit( + steps, + files = { "actions/install/action.yml": action }, + entries = [], +) { + files = { + ...files, + ".github/workflows/ci.yml": stringify({ + on: "push", + jobs: { test: { "runs-on": "ubuntu-latest", steps } }, + }), + }; + const reads = []; + const result = await auditRepository( + { + async getRef() { + return { object: { sha } }; + }, + async getTree() { + return { + truncated: false, + tree: [ + ...Object.keys(files) + .filter((path) => !entries.some((e) => e.path === path)) + .map((path) => ({ path, type: "blob", mode: "100644" })), + ...entries, + ], + }; + }, + async getText(_repo, path, ref) { + assert.equal(ref, sha); + reads.push(path); + assert.ok(Object.hasOwn(files, path), path); + return files[path]; + }, + }, + { name: "fixture", defaultBranch: "main", visibility: "private" }, + ); + return { result, reads }; +} + +test("a same-snapshot checkout mount resolves its local action, preserving uncovered installs", async () => { + const checkout = { uses: "actions/checkout@v4", with: { path: "source" } }; + const local = { uses: "./source/actions/install" }; + const { result, reads } = await audit([checkout, setup, local]); + assert.equal(result.disposition, "integrated"); + assert.ok(reads.includes("actions/install/action.yml")); + assert.equal( + (await audit([checkout, local])).result.disposition, + "needs-sfw", + ); + assert.equal( + (await audit([local, checkout])).result.disposition, + "needs-review", + ); + for (const withInput of [ + { path: "source", repository: "other/repo" }, + { path: "source", ref: "other-branch" }, + { path: "source", ref: "${{ inputs.checker-ref }}" }, + { path: "source", "sparse-checkout": "something-else" }, + { path: "../source" }, + ]) + assert.equal( + (await audit([{ ...checkout, with: withInput }, setup, local])).result + .disposition, + "needs-review", + ); + for (const extra of [ + { if: "inputs.enabled" }, + { "continue-on-error": true }, + { env: { PATH: "/other" } }, + ]) + assert.equal( + (await audit([{ ...checkout, ...extra }, setup, local])).result + .disposition, + "needs-review", + ); + assert.equal( + ( + await audit([ + checkout, + { + uses: "actions/checkout@v4", + with: { repository: "other/repo", path: "${{ env.DESTINATION }}" }, + }, + setup, + local, + ]) + ).result.disposition, + "needs-review", + ); +}); + +test("a caller-selected checker revision is unresolved provenance, not a nonexistent local action", async () => { + const { result, reads } = await audit([ + { uses: "actions/checkout@v4" }, + { + uses: "actions/checkout@v4", + with: { + repository: "workos/fixture", + ref: "${{ inputs.checker-ref }}", + path: ".checker", + }, + }, + { uses: "./.checker/actions/install" }, + ]); + assert.equal(result.disposition, "needs-review"); + const operation = result.workflows[0].jobs[0].operations.find( + (o) => o.kind === "unknown-local-action", + ); + assert.equal(operation.sourceError, "unresolved-checkout-source"); + assert.equal(operation.checkoutRef, "${{ inputs.checker-ref }}"); + assert.equal(reads.includes("actions/install/action.yml"), false); +}); + +test("local SFW entrypoints require all immutable reviewed runtime blobs", async () => { + const files = Object.fromEntries( + Object.entries(captured.contents).map(([path, entry]) => [ + path, + Buffer.from(entry.content, "base64").toString(), + ]), + ); + const entries = captured.tree.tree.filter((e) => e.type === "blob"); + const steps = [ + { uses: "actions/checkout@v4" }, + { uses: "./", with: setup.with }, + { run: "npm ci" }, + ]; + const { result } = await audit(steps, files, entries); + assert.equal(result.disposition, "integrated"); + assert.equal(result.assuranceDisposition, "needs-review"); + assert.ok( + result.workflows[0].jobs[0].integration.notes.includes( + "local-runtime-matches-approved-release", + ), + ); + for (const path of [ + "action.yml", + "scripts/configure.sh", + "scripts/teardown.sh", + "teardown/action.yml", + ]) + assert.notEqual( + ( + await audit( + steps, + files, + entries.map((e) => + e.path === path ? { ...e, sha: "f".repeat(40) } : e, + ), + ) + ).result.disposition, + "integrated", + ); + assert.equal( + ( + await audit( + [ + { uses: "actions/checkout@v4", with: { ref: "other" } }, + ...steps.slice(1), + ], + files, + entries, + ) + ).result.disposition, + "needs-sfw", + ); + assert.notEqual( + ( + await audit( + [steps[0], { ...steps[1], with: {} }, steps[2]], + files, + entries, + ) + ).result.disposition, + "integrated", + ); + assert.notEqual( + (await audit(steps.slice(1), files, entries)).result.disposition, + "integrated", + ); + assert.notEqual( + ( + await audit( + [{ ...steps[0], if: false }, ...steps.slice(1)], + files, + entries, + ) + ).result.disposition, + "integrated", + ); + assert.equal( + ( + await audit( + [...steps, { uses: "./teardown" }, { run: "npm ci" }], + files, + entries, + ) + ).result.disposition, + "needs-sfw", + ); +}); From 35960be81f23011d8652b56500f7b2b1ba087e3d Mon Sep 17 00:00:00 2001 From: "workos-tars[bot]" <269013284+workos-tars[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 20:12:31 +0000 Subject: [PATCH 15/19] fix(audit): retain reusable caller checkout provenance --- README.md | 2 +- tools/rollout/classify.mjs | 13 +++++++++---- tools/rollout/source-resolution.test.mjs | 21 ++++++++++++++++++++- 3 files changed, 30 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index 0fee6a5..ae20a82 100644 --- a/README.md +++ b/README.md @@ -259,7 +259,7 @@ A setup-only `NPM_CONFIG_USERCONFIG` pointing to a named file under `${{ runner. Local reusable workflows are already read at the same repository SHA and contribute their actual primary result instead of an automatic unknown. Missing targets and cycles remain reviewable. A remote reusable call can be resolved as having no observed JS install only when that workflow was already captured in the same organization's inventory at the requested default branch or exact SHA. A different pin/tag is never replaced with the current body. Remote callees with installs require caller-specific context and remain unresolved; no additional sources are fetched by this cross-reference step. -Local action lookup understands literal checkout mount paths only when the preceding checkout selects the captured repository and source (default selection, `github.sha`, or the exact captured SHA). A different repository/ref, conditional or sparse checkout, or unknown potentially overlapping destination remains an explicit checkout-provenance finding. The detector does not evaluate environment assignments or reusable-workflow input expressions to guess a destination/ref. In particular, a caller-selected checker revision cannot be replaced with the checker's current default-branch implementation. +Local action lookup understands literal checkout mount paths only when the preceding checkout selects the captured repository and source (default selection, `github.sha`, or the exact captured SHA). A different repository/ref, conditional or sparse checkout, or unknown potentially overlapping destination remains an explicit checkout-provenance finding. The detector does not evaluate environment assignments or reusable-workflow input expressions to guess a destination/ref. In particular, a caller-selected checker revision cannot be replaced with the checker's current default-branch implementation. In reusable workflows, implicit checkout and `github.repository`/`github.sha` refer to the caller; resolving a local source from the callee requires an explicit matching repository and source selection. A local setup/teardown entrypoint can supply configuration evidence when all four runtime blobs match the immutable approved release, with a preceding resolved checkout. Changed/missing blobs or different source selections do not inherit this recognition. This is a source-equivalence check, not a self-test exemption or proof that earlier runtime code did not modify the files. Matrix-dependent Bun configuration, intentional direct-public fixture installs and expected Yarn failures remain reviewable; no unapproved test or archive exception is added. diff --git a/tools/rollout/classify.mjs b/tools/rollout/classify.mjs index 534e833..f075e1f 100644 --- a/tools/rollout/classify.mjs +++ b/tools/rollout/classify.mjs @@ -586,15 +586,19 @@ export function resolveLocalActionSource(path, context = {}) { if (!root && path !== prefix && !path.startsWith(`${prefix}/`)) continue; const repository = input.repository; const ref = input.ref; + // In a reusable workflow, implicit checkout/github.repository select the + // caller, not necessarily the repository containing this workflow. const sameRepository = - repository === undefined || - repository === context.repository || - normalizeExpression(repository) === "${{github.repository}}"; + (typeof repository === "string" && repository === context.repository) || + (!context.reusableWorkflow && + (repository === undefined || + normalizeExpression(repository) === "${{github.repository}}")); const sameRef = ref === undefined || ref === "" || (context.headSha !== undefined && ref === context.headSha) || - normalizeExpression(ref) === "${{github.sha}}"; + (!context.reusableWorkflow && + normalizeExpression(ref) === "${{github.sha}}"); if ( !sameRepository || !sameRef || @@ -1518,6 +1522,7 @@ export function classifyJob(jobName, job, context = {}, triggers = []) { ); const stepContext = { ...context, + reusableWorkflow: triggers.includes("workflow_call"), offlineBunAllowed: !environmentUncertain(job.env) && !defaultsUncertain(job.defaults) && diff --git a/tools/rollout/source-resolution.test.mjs b/tools/rollout/source-resolution.test.mjs index b8e5b78..86231c2 100644 --- a/tools/rollout/source-resolution.test.mjs +++ b/tools/rollout/source-resolution.test.mjs @@ -20,11 +20,12 @@ async function audit( steps, files = { "actions/install/action.yml": action }, entries = [], + on = "push", ) { files = { ...files, ".github/workflows/ci.yml": stringify({ - on: "push", + on, jobs: { test: { "runs-on": "ubuntu-latest", steps } }, }), }; @@ -207,6 +208,24 @@ test("local SFW entrypoints require all immutable reviewed runtime blobs", async ).result.disposition, "integrated", ); + assert.notEqual( + (await audit(steps, files, entries, "workflow_call")).result.disposition, + "integrated", + ); + assert.equal( + ( + await audit( + [ + { ...steps[0], with: { repository: "workos/fixture", ref: sha } }, + ...steps.slice(1), + ], + files, + entries, + "workflow_call", + ) + ).result.disposition, + "integrated", + ); assert.equal( ( await audit( From d7b87b4013c490eb3b6c238b3a9d5e1e374ac4a4 Mon Sep 17 00:00:00 2001 From: "workos-tars[bot]" <269013284+workos-tars[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:03:40 +0000 Subject: [PATCH 16/19] feat(audit): add deterministic weekly review with durable decisions --- CONTRIBUTING.md | 4 + README.md | 47 ++++ package.json | 2 + tools/rollout/audit.mjs | 46 +++- tools/rollout/classify.mjs | 63 ++++- tools/rollout/fingerprint.mjs | 29 +++ tools/rollout/integration.mjs | 25 ++ tools/rollout/inventory.mjs | 1 + tools/rollout/review-cli.mjs | 179 +++++++++++++ tools/rollout/review.mjs | 376 +++++++++++++++++++++++++++ tools/rollout/review.test.mjs | 476 ++++++++++++++++++++++++++++++++++ 11 files changed, 1235 insertions(+), 13 deletions(-) create mode 100644 tools/rollout/fingerprint.mjs create mode 100644 tools/rollout/review-cli.mjs create mode 100644 tools/rollout/review.mjs create mode 100644 tools/rollout/review.test.mjs diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 74ea349..979d31b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -42,6 +42,10 @@ The minimum test-coverage policy is one shell test suite for every executable sh `npm run inventory` and `npm run audit:live` are read-only manual commands using an existing authorized `gh` session. Confirm organization-wide read access separately before making organization-wide claims. No scheduling or remediation is implicit. See README for the scan scope and limitations. +For recurring agent use, `npm run review:weekly -- --state /durable/private/ledger.json` runs the audit and applies explicit fingerprint-bound decisions. `npm run review -- init|show|record|forget --state ...` manages that private ledger; see README for exact arguments. Never auto-initialize a missing weekly ledger, auto-acknowledge new cases, expire decisions by elapsed time, or publish the ledger/source report in this public repository. The weekly JSON intentionally contains private repository/job metadata; raw audit terminal output remains counts-only. + +Weekly-review changes require deterministic replay tests, unrelated-commit/order stability, new-repository/job and input-drift reopening, immutable repository IDs, durable decisions across visibility loss, and fail-closed missing/corrupt/partial state and concurrent writes. Test the next run after recording/fixing a finding, not only the first clean result. This adds report fingerprints/IDs to schema3 without changing primary or strict classification semantics. + Audit schema3 separates primary observed configuration from strict `assuranceDispositions`. Acceptance tests must show that arbitrary execution before or after setup, pipelines, package scripts and script-only siblings do not invent missing installs or erase configuration. An actual uncovered install, explicit configuration conflict or unresolved relevant source must remain visible. Cover conditional Corepack state, scoped environment/registry changes, quoted data versus executable commands, executor payload boundaries, composite input binding and snapshot-bound reusable-workflow references. Preserve strict security/opacity diagnostics and source acquisition limits. `integrated` is configuration evidence, not runtime proof. `no-js-ci` is absence of an observed install path, not assurance about script bodies. Do not introduce a historical cohort, coverage percentage or blanket repository allowlist. Approved source exclusions must remain explicitly recorded, pin-checked and separate from protection claims. Full results are owner-only, ignored `reports/inventory.json` and `reports/live-audit.json`; terminal JSON contains sanitized counts. Audit `scanErrors`/`scanStatus` distinguish operational failures from discovered gaps. Exit 1 includes partial scans; exit 0 can still include `needs-sfw` or `needs-review`. Do not force-add reports, expose inventory/source in logs, or upload full reports as artifacts. diff --git a/README.md b/README.md index ae20a82..4efe7c5 100644 --- a/README.md +++ b/README.md @@ -210,6 +210,53 @@ npm run audit:live # read-only, default-branch workflow candidate scan `inventory` and `audit:live` require an existing authenticated `gh` session. Organization-wide conclusions require independently confirmed read access to **every intended repository**, including private/internal repositories (typically `repo` and `read:org`, with organization/SSO authorization as applicable). REST/GraphQL reconciliation checks consistency of the token-visible inventory; both APIs agreeing does **not** prove hidden repositories are absent. Archived repositories are counted but not scanned. Each repository's tree and workflow/local-action source use one captured immutable default-branch SHA; different repositories are not captured transactionally. +### Weekly agent review: stable findings and durable decisions + +Use the review command for recurring work, rather than treating every raw `needs-review` row as a new protection request. It runs the same read-only audit, but keeps **unacknowledged protection gaps**, **unacknowledged uncertainty**, **known decisions**, and **unobserved repositories** separate. It creates no schedule, notification, consumer PR, or protection waiver. + +Use a private, durable ledger shared by successive executions. Do not put it in an ephemeral checkout, initialize it every week, or commit it to this public repository. Back it up or version it in your private operations storage. A missing/corrupt ledger fails closed; it never silently restarts with an empty exception list. + +```bash +STATE=/durable/private/socket-firewall-review.json +REPORT=/durable/private/socket-firewall-audit.json + +# Once only. Fails rather than overwriting any existing ledger. +npm run --silent review -- init --state "$STATE" + +# This is the command to give the weekly agent. +npm run --silent review:weekly -- --state "$STATE" --report "$REPORT" +``` + +The JSON result contains `snapshot`, `needsProtection`, `needsReview`, `known`, and `unobserved`. Each case has an opaque stable `id`, immutable GitHub repository ID, repository/workflow/job, reason codes and an input fingerprint. Names and links in this output are private operational metadata; unlike the raw audit CLI's counts-only output, do not post it to a public log. The separate full audit still retains all original classifications and strict assurance diagnostics. + +For a finding the agent has investigated, record a specific reason, evidence URL, and recorder identity against the **snapshot it actually inspected**: + +```bash +npm run --silent review -- record --state "$STATE" \ + --expected '' --case '' \ + --kind known-review \ + --reason 'Known caller-selected source uncertainty; not evidence of a missing setup.' \ + --evidence 'https://github.com/owner/repo/pull/123' --by 'reviewer identity' +``` + +Decision kinds: + +- `known-review`: acknowledge a known uncertainty/limitation without claiming protection. +- `tracked-gap`: retain a real, unfinished gap linked to its remediation work, without presenting it as new every week. +- `exception`: record an explicitly justified policy exception with its approval evidence. The command records a decision; it does **not** grant approval or relabel the underlying audit as integrated. Follow your existing policy authority before using this kind. + +`--repository workos/name` can replace `--case` to record the current findings in that exact repository at the inspected snapshot. It expands to individual fingerprinted cases, **not** a wildcard repository allowlist: a new job or changed case is not included. `review show --state "$STATE"` reads the saved view without GitHub access. `review forget --state "$STATE" --case '' --expected ''` explicitly revokes a decision. Agents should report unacknowledged cases and visibility/scan failures, not repeatedly re-open known cases or automatically acknowledge new findings. + +Determinism and invalidation rules: + +- Stable repository IDs survive renames but do not transfer decisions to a deleted/recreated repository with the same name. Cases are scoped to a workflow/job (or a parse/exclusion finding), never the entire organization or a historical cohort. +- Job inputs, workflow environment/defaults/triggers, declared upstream jobs, consumed local actions, package-manager configuration and relevant local reusable sources bind each decision. Canonical object ordering and YAML parsing remove API completion order/key order and YAML-comment noise. Unrelated commits, README changes, other independent jobs and wall-clock time do not invalidate it. Raw audit timestamps and captured head SHAs are provenance, not exception identities. +- A changed input invalidates its decision durably. Reverting that input later does not silently revive the old decision. Fixing/removing a job and subsequently reintroducing the old gap is a new review event. Decisions have no automatic TTL/calendar expiry. +- A previously tracked finding/decision whose repository is no longer visible is `unobserved`, **not resolved**; it neither expires nor erases a decision. A returning unchanged repository retains it. An unreadable/partial audit or inventory disagreement fails and leaves the last-good ledger untouched. Never use that retained view as proof the failed run was clean. +- Run/record/forget/init serialize through an exclusive `.lock` file; writes replace the ledger atomically with mode `0600`. Concurrent writers fail rather than lose decisions. A crashed writer leaves a lock: verify no writer is active before removing that exact lock. There is no timeout-based lock stealing or automatic ledger reset. + +These are stable acknowledgements of **bounded static findings**, not attestations about arbitrary scripts, remote code or credentials. A known dynamic-source limitation remains a limitation. Default-branch sources can genuinely change between live scans; deterministic output means the same observed inputs and ledger produce the same review list, not that real configuration changes are ignored. Fingerprint format changes require an explicit migration/review, never silently discarding the ledger. + ### Results and privacy Terminal output is sanitized JSON counts, not repository names or workflow source. Full inventory and audit JSON go to `reports/inventory.json` and `reports/live-audit.json`, respectively. These paths are ignored; reports are atomically replaced with owner-only (`0600`) permissions. Keep them private: never force-add, publish, or upload reports as CI artifacts. Both commands take no CLI flags; a later run replaces the previous report. diff --git a/package.json b/package.json index 35e8612..091e91e 100644 --- a/package.json +++ b/package.json @@ -9,6 +9,8 @@ }, "scripts": { "audit:live": "node tools/rollout/cli.mjs audit", + "review": "node tools/rollout/review-cli.mjs", + "review:weekly": "node tools/rollout/review-cli.mjs run", "check": "npm run format:check && npm test", "format": "prettier --write package.json tools/rollout/*.mjs", "format:check": "prettier --check package.json tools/rollout/*.mjs", diff --git a/tools/rollout/audit.mjs b/tools/rollout/audit.mjs index 6133eac..f350371 100644 --- a/tools/rollout/audit.mjs +++ b/tools/rollout/audit.mjs @@ -11,6 +11,7 @@ import { parseYamlSource } from "./yaml.mjs"; import { captureRepositoryInventory } from "./inventory.mjs"; import { APPROVED_RUNTIME_BLOBS, ORGANIZATION } from "./constants.mjs"; import { readRegistryExclusions } from "./exclusions.mjs"; +import { fingerprint, fingerprintYaml } from "./fingerprint.mjs"; import { integrationDisposition, resolveLocalWorkflowCalls, @@ -162,11 +163,14 @@ export async function auditRepository(client, repository) { } try { + const sourceDigests = new Map(); const readSource = async (path) => { if (tree.tree.find((entry) => entry.path === path)?.mode === "120000") { throw new Error("workflow or local action source is a symlink"); } - return client.getText(fullName, path, headSha); + const text = await client.getText(fullName, path, headSha); + if (typeof text === "string") sourceDigests.set(path, fingerprint(text)); + return text; }; const workflowTexts = await mapWithConcurrency( workflowPaths, @@ -183,7 +187,20 @@ export async function auditRepository(client, repository) { // bounds cycles in fetching; the classifier separately bounds expansion. const localActions = new Map(); const sourceContext = { + // Configuration changes matter; unrelated commits and dependency-lock + // churn do not invalidate a reviewed workflow job. + reviewConfiguration: fingerprint( + tree.tree + .filter((entry) => + /(?:^|\/)(?:\.npmrc|\.?bunfig\.toml|\.yarnrc(?:\.yml)?|pnpm-workspace\.yaml|pnpm-config\.json)$/.test( + entry.path, + ), + ) + .map(({ path, mode, type, sha }) => ({ path, mode, type, sha })) + .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)), + ), repository: fullName, + defaultBranch: repository.defaultBranch, headSha, localSfwRelease: Object.entries(APPROVED_RUNTIME_BLOBS).every( ([path, sha]) => @@ -274,13 +291,23 @@ export async function auditRepository(client, repository) { workflowTexts.some((text) => text.includes(PACKAGE_MANAGER_READ)) ? JSON.parse(await readSource("package.json"))?.packageManager : undefined; - const exclusions = await readRegistryExclusions( - repository.name, - readSource, - ); + const exclusions = ( + await readRegistryExclusions(repository.name, readSource) + ).map((entry) => ({ + ...entry, + reviewFingerprint: fingerprint({ + rule: entry, + inputs: Object.fromEntries( + ["package.json", entry.lockfile, ".npmrc"].map((path) => [ + path, + sourceDigests.get(path), + ]), + ), + }), + })); const workflows = resolveLocalWorkflowCalls( - workflowPaths.map((path, index) => - classifyWorkflow(workflowTexts[index], { + workflowPaths.map((path, index) => ({ + ...classifyWorkflow(workflowTexts[index], { ...sourceContext, localActions, packageManager, @@ -291,7 +318,8 @@ export async function auditRepository(client, repository) { ), visibility: repository.visibility, }), - ), + reviewFingerprint: fingerprintYaml(workflowTexts[index]), + })), ); const managers = [ @@ -355,7 +383,7 @@ export async function runAudit(client, options = {}) { const row = await auditRepository(client, repository); completed += 1; progress(completed, inventory.repositories.length, repository.name); - return row; + return { ...row, repositoryId: repository.repositoryId }; }, ); diff --git a/tools/rollout/classify.mjs b/tools/rollout/classify.mjs index f075e1f..5ede3b4 100644 --- a/tools/rollout/classify.mjs +++ b/tools/rollout/classify.mjs @@ -1,5 +1,6 @@ import { parseYamlSource as parse } from "./yaml.mjs"; import { shellCommands, shellTokens } from "./commands.mjs"; +import { fingerprint } from "./fingerprint.mjs"; import { ACTION_REPOSITORY, APPROVED_RELEASE_SHA } from "./constants.mjs"; import { @@ -691,6 +692,7 @@ function classifyUsesStep(step, context) { const actionText = context.localActions?.get(`${prefix}action.yml`) ?? context.localActions?.get(`${prefix}action.yaml`); + context.reviewSources?.set(prefix, actionText ?? null); if (actionText === undefined) { return [ { @@ -1710,12 +1712,14 @@ export function classifyWorkflow(text, context) { const jobEntries = Object.entries(workflow.jobs ?? {}).sort(([a], [b]) => a < b ? -1 : a > b ? 1 : 0, ); - const jobs = jobEntries.map(([jobName, job]) => - classifyJob( + const jobs = jobEntries.map(([jobName, job]) => { + const reviewSources = new Map(); + const result = classifyJob( jobName, job, { ...context, + reviewSources, exclusionRootDirectory: [undefined, ".", "./"].includes( workflow.defaults?.run?.["working-directory"], ), @@ -1726,8 +1730,59 @@ export function classifyWorkflow(text, context) { defaultsUncertain(workflow.defaults), }, triggers, - ), - ); + ); + const upstream = Object.create(null); + const pending = [jobName]; + const visited = new Set(); + while (pending.length) { + const name = pending.pop(); + if (visited.has(name)) continue; + visited.add(name); + const parent = Object.hasOwn(workflow.jobs, name) + ? workflow.jobs[name] + : undefined; + if (name !== jobName) upstream[name] = parent ?? null; + const needs = parent?.needs; + pending.push( + ...(typeof needs === "string" + ? [needs] + : Array.isArray(needs) + ? needs.filter((item) => typeof item === "string") + : []), + ); + } + return { + ...result, + reviewDependencies: [...visited] + .filter((name) => name !== jobName) + .sort(), + reviewFingerprint: fingerprint({ + version: 1, + job, + upstream, + workflow: { + on: workflow.on, + env: workflow.env, + defaults: workflow.defaults, + }, + visibility: context.visibility, + defaultBranch: context.defaultBranch, + configuration: context.reviewConfiguration, + packageManager: context.packageManager, + localSfwRelease: context.localSfwRelease, + approvedRelease: APPROVED_RELEASE_SHA, + sources: Object.fromEntries( + [...reviewSources].map(([path, text]) => { + try { + return [path, parse(text)]; + } catch { + return [path, text]; + } + }), + ), + }), + }; + }); return { jobs, path: context.path, triggers }; } diff --git a/tools/rollout/fingerprint.mjs b/tools/rollout/fingerprint.mjs new file mode 100644 index 0000000..da0bcb8 --- /dev/null +++ b/tools/rollout/fingerprint.mjs @@ -0,0 +1,29 @@ +import { createHash } from "node:crypto"; +import { parseYamlSource } from "./yaml.mjs"; + +// Sort object keys, not arrays: command/step order is semantically significant. +export function canonicalJson(value) { + return ( + JSON.stringify(value, (_key, item) => + item && typeof item === "object" && !Array.isArray(item) + ? Object.fromEntries( + Object.entries(item).sort(([a], [b]) => + a < b ? -1 : a > b ? 1 : 0, + ), + ) + : item, + ) ?? "null" + ); +} + +export function fingerprint(value) { + return createHash("sha256").update(canonicalJson(value)).digest("hex"); +} + +export function fingerprintYaml(text) { + try { + return fingerprint(parseYamlSource(text)); + } catch { + return fingerprint(text); + } +} diff --git a/tools/rollout/integration.mjs b/tools/rollout/integration.mjs index 0084302..4152832 100644 --- a/tools/rollout/integration.mjs +++ b/tools/rollout/integration.mjs @@ -1,4 +1,5 @@ import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +import { fingerprint } from "./fingerprint.mjs"; // Configuration/executable/startup controls, not application credentials. const RELEVANT_ENV = @@ -438,6 +439,11 @@ export function resolveLocalWorkflowCalls(workflows) { if (!callee) return job; return { ...job, + reviewFingerprint: fingerprint([ + job.reviewFingerprint, + callee.reviewFingerprint, + callee.jobs.map((child) => child.reviewFingerprint), + ]), integration: { ...job.integration, disposition: integrationDisposition([callee]), @@ -447,6 +453,25 @@ export function resolveLocalWorkflowCalls(workflows) { }; }), }; + const sourceByJob = new Map( + result.jobs.map((job) => [job.job, job.reviewFingerprint]), + ); + result.jobs = result.jobs.map((job) => + job.reviewDependencies?.length + ? { + ...job, + reviewFingerprint: fingerprint([ + job.reviewFingerprint, + Object.fromEntries( + job.reviewDependencies.map((name) => [ + name, + sourceByJob.get(name) ?? null, + ]), + ), + ]), + } + : job, + ); resolved.set(path, result); return result; } diff --git a/tools/rollout/inventory.mjs b/tools/rollout/inventory.mjs index 12fa778..e9a202d 100644 --- a/tools/rollout/inventory.mjs +++ b/tools/rollout/inventory.mjs @@ -101,6 +101,7 @@ export function reconcileRepositoryInventories( ); return { defaultBranch: repository.default_branch, + ...(repository.id === undefined ? {} : { repositoryId: repository.id }), name, visibility: normalizedVisibility(repository, "REST"), }; diff --git a/tools/rollout/review-cli.mjs b/tools/rollout/review-cli.mjs new file mode 100644 index 0000000..af084a5 --- /dev/null +++ b/tools/rollout/review-cli.mjs @@ -0,0 +1,179 @@ +#!/usr/bin/env node +import { readFile, lstat, mkdir, open, realpath, rm } from "node:fs/promises"; +import { basename, dirname, join, resolve } from "node:path"; +import { parseArgs } from "node:util"; +import { pathToFileURL } from "node:url"; +import { GitHubClient } from "./github.mjs"; +import { runAudit, writeReportAtomically } from "./audit.mjs"; +import { canonicalJson } from "./fingerprint.mjs"; +import { + advanceReview, + newReviewState, + recordDecision, + reviewView, + validateState, +} from "./review.mjs"; + +async function load(path) { + const stat = await lstat(path); + if (!stat.isFile() || stat.isSymbolicLink()) + throw new Error("The ledger must be a regular file"); + return validateState(JSON.parse(await readFile(path, "utf8"))); +} + +// One writer across run/record/forget/init. A crashed writer leaves an explicit +// lock error, never an empty ledger or a lost acknowledgement. No TTL expiry. +export async function withLedgerLock(path, action) { + const lock = `${path}.lock`; + const handle = await open(lock, "wx", 0o600); + try { + await handle.writeFile(`${process.pid}\n`); + return await action(); + } finally { + await handle.close(); + await rm(lock); + } +} + +export async function reviewMain(argv, options = {}) { + const { values, positionals } = parseArgs({ + args: argv, + allowPositionals: true, + strict: true, + options: Object.fromEntries( + [ + "state", + "report", + "case", + "repository", + "kind", + "reason", + "evidence", + "by", + "expected", + ].map((name) => [name, { type: "string" }]), + ), + }); + const [command] = positionals; + const allowed = { + init: ["state"], + run: ["state", "report"], + show: ["state"], + record: [ + "state", + "case", + "repository", + "kind", + "reason", + "evidence", + "by", + "expected", + ], + forget: ["state", "case", "expected"], + }; + if ( + positionals.length !== 1 || + !allowed[command] || + !values.state || + Object.keys(values).some((key) => !allowed[command].includes(key)) + ) + throw new Error( + "usage: review --state [options]", + ); + const statePath = resolve(values.state); + const reportPath = resolve(values.report ?? "reports/review-audit.json"); + if ( + command === "run" && + [statePath, `${statePath}.lock`].includes(reportPath) + ) + throw new Error("Audit report and ledger/lock paths must differ"); + if (command === "init") + await mkdir(dirname(statePath), { recursive: true, mode: 0o700 }); + const output = options.output ?? process.stdout; + if (command === "show") { + const view = reviewView(await load(statePath)); + output.write(`${canonicalJson(view)}\n`); + return view; + } + return withLedgerLock(statePath, async () => { + let state; + if (command === "init") { + try { + await lstat(statePath); + throw new Error("Ledger already exists; init never replaces decisions"); + } catch (error) { + if (error.code !== "ENOENT") throw error; + } + state = newReviewState(); + } else { + state = await load(statePath); // Missing/corrupt state is an error, never []/auto-init. + if (command === "run") { + await mkdir(dirname(reportPath), { recursive: true, mode: 0o700 }); + const target = join( + await realpath(dirname(reportPath)), + basename(reportPath), + ); + const ledger = join( + await realpath(dirname(statePath)), + basename(statePath), + ); + if ([ledger, `${ledger}.lock`].includes(target)) + throw new Error("Audit report and ledger/lock paths must differ"); + const report = await runAudit(options.client ?? new GitHubClient(), { + progress: (done, total) => { + if (done % 25 === 0 || done === total) + (options.progress ?? process.stderr).write( + `audited ${done}/${total} repositories\n`, + ); + }, + }); + await writeReportAtomically(reportPath, report); + state = advanceReview(state, report); // Partial results never replace last-good ledger. + } else if (command === "record") { + state = recordDecision(state, { + id: values.case, + repository: values.repository, + kind: values.kind, + reason: values.reason, + evidence: values.evidence, + recordedBy: values.by, + expected: values.expected, + }); + } else { + if ( + reviewView(state).snapshot !== values.expected || + !Object.hasOwn(state.decisions, values.case ?? "") + ) + throw new Error( + "Current snapshot and an existing exact decision ID are required to forget", + ); + delete state.decisions[values.case]; + } + } + await writeReportAtomically(statePath, JSON.parse(canonicalJson(state))); + const view = reviewView(state); + output.write(`${canonicalJson(view)}\n`); + return view; + }); +} + +if ( + process.argv[1] && + import.meta.url === pathToFileURL(process.argv[1]).href +) { + reviewMain(process.argv.slice(2)).catch((error) => { + // No source/API body in logs, and never a success-shaped empty finding list. + console.error( + JSON.stringify({ + status: "failed", + error: + error.code === "EEXIST" + ? "Ledger locked; verify no writer is active before removing its .lock file." + : error.code === "ERR_ASSERTION" + ? error.message + : "Review failed; check arguments, ledger, permissions, API availability, and private audit scanStatus. Existing ledger was not reset.", + }), + ); + process.exitCode = 1; + }); +} diff --git a/tools/rollout/review.mjs b/tools/rollout/review.mjs new file mode 100644 index 0000000..62cf1c1 --- /dev/null +++ b/tools/rollout/review.mjs @@ -0,0 +1,376 @@ +import assert from "node:assert/strict"; +import { fingerprint } from "./fingerprint.mjs"; +import { ORGANIZATION } from "./constants.mjs"; + +const HASH = /^[0-9a-f]{64}$/; +const statuses = new Set(["needs-sfw", "needs-review"]); +const kinds = new Set(["known-review", "tracked-gap", "exception"]); +const mapping = (value) => + value && typeof value === "object" && !Array.isArray(value); +const sorted = (rows) => + [...rows].sort((a, b) => { + const left = `${a.repository}/${a.path ?? ""}/${a.job ?? ""}/${a.id ?? ""}`; + const right = `${b.repository}/${b.path ?? ""}/${b.job ?? ""}/${b.id ?? ""}`; + return left < right ? -1 : left > right ? 1 : 0; + }); + +export function newReviewState() { + return { + schemaVersion: 1, + organization: ORGANIZATION, + decisions: {}, + findings: [], + unobserved: [], + }; +} + +function validRepository(value) { + return ( + Number.isSafeInteger(value.repositoryId) && + value.repositoryId > 0 && + typeof value.repository === "string" && + /^workos\/[\w.-]+$/.test(value.repository) + ); +} + +export function validateState(state) { + assert( + mapping(state) && + state.schemaVersion === 1 && + state.organization === ORGANIZATION, + "Unsupported review ledger", + ); + assert( + Object.keys(state).every((key) => + [ + "schemaVersion", + "organization", + "decisions", + "findings", + "unobserved", + ].includes(key), + ), + "Unknown ledger fields", + ); + assert( + mapping(state.decisions) && + Array.isArray(state.findings) && + Array.isArray(state.unobserved), + "Malformed review ledger", + ); + assert( + new Set(state.findings.map((item) => item.id)).size === + state.findings.length, + "Duplicate review finding", + ); + for (const item of state.findings) { + assert( + validRepository(item) && + HASH.test(item.id) && + HASH.test(item.fingerprint) && + statuses.has(item.status), + "Malformed review finding", + ); + assert( + ["job", "workflow", "exclusion"].includes(item.kind) && + typeof item.path === "string" && + (item.job === null || typeof item.job === "string"), + "Malformed finding scope", + ); + assert( + item.id === + fingerprint([item.repositoryId, item.kind, item.path, item.job]), + "Finding identity does not match its scope", + ); + assert( + Array.isArray(item.reasons) && + item.reasons.every((reason) => typeof reason === "string"), + "Malformed finding reasons", + ); + } + for (const [id, decision] of Object.entries(state.decisions)) { + assert( + HASH.test(id) && + validRepository(decision) && + HASH.test(decision.fingerprint) && + statuses.has(decision.status) && + kinds.has(decision.kind), + "Malformed review decision", + ); + assert( + mapping(decision.scope) && + ["job", "workflow", "exclusion"].includes(decision.scope.kind) && + typeof decision.scope.path === "string" && + (decision.scope.job === null || typeof decision.scope.job === "string"), + "Missing exact decision scope", + ); + assert( + id === + fingerprint([ + decision.repositoryId, + decision.scope.kind, + decision.scope.path, + decision.scope.job, + ]), + "Decision identity does not match its scope", + ); + assert( + decision.kind === "exception" || + decision.status === + (decision.kind === "known-review" ? "needs-review" : "needs-sfw"), + "Decision kind conflicts with classification", + ); + assert(typeof decision.active === "boolean", "Missing decision validity"); + validateRationale(decision); + } + assert( + state.unobserved.every(validRepository), + "Malformed visibility history", + ); + return state; +} + +function validateRationale({ reason, evidence, recordedBy }) { + assert( + typeof reason === "string" && + reason.trim().length >= 10 && + reason.length <= 4000, + "A specific reason is required", + ); + assert( + typeof recordedBy === "string" && + recordedBy.trim() && + recordedBy.length <= 200, + "Recorded-by identity is required", + ); + const url = new URL(evidence); + assert( + url.protocol === "https:" && + !url.username && + !url.password && + evidence.length <= 2048, + "Evidence must be a credential-free HTTPS URL", + ); +} + +function collect(report) { + assert( + report.schemaVersion === 3 && + report.organization === ORGANIZATION && + report.scanStatus === "complete" && + report.scanErrors === 0, + "Incomplete audit: ledger and last-good review must not advance", + ); + assert( + Array.isArray(report.repositories) && + report.repositories.length === report.inventory.activeCount, + "Incomplete repository inventory", + ); + assert( + Object.values(report.inventory.differences).every( + (values) => Array.isArray(values) && values.length === 0, + ), + "Inventory sources disagree", + ); + const seen = new Map(); + const facts = new Map(); + const findings = []; + const add = (repo, kind, path, job, status, source, reasons) => { + assert( + HASH.test(source), + "Audit lacks review fingerprints; run the current detector", + ); + const item = { + id: fingerprint([repo.repositoryId, kind, path, job]), + repositoryId: repo.repositoryId, + repository: `${ORGANIZATION}/${repo.name}`, + kind, + path, + job, + status, + fingerprint: source, + reasons: [...new Set(reasons)].sort(), + }; + assert(!facts.has(item.id), "Duplicate finding scope"); + facts.set(item.id, item); + if (statuses.has(status)) findings.push(item); + }; + for (const repo of report.repositories) { + assert( + validRepository({ + repositoryId: repo.repositoryId, + repository: `${ORGANIZATION}/${repo.name}`, + }) && !seen.has(repo.repositoryId), + "Missing or duplicate immutable repository ID", + ); + assert(repo.disposition !== "audit-error", "Partial repository scan"); + seen.set(repo.repositoryId, repo.name); + for (const workflow of repo.workflows) { + if (workflow.parseError !== undefined) + add( + repo, + "workflow", + workflow.path, + null, + "needs-review", + workflow.reviewFingerprint, + ["workflow-source-unparsed"], + ); + for (const job of workflow.jobs) { + const integration = job.integration; + assert( + integration && typeof integration.disposition === "string", + "Missing job classification", + ); + add( + repo, + "job", + workflow.path, + job.job, + integration.disposition, + job.reviewFingerprint, + [ + ...(integration.downloads ?? []) + .filter((item) => ["gap", "unresolved"].includes(item.status)) + .map((item) => item.reason), + ...(integration.additionalJsPaths ?? []) + .filter((item) => item.status === "unresolved") + .map((item) => item.reason), + ...job.operations + .filter((item) => item.sourceError) + .map((item) => item.sourceError), + ...(job.operations.some((item) => item.kind === "reusable-call") + ? ["reusable-workflow-context"] + : []), + ], + ); + } + } + for (const exclusion of repo.exclusions ?? []) { + add( + repo, + "exclusion", + exclusion.id, + null, + exclusion.status === "stale" + ? "needs-review" + : "integrated-with-exclusions", + exclusion.reviewFingerprint, + ["approved-exclusion-drift"], + ); + } + } + return { seen, facts, findings: sorted(findings) }; +} + +// Acknowledgements bind to source inputs, never a repository-wide waiver or a +// last-seen timestamp. Missing visibility is not evidence of resolution. +export function advanceReview(previous, report) { + validateState(previous); + const { seen, facts, findings } = collect(report); + const state = structuredClone(previous); + for (const [id, decision] of Object.entries(state.decisions)) { + if (!seen.has(decision.repositoryId)) continue; + const fact = facts.get(id); + if ( + !fact || + fact.fingerprint !== decision.fingerprint || + (statuses.has(fact.status) && fact.status !== decision.status) + ) + decision.active = false; + decision.repository = `${ORGANIZATION}/${seen.get(decision.repositoryId)}`; + } + const prior = [ + ...previous.findings, + ...Object.values(previous.decisions), + ...previous.unobserved, + ]; + state.unobserved = sorted([ + ...new Map( + prior + .filter((item) => !seen.has(item.repositoryId)) + .map((item) => [ + item.repositoryId, + { repositoryId: item.repositoryId, repository: item.repository }, + ]), + ).values(), + ]); + state.findings = findings; + return validateState(state); +} + +export function reviewView(state) { + validateState(state); + const pending = []; + const known = []; + for (const item of sorted(state.findings)) { + const decision = state.decisions[item.id]; + if ( + decision?.active && + decision.fingerprint === item.fingerprint && + decision.status === item.status + ) { + known.push({ + ...item, + decision: { + kind: decision.kind, + reason: decision.reason, + evidence: decision.evidence, + recordedBy: decision.recordedBy, + }, + }); + } else pending.push(item); + } + return { + schemaVersion: 1, + snapshot: fingerprint(sorted(state.findings)), + needsProtection: pending.filter((item) => item.status === "needs-sfw"), + needsReview: pending.filter((item) => item.status === "needs-review"), + known, + unobserved: sorted(state.unobserved), + }; +} + +export function recordDecision( + previous, + { expected, id, repository, kind, reason, evidence, recordedBy }, +) { + validateState(previous); + assert( + HASH.test(expected) && reviewView(previous).snapshot === expected, + "Review snapshot changed; inspect it before recording a decision", + ); + assert( + Boolean(id) !== Boolean(repository) && kinds.has(kind), + "Select one exact case or current repository findings and a valid decision kind", + ); + validateRationale({ reason, evidence, recordedBy }); + const selected = previous.findings.filter((item) => + id ? item.id === id : item.repository === repository, + ); + assert(selected.length > 0, "No current finding matches this selection"); + assert( + selected.every( + (item) => + kind === "exception" || + item.status === + (kind === "known-review" ? "needs-review" : "needs-sfw"), + ), + "Use known-review for uncertainty, tracked-gap for unfinished protection, or an explicitly justified exception", + ); + const state = structuredClone(previous); + for (const item of selected) + state.decisions[item.id] = { + repositoryId: item.repositoryId, + repository: item.repository, + fingerprint: item.fingerprint, + scope: { kind: item.kind, path: item.path, job: item.job }, + status: item.status, + kind, + reason, + evidence, + recordedBy, + active: true, + }; + return validateState(state); +} diff --git a/tools/rollout/review.test.mjs b/tools/rollout/review.test.mjs new file mode 100644 index 0000000..7692e1f --- /dev/null +++ b/tools/rollout/review.test.mjs @@ -0,0 +1,476 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { + mkdtemp, + readFile, + rm, + stat, + writeFile, + symlink, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { stringify } from "yaml"; +import { runAudit } from "./audit.mjs"; +import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +import { canonicalJson, fingerprint } from "./fingerprint.mjs"; +import { + advanceReview, + newReviewState, + recordDecision, + reviewView, +} from "./review.mjs"; +import { reviewMain, withLedgerLock } from "./review-cli.mjs"; + +const setup = { + uses: `workos/setup-socket-firewall@${APPROVED_RELEASE_SHA}`, + with: { token: "${{ secrets.SOCKET_FIREWALL_TOKEN }}" }, +}; +const job = { + "runs-on": "ubuntu-latest", + steps: [{ uses: "actions/checkout@v4" }, { run: "npm ci" }], +}; +const workflow = (jobs = { install: job }) => stringify({ on: "push", jobs }); +function client({ + files = { ".github/workflows/ci.yml": workflow() }, + head = "a".repeat(40), + id = 42, + name = "demo", + partial = false, + visible = true, + reverse = false, +} = {}) { + return { + async listRestRepositories() { + return visible + ? [ + { + id, + name, + default_branch: "main", + visibility: "private", + archived: false, + }, + ] + : []; + }, + async listGraphqlRepositories() { + return visible + ? [{ name, visibility: "PRIVATE", isArchived: false }] + : []; + }, + async getRef() { + return { object: { sha: head } }; + }, + async getTree() { + const tree = Object.entries(files).map(([path, text]) => ({ + path, + mode: "100644", + type: "blob", + sha: fingerprint(text).slice(0, 40), + })); + return { truncated: partial, tree: reverse ? tree.reverse() : tree }; + }, + async getText(_repo, path, ref) { + assert.equal(ref, head); + assert.ok(Object.hasOwn(files, path)); + return files[path]; + }, + }; +} +const scan = (options) => runAudit(client(options)); +const reason = { + reason: "Tracked fixture gap with a separate remediation PR.", + evidence: "https://github.com/workos/example/pull/1", + recordedBy: "test reviewer", +}; +function acknowledge(state, extra = {}) { + return recordDecision(state, { + expected: reviewView(state).snapshot, + id: state.findings[0].id, + kind: "tracked-gap", + ...reason, + ...extra, + }); +} + +test("weekly findings and decisions are stable across order, timestamps, unrelated commits, YAML comments and other jobs", async () => { + const first = advanceReview(newReviewState(), await scan()); + assert.equal(reviewView(first).needsProtection.length, 1); + const known = acknowledge(first); + const exception = acknowledge(first, { kind: "exception" }); + assert.equal( + reviewView(advanceReview(exception, await scan())).known[0].decision.kind, + "exception", + ); + assert.equal(reviewView(exception).known[0].status, "needs-sfw"); + const second = await scan({ + head: "b".repeat(40), + reverse: true, + files: { + "README.md": "unrelated documentation", + ".github/workflows/ci.yml": + "# a YAML comment\n" + + workflow({ + unrelated: { steps: [{ run: "echo okay" }] }, + install: job, + }), + }, + }); + second.generatedAt = "another week"; + const repeated = advanceReview(known, second); + assert.equal(canonicalJson(repeated), canonicalJson(known)); + assert.equal( + canonicalJson(reviewView(repeated)), + canonicalJson(reviewView(known)), + ); + assert.equal(reviewView(repeated).known[0].status, "needs-sfw"); // Not falsely relabelled integrated. +}); + +test("new jobs, new repos and changed existing installs cannot inherit an acknowledgement", async () => { + const known = acknowledge(advanceReview(newReviewState(), await scan())); + const added = advanceReview( + known, + await scan({ + files: { + ".github/workflows/ci.yml": workflow({ install: job, another: job }), + }, + }), + ); + assert.equal(reviewView(added).known.length, 1); + assert.equal(reviewView(added).needsProtection.length, 1); + const changed = advanceReview( + known, + await scan({ + files: { + ".github/workflows/ci.yml": workflow({ + install: { + ...job, + steps: [...job.steps, { run: "npm install another" }], + }, + }), + }, + }), + ); + assert.equal(reviewView(changed).needsProtection.length, 1); + assert.equal(Object.values(changed.decisions)[0].active, false); + assert.equal( + reviewView(advanceReview(changed, await scan())).needsProtection.length, + 1, + "reverting inputs does not silently reactivate a stale decision", + ); + const recreated = advanceReview(known, await scan({ id: 43 })); + assert.equal(reviewView(recreated).needsProtection.length, 1); + assert.equal(recreated.unobserved[0].repositoryId, 42); + const renamed = advanceReview(known, await scan({ name: "renamed" })); + assert.equal(reviewView(renamed).needsProtection.length, 0); + assert.equal(reviewView(renamed).known[0].repository, "workos/renamed"); +}); + +test("resolution followed by regression reopens; absent visibility does not resolve, expire or discard decisions", async () => { + const known = acknowledge(advanceReview(newReviewState(), await scan())); + const protectedReport = await scan({ + files: { + ".github/workflows/ci.yml": workflow({ + install: { ...job, steps: [setup, ...job.steps] }, + }), + }, + }); + const resolved = advanceReview(known, protectedReport); + assert.equal(resolved.findings.length, 0); + assert.equal( + reviewView(advanceReview(resolved, await scan())).needsProtection.length, + 1, + ); + const missing = advanceReview(known, await scan({ visible: false })); + assert.equal(missing.unobserved.length, 1); + assert.equal(Object.values(missing.decisions)[0].active, true); + assert.deepEqual( + advanceReview(missing, await scan({ visible: false })), + missing, + ); + assert.deepEqual(advanceReview(missing, await scan()), known); +}); + +test("only referenced helpers, configuration and upstream producers invalidate their reviewed consumers", async () => { + const files = { + ".github/workflows/ci.yml": workflow({ + producer: { steps: [{ run: "echo value=one" }] }, + install: { + ...job, + needs: "producer", + steps: [job.steps[0], { uses: "./.github/actions/install" }], + }, + }), + ".github/actions/install/action.yml": stringify({ + runs: { using: "composite", steps: [{ shell: "bash", run: "npm ci" }] }, + }), + ".github/actions/unused/action.yml": "unused", + ".npmrc": "registry=https://registry.npmjs.org/", + }; + const known = acknowledge( + advanceReview(newReviewState(), await scan({ files })), + ); + assert.deepEqual( + advanceReview( + known, + await scan({ + files: { ...files, ".github/actions/unused/action.yml": "changed" }, + reverse: true, + }), + ), + known, + ); + for (const [path, value] of [ + [ + ".github/actions/install/action.yml", + files[".github/actions/install/action.yml"].replace( + "npm ci", + "npm install", + ), + ], + [".npmrc", "registry=https://other.invalid/"], + [ + ".github/workflows/ci.yml", + files[".github/workflows/ci.yml"].replace("value=one", "value=two"), + ], + ]) + assert.equal( + Object.values( + advanceReview(known, await scan({ files: { ...files, [path]: value } })) + .decisions, + )[0].active, + false, + path, + ); +}); + +test("review uncertainty is separately acknowledged, not a blanket exception for a later proven gap", async () => { + const files = { + ".github/workflows/ci.yml": workflow({ + install: { ...job, steps: [{ uses: "./missing" }] }, + }), + }; + const initial = advanceReview(newReviewState(), await scan({ files })); + assert.equal(reviewView(initial).needsReview.length, 1); + assert.throws(() => acknowledge(initial), /known-review/); + const known = acknowledge(initial, { kind: "known-review" }); + assert.equal( + reviewView(advanceReview(known, await scan())).needsProtection.length, + 1, + ); + assert.throws( + () => + acknowledge(initial, { kind: "known-review", expected: "0".repeat(64) }), + /snapshot changed/, + ); + assert.throws( + () => acknowledge(initial, { kind: "known-review", reason: "" }), + /reason/, + ); + assert.throws( + () => + acknowledge(initial, { + kind: "known-review", + evidence: "https://user:secret@example.com", + }), + /credential-free/, + ); + assert.throws( + () => acknowledge(initial, { kind: "known-review", id: "*" }), + /No current finding/, + ); +}); + +test("partial reports, inventory disagreements, missing IDs and legacy reports cannot reset good state", async () => { + const state = acknowledge(advanceReview(newReviewState(), await scan())); + const bytes = canonicalJson(state); + assert.throws( + () => + advanceReview(state, { + schemaVersion: 3, + organization: "workos", + scanStatus: "partial", + scanErrors: 1, + }), + /Incomplete audit/, + ); + for (const change of [ + (r) => { + delete r.repositories[0].repositoryId; + }, + (r) => { + delete r.repositories[0].workflows[0].jobs[0].reviewFingerprint; + }, + (r) => { + r.inventory.differences.activeOnlyRest = ["demo"]; + }, + (r) => { + r.repositories.push(r.repositories[0]); + r.inventory.activeCount++; + }, + ]) { + const report = await scan(); + change(report); + assert.throws(() => advanceReview(state, report)); + } + assert.equal(canonicalJson(state), bytes); +}); + +test("upstream helper/reusable changes and stale archive inputs reopen their recorded scopes", async () => { + const files = { + ".github/workflows/ci.yml": workflow({ + producer: { uses: "./.github/workflows/producer.yml" }, + install: { ...job, needs: "producer" }, + }), + ".github/workflows/producer.yml": stringify({ + on: "workflow_call", + jobs: { + value: { + steps: [ + { + uses: "actions/checkout@v4", + with: { repository: "workos/demo" }, + }, + { uses: "./.github/actions/producer" }, + ], + }, + }, + }), + ".github/actions/producer/action.yml": stringify({ + runs: { + using: "composite", + steps: [{ shell: "bash", run: "echo value=one" }], + }, + }), + }; + const first = advanceReview(newReviewState(), await scan({ files })); + const gap = first.findings.find((item) => item.status === "needs-sfw"); + const known = acknowledge(first, { id: gap.id }); + const changed = advanceReview( + known, + await scan({ + files: { + ...files, + ".github/actions/producer/action.yml": files[ + ".github/actions/producer/action.yml" + ].replace("value=one", "value=two"), + }, + }), + ); + assert.equal(changed.decisions[gap.id].active, false); + + const archiveFiles = { + ".github/workflows/ci.yml": workflow(), + "package.json": "{}", + ".npmrc": "registry=https://registry.npmjs.org/", + "package-lock.json": JSON.stringify({ + lockfileVersion: 3, + packages: { + "": {}, + "node_modules/example": { + version: "1.0.0", + resolved: "https://example.invalid/archive.tgz", + }, + }, + }), + }; + const stale = advanceReview( + newReviewState(), + await scan({ files: archiveFiles, name: "openapi-spec" }), + ); + const archive = stale.findings.find((item) => item.kind === "exclusion"); + const recorded = acknowledge(stale, { id: archive.id, kind: "known-review" }); + assert.deepEqual( + advanceReview( + recorded, + await scan({ + files: archiveFiles, + name: "openapi-spec", + head: "b".repeat(40), + }), + ), + recorded, + ); + const drift = advanceReview( + recorded, + await scan({ + name: "openapi-spec", + files: { + ...archiveFiles, + "package-lock.json": archiveFiles["package-lock.json"].replace( + "1.0.0", + "2.0.0", + ), + }, + }), + ); + assert.equal(drift.decisions[archive.id].active, false); +}); + +test("CLI persistence survives restart; missing/corrupt state, races and partial scans fail without replacing decisions", async () => { + const dir = await mkdtemp(join(tmpdir(), "sfw-weekly-")); + const path = join(dir, "state.json"); + const report = join(dir, "audit.json"); + const options = { + client: client(), + output: { write() {} }, + progress: { write() {} }, + }; + const run = (command, args = [], extra = {}) => + reviewMain([command, "--state", path, ...args], { ...options, ...extra }); + try { + await assert.rejects(run("run", ["--report", report]), /ENOENT/); + await run("init"); + const first = await run("run", ["--report", report]); + await run("record", [ + "--expected", + first.snapshot, + "--case", + first.needsProtection[0].id, + "--kind", + "tracked-gap", + "--reason", + reason.reason, + "--evidence", + reason.evidence, + "--by", + reason.recordedBy, + ]); + const bytes = await readFile(path, "utf8"); + assert.equal((await stat(path)).mode & 0o777, 0o600); + assert.equal((await stat(report)).mode & 0o777, 0o600); + assert.equal((await run("show")).known.length, 1); + await run("run", ["--report", report], { + client: client({ head: "b".repeat(40) }), + }); + assert.equal(await readFile(path, "utf8"), bytes); + await assert.rejects(run("init"), /already exists/); + await assert.rejects(run("run", ["--report", path]), /must differ/); + const alias = join(dir, "alias"); + await symlink(dir, alias); + await assert.rejects( + run("run", ["--report", join(alias, "state.json")]), + /must differ/, + ); + await assert.rejects( + run("run", ["--report", report], { client: client({ partial: true }) }), + /Incomplete audit/, + ); + assert.equal(await readFile(path, "utf8"), bytes); + await withLedgerLock(path, async () => { + await assert.rejects(run("run", ["--report", report]), { + code: "EEXIST", + }); + }); + assert.equal(await readFile(path, "utf8"), bytes); + await writeFile(path, "{bad json"); + await assert.rejects(run("run", ["--report", report])); + assert.equal(await readFile(path, "utf8"), "{bad json"); + await rm(path); + await symlink(report, path); + await assert.rejects(run("show"), /regular file/); + } finally { + await rm(dir, { recursive: true, force: true }); + } +}); From 86b2fa875fb08e7b8697a0d40b0da5f4d1336f03 Mon Sep 17 00:00:00 2001 From: "workos-tars[bot]" <269013284+workos-tars[bot]@users.noreply.github.com> Date: Tue, 15 Sep 2026 22:08:14 +0000 Subject: [PATCH 17/19] fix(audit): retry bare EOF reads without changing review state --- tools/rollout/github.mjs | 2 +- tools/rollout/review.test.mjs | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/tools/rollout/github.mjs b/tools/rollout/github.mjs index 67bc158..547de30 100644 --- a/tools/rollout/github.mjs +++ b/tools/rollout/github.mjs @@ -84,7 +84,7 @@ export class GitHubClient { /rate.?limit/i.test(error.message)); const transientNetwork = error.status === undefined && - /timeout|timed out|connection re|unexpected EOF|temporary failure/i.test( + /timeout|timed out|connection re|(?:^|unexpected |:\s*)EOF\b|temporary failure/i.test( error.message, ); const retryable = diff --git a/tools/rollout/review.test.mjs b/tools/rollout/review.test.mjs index 7692e1f..87f535c 100644 --- a/tools/rollout/review.test.mjs +++ b/tools/rollout/review.test.mjs @@ -13,6 +13,7 @@ import { join } from "node:path"; import { stringify } from "yaml"; import { runAudit } from "./audit.mjs"; import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +import { GitHubClient, GhCommandError } from "./github.mjs"; import { canonicalJson, fingerprint } from "./fingerprint.mjs"; import { advanceReview, @@ -94,6 +95,43 @@ function acknowledge(state, extra = {}) { }); } +test("bare transport EOF is retried within the existing bound, not mistaken for changed repository state", async () => { + let calls = 0; + const delays = []; + const api = new GitHubClient({ + execute: async () => { + calls++; + if (calls === 1) + throw new GhCommandError( + 'Get "https://api.github.com/repos/owner/example": EOF', + ); + return { stdout: "{}" }; + }, + sleep: async (delay) => { + delays.push(delay); + }, + }); + assert.deepEqual(await api.api("repos/owner/example"), {}); + assert.equal(calls, 2); + assert.deepEqual(delays, [5000]); + calls = 0; + delays.length = 0; + api.execute = async () => { + calls++; + throw new GhCommandError("EOF"); + }; + await assert.rejects(api.api("repos/owner/example"), /failed/); + assert.equal(calls, 4); + assert.deepEqual(delays, [5000, 10000, 20000]); + calls = 0; + api.execute = async () => { + calls++; + throw new GhCommandError("denied", { status: 403 }); + }; + await assert.rejects(api.api("repos/owner/example"), /denied/); + assert.equal(calls, 1); +}); + test("weekly findings and decisions are stable across order, timestamps, unrelated commits, YAML comments and other jobs", async () => { const first = advanceReview(newReviewState(), await scan()); assert.equal(reviewView(first).needsProtection.length, 1); From 1554d98f80df26fd2cb448c5e7b0b4e76ee37ba4 Mon Sep 17 00:00:00 2001 From: "workos-tars[bot]" <269013284+workos-tars[bot]@users.noreply.github.com> Date: Wed, 16 Sep 2026 12:01:44 +0000 Subject: [PATCH 18/19] fix(audit): record accepted sources without version churn --- README.md | 8 +- tools/rollout/audit.mjs | 7 +- tools/rollout/exclusions.mjs | 90 ++++++++++- tools/rollout/exclusions.test.mjs | 37 ++++- tools/rollout/github.mjs | 38 ++++- tools/rollout/review.test.mjs | 66 ++++++++ tools/rollout/source-approvals.test.mjs | 191 ++++++++++++++++++++++++ 7 files changed, 414 insertions(+), 23 deletions(-) create mode 100644 tools/rollout/source-approvals.test.mjs diff --git a/README.md b/README.md index 4efe7c5..0ca7e90 100644 --- a/README.md +++ b/README.md @@ -278,11 +278,13 @@ Strict job `status` and `assuranceDisposition` retain opaque execution and secur ### Approved source exclusions -`tools/rollout/exclusions.mjs` records the approved `tree-sitter-kotlin` GitHub archive in `oagen`, `oagen-emitters` and `openapi-spec`, including its exact path, version, URL, integrity pin and approval provenance. This archive intentionally downloads outside SFW; npm-registry dependencies still use the firewall. The existing `NPM_CONFIG_REPLACE_REGISTRY_HOST=npmjs` setting is retained because rewriting GitHub archives to the npm proxy breaks installation. +`tools/rollout/exclusions.mjs` records the approved `tree-sitter-kotlin` GitHub source in `oagen`, `oagen-emitters`, `openapi-spec` and `workos`, including its exact path, version, URL, integrity pin and approval provenance. This archive intentionally downloads outside SFW; npm-registry dependencies still use the firewall. The existing `NPM_CONFIG_REPLACE_REGISTRY_HOST=npmjs` setting is retained because rewriting GitHub archives to the npm proxy breaks installation. -Each scan reads the manifest and version-3 lockfile at the same captured repository SHA. Manifest dependency drift, workspace or unrecorded override configuration, a changed pin or any additional non-npm source makes the exclusion `stale` and keeps the repository in review; an unreadable lockfile is an audit error. For step-level environment exceptions, only direct, repository-root `npm install`/`npm ci` steps with that exact setting in the recorded workflows receive the exclusion, after one unconditional default-source checkout. Alternate checkout refs/repositories/paths, nested working directories and composite-action installs are not excused. Missing setup, additional gaps, unknown execution contexts and other configuration overrides remain findings. Qualified results use `integrated-with-exclusions`, never an unqualified protection claim. Strict assurance remains unchanged. +For the npm repositories, each scan reads the manifest and version-3 lockfile at the same captured repository SHA. Manifest/lock disagreement, workspace or unsupported override configuration, a changed pin or any additional non-npm source makes the exclusion `stale` and keeps the repository in review; an unreadable lockfile is an audit error. OpenAPI's approval allows ordinary npm-registry dependency bumps and flat registry-only version overrides (including npm aliases): it is not tied to unrelated version strings. Nested override forms, Git/URL/file overrides, a changed Kotlin source/version/integrity or changed project registry settings still require review. This override policy does not extend to the other npm repositories. For step-level environment exceptions, only direct, repository-root `npm install`/`npm ci` steps with that exact setting in the recorded workflows receive the exclusion, after one unconditional default-source checkout. Alternate checkout refs/repositories/paths, nested working directories and composite-action installs are not excused. Missing setup, additional gaps, unknown execution contexts and other configuration overrides remain findings. Qualified results use `integrated-with-exclusions`, never an unqualified protection claim. Strict assurance remains unchanged. -OpenAPI's exception is recorded from its project `.npmrc`, not used to waive any step-level environment override. Its effective config lines and existing `js-yaml`/`lodash` override snapshot must match the record; changes return it to review. Unresolved checkout/action provenance and conditional execution remain findings even when the archive pin matches. +WorkOS's separate approval checks the exact Git override in Rush's `pnpm-config.json` against the pnpm v9 lockfile, including the Kotlin codeload URL, version and integrity. Ordinary registry override versions may change together in the config and lock; additional network sources or a changed approved source require review. Existing local vendored tarballs are not network exceptions. This record never waives a CI step or certifies arbitrary workspace scripts. Large source files omitted by GitHub's Contents API are read via its returned immutable Git blob (up to 10 MiB), with size, encoding and blob-SHA verification; an incomplete read remains an audit error. + +OpenAPI's exception is recorded from its project `.npmrc`, not used to waive any step-level environment override. Unresolved checkout/action provenance and conditional execution remain findings even when the archive pin matches. ### Static limits diff --git a/tools/rollout/audit.mjs b/tools/rollout/audit.mjs index f350371..421ad08 100644 --- a/tools/rollout/audit.mjs +++ b/tools/rollout/audit.mjs @@ -298,10 +298,9 @@ export async function auditRepository(client, repository) { reviewFingerprint: fingerprint({ rule: entry, inputs: Object.fromEntries( - ["package.json", entry.lockfile, ".npmrc"].map((path) => [ - path, - sourceDigests.get(path), - ]), + [entry.manifest ?? "package.json", entry.lockfile, ".npmrc"].map( + (path) => [path, sourceDigests.get(path)], + ), ), }), })); diff --git a/tools/rollout/exclusions.mjs b/tools/rollout/exclusions.mjs index 62d3ccf..26626e7 100644 --- a/tools/rollout/exclusions.mjs +++ b/tools/rollout/exclusions.mjs @@ -1,4 +1,5 @@ import { isDeepStrictEqual } from "node:util"; +import { parseYamlSource } from "./yaml.mjs"; // Approved source exclusions, not repository exemptions. A changed archive, // integrity pin, or additional non-npm source must be reviewed again. @@ -36,16 +37,91 @@ export const REGISTRY_EXCLUSIONS = [ "omit-lockfile-registry-resolved=true", "replace-registry-host=npmjs", ], - manifestOverrides: { "js-yaml": "^5.4.1", lodash: "^4.17.23" }, + registryOverrides: true, + registryOverridesApprovalRequestId: + "61c52b1b-8081-48fa-9c8d-e2462e931ef7", } : {}), })); +const workosTarball = + "https://codeload.github.com/fwcd/tree-sitter-kotlin/tar.gz/f66d2908542e93c0204c6c241f794afe4e9cd5d1"; +REGISTRY_EXCLUSIONS.push({ + ...REGISTRY_EXCLUSIONS[0], + id: "workos-tree-sitter-kotlin", + repository: "workos", + manifest: "common/config/rush/pnpm-config.json", + lockfile: "common/config/rush/pnpm-lock.yaml", + specifier: + "github:fwcd/tree-sitter-kotlin#f66d2908542e93c0204c6c241f794afe4e9cd5d1", + resolved: workosTarball, + packagePath: `tree-sitter-kotlin@${workosTarball}`, + workflows: [], + approvalRequestId: "61c52b1b-8081-48fa-9c8d-e2462e931ef7", +}); + +const mapping = (value) => + value && typeof value === "object" && !Array.isArray(value); +const registrySpec = (value) => + typeof value === "string" && + value.trim().length > 0 && + ![".", "..", "~"].includes(value) && + !/\.t(?:gz|ar(?:\.gz)?)$/i.test(value) && + /^(?:npm:(?:@[\w.-]+\/)?[\w.-]+@)?[\w.*^~|<>= +-]+$/.test(value); + +async function readWorkosSource(rule, readSource) { + // Rush JSONC: preserve quoted strings, replace comments with whitespace so + // malformed adjacent tokens cannot become valid JSON by concatenation. + const manifest = JSON.parse( + (await readSource(rule.manifest)).replace( + /"(?:\\.|[^"\\])*"|\/\*[\s\S]*?\*\/|\/\/[^\r\n]*/g, + (part) => (part.startsWith('"') ? part : " "), + ), + ); + const lock = parseYamlSource(await readSource(rule.lockfile)); + const entry = lock?.packages?.[rule.packagePath]; + const matched = + manifest?.globalOverrides?.["tree-sitter-kotlin"] === rule.specifier && + lock?.lockfileVersion === "9.0" && + isDeepStrictEqual(manifest.globalOverrides, lock.overrides) && + Object.entries(lock.overrides).every(([name, spec]) => + name === "tree-sitter-kotlin" + ? spec === rule.specifier + : registrySpec(spec), + ) && + mapping(lock.packages) && + entry?.version === rule.version && + isDeepStrictEqual(entry.resolution, { + gitHosted: true, + integrity: rule.integrity, + tarball: rule.resolved, + }) && + Object.entries(lock.packages).every(([path, item]) => { + if (path === rule.packagePath) return true; + const resolution = item?.resolution; + // Other locked sources must be registry packages or local vendored files, + // not another network exception. This does not certify workspace scripts. + return ( + mapping(resolution) && + typeof resolution.integrity === "string" && + Object.keys(resolution).every((key) => + ["integrity", "tarball"].includes(key), + ) && + (resolution.tarball === undefined || + (typeof resolution.tarball === "string" && + (resolution.tarball.startsWith("https://registry.npmjs.org/") || + /^file:(?!\/)[\w./@-]+\.tgz$/.test(resolution.tarball)))) + ); + }); + return [{ ...rule, status: matched ? "matched" : "stale" }]; +} + export async function readRegistryExclusions(repository, readSource) { const rule = REGISTRY_EXCLUSIONS.find( (entry) => entry.repository === repository, ); if (!rule) return []; + if (repository === "workos") return readWorkosSource(rule, readSource); // Read errors are audit errors, not permission to apply an exclusion. const lock = JSON.parse(await readSource(rule.lockfile)); const manifest = JSON.parse(await readSource("package.json")); @@ -66,7 +142,11 @@ export async function readRegistryExclusions(repository, readSource) { typeof manifest === "object" && !Array.isArray(manifest) && !manifest.workspaces && - isDeepStrictEqual(manifest.overrides, rule.manifestOverrides) && + (rule.registryOverrides + ? manifest.overrides === undefined || + (mapping(manifest.overrides) && + Object.values(manifest.overrides).every(registrySpec)) + : manifest.overrides === undefined) && [ "dependencies", "devDependencies", @@ -103,11 +183,7 @@ export async function readRegistryExclusions(repository, readSource) { typeof entry[key] === "object" && !Array.isArray(entry[key]) && Object.values(entry[key]).every( - (spec) => - typeof spec === "string" && - (spec === rule.resolved || - !/[:/#]/.test(spec) || - /^npm:(?:@[\w.-]+\/)?[\w.-]+@[\w.*^~|<>= +-]+$/.test(spec)), + (spec) => spec === rule.resolved || registrySpec(spec), )), ), ); diff --git a/tools/rollout/exclusions.test.mjs b/tools/rollout/exclusions.test.mjs index 919c775..150824c 100644 --- a/tools/rollout/exclusions.test.mjs +++ b/tools/rollout/exclusions.test.mjs @@ -28,11 +28,12 @@ async function scan({ failRead = false, manifest = { dependencies }, npmrc, + lockDependencies = dependencies, } = {}) { const lock = { lockfileVersion: 3, packages: { - "": { dependencies }, + "": { dependencies: lockDependencies }, "node_modules/ordinary": { version: "1.0.0", resolved: "https://registry.npmjs.org/ordinary/-/ordinary-1.0.0.tgz", @@ -156,9 +157,10 @@ test("OpenAPI records the approved archive without waiving other findings", asyn for (const changes of [ { npmrc: npmrc.replace("=npmjs", "=never") }, { npmrc: `${npmrc}@other:registry=https://example.invalid/\n` }, - { - manifest: { dependencies, overrides: { ...overrides, lodash: "^5.0.0" } }, - }, + { manifest: { dependencies, overrides: { extra: "other/archive" } } }, + { manifest: { dependencies, overrides: { extra: "file:archive.tgz" } } }, + { manifest: { dependencies, overrides: { extra: "1.tgz" } } }, + { manifest: { dependencies, overrides: { extra: { nested: "1.0.0" } } } }, { manifest: { dependencies, @@ -183,7 +185,32 @@ test("OpenAPI records the approved archive without waiving other findings", asyn (await scan({ ...options, npmrc: undefined })).disposition, "audit-error", ); - // The OpenAPI override snapshot never authorizes overrides in the other repos. + for (const nextOverrides of [ + undefined, + { ...overrides, "js-yaml": "^5.4.2" }, + { ...overrides, lodash: "^5.0.0", another: "npm:@scope/alias@^2.0.0" }, + ]) { + const nextDependencies = { ...dependencies, ordinary: "^2.0.0" }; + const updated = await scan({ + ...options, + manifest: { dependencies: nextDependencies, overrides: nextOverrides }, + lockDependencies: nextDependencies, + extraPackages: { + "node_modules/ordinary": { + version: "2.0.0", + resolved: "https://registry.npmjs.org/ordinary/-/ordinary-2.0.0.tgz", + }, + }, + }); + assert.equal(updated.exclusions[0].status, "matched"); + assert.equal(updated.disposition, "integrated-with-exclusions"); + assert.equal( + updated.workflows[0].jobs[0].reviewFingerprint, + result.workflows[0].jobs[0].reviewFingerprint, + "registry-only dependency bumps do not invalidate reviewed jobs", + ); + } + // OpenAPI's registry-only override policy does not authorize other repos. assert.equal( (await scan({ manifest: { dependencies, overrides } })).exclusions[0] .status, diff --git a/tools/rollout/github.mjs b/tools/rollout/github.mjs index 547de30..0fe63ca 100644 --- a/tools/rollout/github.mjs +++ b/tools/rollout/github.mjs @@ -1,5 +1,6 @@ import { execFile as execFileCallback } from "node:child_process"; import { promisify } from "node:util"; +import { createHash } from "node:crypto"; import { setTimeout as sleep } from "node:timers/promises"; const execFile = promisify(execFileCallback); @@ -138,10 +139,32 @@ export class GitHubClient { } async getText(repository, path, ref) { - const response = await this.api( + let response = await this.api( `repos/${repository}/contents/${path.split("/").map(encodeURIComponent).join("/")}?ref=${encodeURIComponent(ref)}`, `read ${repository}/${path} at ${ref}`, ); + let blobSha; + // Contents omits bytes above 1 MiB (the Rush lockfile is ~2 MiB). + // Resolve only the returned immutable blob; never fall back to another ref. + if ( + response?.type === "file" && + response.encoding === "none" && + response.content === "" && + Number.isSafeInteger(response.size) && + response.size > 1024 * 1024 && + response.size <= 10 * 1024 * 1024 && + typeof response.sha === "string" && + /^[0-9a-f]{40}$/.test(response.sha) + ) { + blobSha = response.sha; + const blob = await this.api( + `repos/${repository}/git/blobs/${blobSha}`, + `read ${repository}/${path} blob ${blobSha}`, + ); + if (blob?.sha !== blobSha || blob.size !== response.size) + throw new Error("Large source blob does not match file metadata"); + response = { ...blob, type: "file" }; + } const content = typeof response?.content === "string" ? response.content.replaceAll("\n", "") @@ -152,9 +175,8 @@ export class GitHubClient { content === undefined || !Number.isSafeInteger(response.size) || response.size < 0 || - !/^(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|[A-Za-z0-9+/]{3}=)?$/.test( - content, - ) + content.length % 4 !== 0 || + !/^[A-Za-z0-9+/]*={0,2}$/.test(content) ) { throw new Error( `${repository}/${path} at ${ref} is not a complete base64 GitHub file response`, @@ -169,6 +191,14 @@ export class GitHubClient { `${repository}/${path} at ${ref} has incomplete or malformed content`, ); } + if ( + blobSha && + createHash("sha1") + .update(`blob ${bytes.length}\0`) + .update(bytes) + .digest("hex") !== blobSha + ) + throw new Error("Large source content does not match its Git blob SHA"); return new TextDecoder("utf-8", { fatal: true }).decode(bytes); } diff --git a/tools/rollout/review.test.mjs b/tools/rollout/review.test.mjs index 87f535c..3483669 100644 --- a/tools/rollout/review.test.mjs +++ b/tools/rollout/review.test.mjs @@ -13,6 +13,7 @@ import { join } from "node:path"; import { stringify } from "yaml"; import { runAudit } from "./audit.mjs"; import { APPROVED_RELEASE_SHA } from "./constants.mjs"; +import { REGISTRY_EXCLUSIONS } from "./exclusions.mjs"; import { GitHubClient, GhCommandError } from "./github.mjs"; import { canonicalJson, fingerprint } from "./fingerprint.mjs"; import { @@ -446,6 +447,71 @@ test("upstream helper/reusable changes and stale archive inputs reopen their rec assert.equal(drift.decisions[archive.id].active, false); }); +test("accepted OpenAPI cases survive registry bumps but not new source exceptions or routing changes", async () => { + const rule = REGISTRY_EXCLUSIONS.find( + (item) => item.repository === "openapi-spec", + ); + const files = { + ".github/workflows/ci.yml": workflow({ + install: { + steps: [job.steps[0], setup, { run: "npm ci" }, { uses: "./missing" }], + }, + }), + ".npmrc": rule.projectNpmrc.join("\n"), + }; + const dependencies = { + ordinary: "1.0.0", + "tree-sitter-kotlin": rule.resolved, + }; + const manifest = { dependencies, overrides: { "js-yaml": "^5.4.2" } }; + const lock = { + lockfileVersion: 3, + packages: { + "": { dependencies }, + "node_modules/ordinary": { + version: "1.0.0", + resolved: "https://registry.npmjs.org/ordinary/-/ordinary-1.0.0.tgz", + }, + [rule.packagePath]: { + version: rule.version, + resolved: rule.resolved, + integrity: rule.integrity, + }, + }, + }; + const run = (config = files[".npmrc"]) => + scan({ + name: "openapi-spec", + files: { + ...files, + ".npmrc": config, + "package.json": JSON.stringify(manifest), + "package-lock.json": JSON.stringify(lock), + }, + }); + const accepted = acknowledge(advanceReview(newReviewState(), await run()), { + kind: "exception", + }); + dependencies.ordinary = "2.0.0"; + manifest.overrides["js-yaml"] = "^5.4.3"; + lock.packages["node_modules/ordinary"] = { + version: "2.0.0", + resolved: "https://registry.npmjs.org/ordinary/-/ordinary-2.0.0.tgz", + }; + const repeated = advanceReview(accepted, await run()); + assert.equal(canonicalJson(repeated), canonicalJson(accepted)); + assert.equal(reviewView(repeated).known[0].decision.kind, "exception"); + lock.packages[rule.packagePath].integrity = "sha512-unapproved"; + const changedSource = reviewView(advanceReview(repeated, await run())); + assert.equal(changedSource.needsReview.length, 1); + assert.equal(changedSource.needsReview[0].kind, "exclusion"); + assert.ok( + reviewView( + advanceReview(repeated, await run("registry=https://other.invalid/")), + ).needsReview.some((item) => item.kind === "job"), + ); +}); + test("CLI persistence survives restart; missing/corrupt state, races and partial scans fail without replacing decisions", async () => { const dir = await mkdtemp(join(tmpdir(), "sfw-weekly-")); const path = join(dir, "state.json"); diff --git a/tools/rollout/source-approvals.test.mjs b/tools/rollout/source-approvals.test.mjs new file mode 100644 index 0000000..d63b3cf --- /dev/null +++ b/tools/rollout/source-approvals.test.mjs @@ -0,0 +1,191 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import test from "node:test"; +import { stringify } from "yaml"; +import { GitHubClient } from "./github.mjs"; +import { readRegistryExclusions, REGISTRY_EXCLUSIONS } from "./exclusions.mjs"; + +const approval = "61c52b1b-8081-48fa-9c8d-e2462e931ef7"; + +test("large file reads use the metadata's immutable blob and reject incomplete or substituted content", async () => { + const bytes = Buffer.alloc(1024 * 1024 + 1, "a"); + const sha = createHash("sha1") + .update(`blob ${bytes.length}\0`) + .update(bytes) + .digest("hex"); + const metadata = { + type: "file", + encoding: "none", + content: "", + size: bytes.length, + sha, + }; + const blob = { + sha, + size: bytes.length, + encoding: "base64", + content: bytes.toString("base64"), + }; + const calls = []; + let response = blob; + const api = new GitHubClient(); + api.api = async (endpoint) => { + calls.push(endpoint); + return endpoint.includes("/contents/") ? metadata : response; + }; + assert.equal( + await api.getText("workos/example", "large.lock", "a".repeat(40)), + bytes.toString(), + ); + assert.equal(calls[1], `repos/workos/example/git/blobs/${sha}`); + for (const change of [ + { sha: "b".repeat(40) }, + { size: bytes.length - 1 }, + { content: blob.content.slice(4) }, + { content: Buffer.alloc(bytes.length, "b").toString("base64") }, + { encoding: "none" }, + ]) { + response = { ...blob, ...change }; + await assert.rejects( + api.getText("workos/example", "large.lock", "a".repeat(40)), + ); + } + metadata.size = 11 * 1024 * 1024; + calls.length = 0; + await assert.rejects( + api.getText("workos/example", "large.lock", "a".repeat(40)), + ); + assert.equal( + calls.length, + 1, + "unsupported size is rejected before fetching the blob", + ); +}); + +test("WorkOS approval binds the exact Rush Git override, pnpm source and integrity, not unrelated registry versions", async () => { + const rule = REGISTRY_EXCLUSIONS.find((item) => item.repository === "workos"); + assert.ok(rule, "the requesting employee approved the WorkOS Kotlin source"); + assert.equal(rule.approvalRequestId, approval); + assert.deepEqual( + rule.workflows, + [], + "source approval never waives a CI installation", + ); + // Independently transcribed approved source, not generated from the rule. + const specifier = + "github:fwcd/tree-sitter-kotlin#f66d2908542e93c0204c6c241f794afe4e9cd5d1"; + const tarball = + "https://codeload.github.com/fwcd/tree-sitter-kotlin/tar.gz/f66d2908542e93c0204c6c241f794afe4e9cd5d1"; + const integrity = + "sha512-7pk1Tg/gXh+6hM4E0F2rPKeLyA/bNlYyqVu6T1Md/AV/vloakbvEZG0R4CYwUfVtgFAMZRbOtA8JBzX1SFatBA=="; + assert.equal(rule.specifier, specifier); + assert.equal(rule.resolved, tarball); + assert.equal(rule.integrity, integrity); + assert.equal(rule.version, "0.4.0"); + const config = { + $schema: "https://example.invalid/schema.json", + note: "/* quoted, not a comment */", + globalOverrides: { "tree-sitter-kotlin": specifier, ordinary: "^1.0.0" }, + }; + const entry = { + version: "0.4.0", + resolution: { gitHosted: true, tarball, integrity }, + }; + const lock = { + lockfileVersion: "9.0", + overrides: { ...config.globalOverrides }, + packages: { + [rule.packagePath]: entry, + "ordinary@1.0.0": { resolution: { integrity: "sha512-fixture" } }, + "vendored@file:../../third_party/npm/example.tgz": { + resolution: { + integrity: "sha512-local", + tarball: "file:../../third_party/npm/example.tgz", + }, + }, + }, + }; + const read = async (c = config, l = lock) => { + const files = { + [rule.manifest]: `/** Rush configuration */\n// preserve URLs inside strings\n${JSON.stringify(c)}`, + [rule.lockfile]: stringify(l), + }; + const [result] = await readRegistryExclusions("workos", async (path) => { + assert.ok(Object.hasOwn(files, path), path); + return files[path]; + }); + return result.status; + }; + assert.equal(await read(), "matched"); + assert.equal( + await read( + { + ...config, + globalOverrides: { ...config.globalOverrides, ordinary: "^2.0.0" }, + }, + { ...lock, overrides: { ...lock.overrides, ordinary: "^2.0.0" } }, + ), + "matched", + ); + assert.equal( + await read({ + ...config, + globalOverrides: { "tree-sitter-kotlin": "github:other/repo#unapproved" }, + }), + "stale", + ); + assert.equal(await read(config, { ...lock, overrides: {} }), "stale"); + assert.equal( + await read(config, { ...lock, lockfileVersion: "10.0" }), + "stale", + ); + for (const change of [ + { version: "0.5.0" }, + { resolution: { ...entry.resolution, integrity: "sha512-other" } }, + { + resolution: { + ...entry.resolution, + tarball: "https://example.invalid/other.tgz", + }, + }, + { resolution: { ...entry.resolution, repo: "https://example.invalid" } }, + ]) + assert.equal( + await read(config, { + ...lock, + packages: { + ...lock.packages, + [rule.packagePath]: { ...entry, ...change }, + }, + }), + "stale", + ); + for (const resolution of [ + { + tarball: "https://example.invalid/unapproved.tgz", + integrity: "sha512-other", + }, + { + type: "git", + repo: "https://example.invalid/repo", + commit: "b".repeat(40), + }, + { tarball: "file://remote.example/archive.tgz", integrity: "sha512-other" }, + ]) + assert.equal( + await read(config, { + ...lock, + packages: { ...lock.packages, extra: { resolution } }, + }), + "stale", + ); + await assert.rejects( + readRegistryExclusions("workos", async () => { + throw new Error("unavailable"); + }), + /unavailable/, + ); + await assert.rejects( + readRegistryExclusions("workos", async () => "malformed"), + ); +}); From 7638fe402efd058777c871ceec76ca69bf19459a Mon Sep 17 00:00:00 2001 From: "workos-tars[bot]" <269013284+workos-tars[bot]@users.noreply.github.com> Date: Wed, 16 Sep 2026 12:09:16 +0000 Subject: [PATCH 19/19] fix(audit): keep accepted Rush cases stable across registry bumps --- README.md | 2 +- tools/rollout/audit.mjs | 31 +++++++++++------- tools/rollout/exclusions.mjs | 26 ++++++++++++++- tools/rollout/review.test.mjs | 62 +++++++++++++++++++++++++++++++++++ 4 files changed, 107 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 0ca7e90..780db1c 100644 --- a/README.md +++ b/README.md @@ -282,7 +282,7 @@ Strict job `status` and `assuranceDisposition` retain opaque execution and secur For the npm repositories, each scan reads the manifest and version-3 lockfile at the same captured repository SHA. Manifest/lock disagreement, workspace or unsupported override configuration, a changed pin or any additional non-npm source makes the exclusion `stale` and keeps the repository in review; an unreadable lockfile is an audit error. OpenAPI's approval allows ordinary npm-registry dependency bumps and flat registry-only version overrides (including npm aliases): it is not tied to unrelated version strings. Nested override forms, Git/URL/file overrides, a changed Kotlin source/version/integrity or changed project registry settings still require review. This override policy does not extend to the other npm repositories. For step-level environment exceptions, only direct, repository-root `npm install`/`npm ci` steps with that exact setting in the recorded workflows receive the exclusion, after one unconditional default-source checkout. Alternate checkout refs/repositories/paths, nested working directories and composite-action installs are not excused. Missing setup, additional gaps, unknown execution contexts and other configuration overrides remain findings. Qualified results use `integrated-with-exclusions`, never an unqualified protection claim. Strict assurance remains unchanged. -WorkOS's separate approval checks the exact Git override in Rush's `pnpm-config.json` against the pnpm v9 lockfile, including the Kotlin codeload URL, version and integrity. Ordinary registry override versions may change together in the config and lock; additional network sources or a changed approved source require review. Existing local vendored tarballs are not network exceptions. This record never waives a CI step or certifies arbitrary workspace scripts. Large source files omitted by GitHub's Contents API are read via its returned immutable Git blob (up to 10 MiB), with size, encoding and blob-SHA verification; an incomplete read remains an audit error. +WorkOS's separate approval checks the exact Git override in Rush's `pnpm-config.json` against the pnpm v9 lockfile, including the Kotlin codeload URL, version and integrity. Ordinary registry override versions may change together in the config and lock without invalidating accepted jobs. The fingerprint still covers override names and alias targets, the exact Git source and all other Rush settings (including lifecycle-script policy). This normalization applies only while the source approval matches; additional network sources or a changed approved source require review. Existing local vendored tarballs are not network exceptions. This record never waives a CI step or certifies arbitrary workspace scripts. Large source files omitted by GitHub's Contents API are read via its returned immutable Git blob (up to 10 MiB), with size, encoding and blob-SHA verification; an incomplete read remains an audit error. OpenAPI's exception is recorded from its project `.npmrc`, not used to waive any step-level environment override. Unresolved checkout/action provenance and conditional execution remain findings even when the archive pin matches. diff --git a/tools/rollout/audit.mjs b/tools/rollout/audit.mjs index 421ad08..33c8780 100644 --- a/tools/rollout/audit.mjs +++ b/tools/rollout/audit.mjs @@ -187,18 +187,6 @@ export async function auditRepository(client, repository) { // bounds cycles in fetching; the classifier separately bounds expansion. const localActions = new Map(); const sourceContext = { - // Configuration changes matter; unrelated commits and dependency-lock - // churn do not invalidate a reviewed workflow job. - reviewConfiguration: fingerprint( - tree.tree - .filter((entry) => - /(?:^|\/)(?:\.npmrc|\.?bunfig\.toml|\.yarnrc(?:\.yml)?|pnpm-workspace\.yaml|pnpm-config\.json)$/.test( - entry.path, - ), - ) - .map(({ path, mode, type, sha }) => ({ path, mode, type, sha })) - .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)), - ), repository: fullName, defaultBranch: repository.defaultBranch, headSha, @@ -304,6 +292,25 @@ export async function auditRepository(client, repository) { ), }), })); + // Configuration changes matter, but validated registry-only override + // version bumps in the approved Rush source do not change SFW routing. + sourceContext.reviewConfiguration = fingerprint( + tree.tree + .filter((entry) => + /(?:^|\/)(?:\.npmrc|\.?bunfig\.toml|\.yarnrc(?:\.yml)?|pnpm-workspace\.yaml|pnpm-config\.json)$/.test( + entry.path, + ), + ) + .map(({ path, mode, type, sha }) => ({ + path, + mode, + type, + sha: + exclusions.find((entry) => entry.manifest === path) + ?.configurationFingerprint ?? sha, + })) + .sort((a, b) => (a.path < b.path ? -1 : a.path > b.path ? 1 : 0)), + ); const workflows = resolveLocalWorkflowCalls( workflowPaths.map((path, index) => ({ ...classifyWorkflow(workflowTexts[index], { diff --git a/tools/rollout/exclusions.mjs b/tools/rollout/exclusions.mjs index 26626e7..9ea1c75 100644 --- a/tools/rollout/exclusions.mjs +++ b/tools/rollout/exclusions.mjs @@ -1,5 +1,6 @@ import { isDeepStrictEqual } from "node:util"; import { parseYamlSource } from "./yaml.mjs"; +import { fingerprint } from "./fingerprint.mjs"; // Approved source exclusions, not repository exemptions. A changed archive, // integrity pin, or additional non-npm source must be reviewed again. @@ -113,7 +114,30 @@ async function readWorkosSource(rule, readSource) { /^file:(?!\/)[\w./@-]+\.tgz$/.test(resolution.tarball)))) ); }); - return [{ ...rule, status: matched ? "matched" : "stale" }]; + return [ + { + ...rule, + status: matched ? "matched" : "stale", + ...(matched + ? { + // Only validated registry-version values are irrelevant to SFW routing. + // Retain override names/alias targets, the exact Git pin and every other + // Rush setting, including lifecycle-script policy and package extensions. + configurationFingerprint: fingerprint({ + ...manifest, + globalOverrides: Object.fromEntries( + Object.entries(manifest.globalOverrides).map(([name, spec]) => [ + name, + name === "tree-sitter-kotlin" + ? spec + : `${spec.startsWith("npm:") ? spec.slice(0, spec.lastIndexOf("@") + 1) : ""}`, + ]), + ), + }), + } + : {}), + }, + ]; } export async function readRegistryExclusions(repository, readSource) { diff --git a/tools/rollout/review.test.mjs b/tools/rollout/review.test.mjs index 3483669..5bacbb3 100644 --- a/tools/rollout/review.test.mjs +++ b/tools/rollout/review.test.mjs @@ -447,6 +447,68 @@ test("upstream helper/reusable changes and stale archive inputs reopen their rec assert.equal(drift.decisions[archive.id].active, false); }); +test("accepted WorkOS cases ignore registry override versions, not installation policy changes", async () => { + const rule = REGISTRY_EXCLUSIONS.find((item) => item.repository === "workos"); + const manifest = { + globalOverrides: { + "tree-sitter-kotlin": rule.specifier, + "body-parser@1": "1.20.6", + alias: "npm:one@^1.0.0", + }, + }; + const lock = { + lockfileVersion: "9.0", + overrides: manifest.globalOverrides, + packages: { + [rule.packagePath]: { + version: rule.version, + resolution: { + gitHosted: true, + integrity: rule.integrity, + tarball: rule.resolved, + }, + }, + }, + }; + const run = () => + scan({ + name: "workos", + files: { + ".github/workflows/ci.yml": workflow({ + install: { steps: [job.steps[0], { uses: "./missing" }] }, + }), + [rule.manifest]: JSON.stringify(manifest), + [rule.lockfile]: stringify(lock), + }, + }); + const accepted = acknowledge(advanceReview(newReviewState(), await run()), { + kind: "exception", + }); + manifest.globalOverrides["body-parser@1"] = "1.20.8"; + manifest.globalOverrides.alias = "npm:one@^2.0.0"; + const repeated = advanceReview(accepted, await run()); + assert.equal(canonicalJson(repeated), canonicalJson(accepted)); + manifest.globalOverrides.alias = "npm:two@^2.0.0"; + assert.equal( + reviewView(advanceReview(repeated, await run())).needsReview.length, + 1, + ); + manifest.globalOverrides.alias = "npm:one@^2.0.0"; + manifest.globalOnlyBuiltDependencies = ["new-install-script"]; + assert.equal( + reviewView(advanceReview(repeated, await run())).needsReview.length, + 1, + ); + delete manifest.globalOnlyBuiltDependencies; + manifest.globalOverrides["tree-sitter-kotlin"] = + "github:other/repo#unapproved"; + assert.ok( + reviewView(advanceReview(repeated, await run())).needsReview.some( + (item) => item.kind === "exclusion", + ), + ); +}); + test("accepted OpenAPI cases survive registry bumps but not new source exceptions or routing changes", async () => { const rule = REGISTRY_EXCLUSIONS.find( (item) => item.repository === "openapi-spec",