diff --git a/.github/workflows/secret-scanner-consumer-canaries.yml b/.github/workflows/secret-scanner-consumer-canaries.yml new file mode 100644 index 0000000..98cc828 --- /dev/null +++ b/.github/workflows/secret-scanner-consumer-canaries.yml @@ -0,0 +1,97 @@ +# SPDX-License-Identifier: MPL-2.0 +# Read-only corpus audits, NOT consumer deployment or consumer PR validation. +# Execute only pinned suite scripts; never execute code from a subject repo. +name: Secret scanner consumer canaries +on: + pull_request: + paths: + - '.github/workflows/secret-scanner-consumer-canaries.yml' + - 'config/gitleaks/consumer-canaries.json' +permissions: + contents: read +concurrency: + group: scanner-consumer-canaries-${{ github.ref }} + cancel-in-progress: true +jobs: + plan: + runs-on: ubuntu-24.04 + timeout-minutes: 5 + outputs: + subjects: ${{ steps.plan.outputs.subjects }} + suite_sha: ${{ steps.plan.outputs.suite_sha }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - name: Validate bounded immutable manifest and published suite pin + id: plan + shell: bash + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + manifest=config/gitleaks/consumer-canaries.json + jq -e ' + .schema_version == 1 and + (.suite_sha | test("^[0-9a-f]{40}$")) and + (.subjects | length >= 1 and length <= 5) and + all(.subjects[]; + (.repository | test("^hyperpolymath/[A-Za-z0-9_.-]+$")) and + (.sha | test("^[0-9a-f]{40}$"))) + ' "$manifest" >/dev/null + sha=$(jq -r .suite_sha "$manifest") + bash scripts/verify-published-scanner-pin.sh "$sha" + echo "suite_sha=$sha" >> "$GITHUB_OUTPUT" + echo "subjects=$(jq -c .subjects "$manifest")" >> "$GITHUB_OUTPUT" + audit: + needs: plan + name: Audit ${{ matrix.subject.repository }} + runs-on: ubuntu-24.04 + timeout-minutes: 15 + strategy: + fail-fast: false + max-parallel: 3 + matrix: + subject: ${{ fromJSON(needs.plan.outputs.subjects) }} + steps: + - name: Check out immutable published subject (read only) + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: ${{ matrix.subject.repository }} + ref: ${{ matrix.subject.sha }} + path: subject + fetch-depth: 0 + persist-credentials: false + - name: Check out published scanner implementation outside subject + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: hyperpolymath/cicd-suite + ref: ${{ needs.plan.outputs.suite_sha }} + path: suite + persist-credentials: false + - name: Verify both checkouts and install pinned scanner + shell: bash + env: + SUBJECT_SHA: ${{ matrix.subject.sha }} + SUITE_SHA: ${{ needs.plan.outputs.suite_sha }} + run: | + set -euo pipefail + [[ $(git -C subject rev-parse HEAD) == "$SUBJECT_SHA" ]] || exit 2 + [[ $(git -C suite rev-parse HEAD) == "$SUITE_SHA" ]] || exit 2 + bash suite/actions/secret-scan-check/install.sh "$RUNNER_TEMP/canary-gitleaks" + - name: Audit full history, tree and supplemental populations + shell: bash + env: + SUBJECT: ${{ github.workspace }}/subject + GITLEAKS_BIN: ${{ runner.temp }}/canary-gitleaks/gitleaks + ESTATE_BASELINE: ${{ github.workspace }}/suite/config/gitleaks/estate-baseline.toml + SCAN_MODE: full + SCAN_TIMEOUT_SECONDS: '180' + run: | + set -euo pipefail + # This is an offline audit of a published consumer default-branch + # snapshot, not a scan of the cicd-suite PR base/head. Clear that + # unrelated caller event context for this child process only. + # Scanner controls and all finding/infrastructure failures enforce. + env -u GITHUB_EVENT_NAME -u PR_BASE -u PR_HEAD \ + bash suite/actions/secret-scan-check/check.sh diff --git a/.github/workflows/secret-scanner-reusable.yml b/.github/workflows/secret-scanner-reusable.yml index 6981d9e..81b2ba1 100644 --- a/.github/workflows/secret-scanner-reusable.yml +++ b/.github/workflows/secret-scanner-reusable.yml @@ -12,6 +12,10 @@ permissions: contents: read jobs: gitleaks: + outputs: + subject_sha: ${{ steps.scan.outputs.subject_sha }} + rust_status: ${{ steps.scan.outputs.rust_status }} + shell_status: ${{ steps.scan.outputs.shell_status }} runs-on: ubuntu-24.04 timeout-minutes: 15 permissions: @@ -61,6 +65,7 @@ jobs: GITLEAKS_BIN: ${{ runner.temp }}/suite-gitleaks/gitleaks run: bash suite/actions/secret-scan-check/control.sh - name: Scan subject (bounded, redacted, both populations) + id: scan env: GITLEAKS_BIN: ${{ runner.temp }}/suite-gitleaks/gitleaks SUBJECT: ${{ github.workspace }}/subject @@ -80,3 +85,36 @@ jobs: BASE_SHA=$(git -C "$SUBJECT" merge-base "$PR_BASE" "$PR_HEAD") fi bash suite/actions/secret-scan-check/check.sh + + supplemental-results: + # Compatibility contexts consumed by fleet branch rules. These validate + # receipts from detectors already executed in gitleaks, not echo-only passes. + needs: gitleaks + if: ${{ always() }} + name: ${{ matrix.language }}-secrets + runs-on: ubuntu-24.04 + timeout-minutes: 2 + strategy: + fail-fast: false + matrix: + language: [rust, shell] + steps: + - name: Verify completed detector result and subject binding + shell: bash + env: + LANGUAGE: ${{ matrix.language }} + UPSTREAM_RESULT: ${{ needs.gitleaks.result }} + SUBJECT_SHA: ${{ needs.gitleaks.outputs.subject_sha }} + EXPECTED_SHA: ${{ github.event.pull_request.head.sha || github.sha }} + DETECTOR_STATUS: ${{ needs.gitleaks.outputs[format('{0}_status', matrix.language)] }} + run: | + set -euo pipefail + case "$LANGUAGE" in rust|shell) ;; *) exit 2 ;; esac + case "$UPSTREAM_RESULT" in success|failure) ;; *) echo 'No completed scan receipt'; exit 2 ;; esac + [[ "$EXPECTED_SHA" =~ ^[0-9a-f]{40}$ && "$SUBJECT_SHA" == "$EXPECTED_SHA" ]] \ + || { echo 'Missing or mismatched subject receipt'; exit 2; } + case "$DETECTOR_STATUS" in + 0) echo "$LANGUAGE detector actually completed successfully for $SUBJECT_SHA" ;; + 1) echo "$LANGUAGE detector found hardcoded credentials"; exit 1 ;; + *) echo "$LANGUAGE detector did not produce a trustworthy passing result"; exit 2 ;; + esac diff --git a/.github/workflows/secret-scanner-self-test.yml b/.github/workflows/secret-scanner-self-test.yml index bf301eb..4c9182d 100644 --- a/.github/workflows/secret-scanner-self-test.yml +++ b/.github/workflows/secret-scanner-self-test.yml @@ -26,6 +26,8 @@ jobs: yq '.' actions/secret-scan-check/action.yml >/dev/null - name: Shell/control-flow fixtures and vacuous-scanner mutant run: bash actions/secret-scan-check/test.sh + - name: Compatibility contexts require real SHA-bound detector receipts + run: bash tests/secret-scanner-receipts.sh - name: Preserve pinned legacy Rust and shell detection without leaking values run: bash tests/secret-scanner-supplements.sh - name: Install the pinned real scanner diff --git a/.github/workflows/secret-scanner.yml b/.github/workflows/secret-scanner.yml index 8f8e73c..89e96bf 100644 --- a/.github/workflows/secret-scanner.yml +++ b/.github/workflows/secret-scanner.yml @@ -16,7 +16,7 @@ concurrency: cancel-in-progress: true jobs: scan: - uses: hyperpolymath/cicd-suite/.github/workflows/secret-scanner-reusable.yml@d285b0a6f9bd85c7d678619dd6269437127dddb5 + uses: hyperpolymath/cicd-suite/.github/workflows/secret-scanner-reusable.yml@41f8a325cf5ceae0f8ce213f991c86b95c987f9b with: # Keep false until an initial full audit passes and the security owner # accepts introduced-commit PR coverage. All non-PR runs remain full. diff --git a/actions/secret-scan-check/check.sh b/actions/secret-scan-check/check.sh index 5ccfb7d..ce1545b 100644 --- a/actions/secret-scan-check/check.sh +++ b/actions/secret-scan-check/check.sh @@ -114,6 +114,9 @@ fi (cd "$scan_dir" && GITLEAKS_BIN="$scanner" bash "$script_dir/control.sh" "$config") || exit 2 version=$("$scanner" version) || exit 2 printf 'scanner=%s mode=%s head=%s history_commits=%s\n' "$version" "$mode" "$head" "$population" +if [[ -n ${GITHUB_OUTPUT:-} ]]; then + printf 'subject_sha=%s\n' "$head" >> "$GITHUB_OUTPUT" || exit 2 +fi verdict=0 for pass in tree history; do started=$SECONDS @@ -139,6 +142,9 @@ for language in rust shell; do (cd "$subject" && SHELL_IGNORE_FILE="$scan_dir/.shell-secrets-ignore" \ timeout --signal=TERM --kill-after=10s 90s bash "$script_dir/$language-supplement.sh") || rc=$? printf 'supplement=%s exit=%s\n' "$language" "$rc" + if [[ -n ${GITHUB_OUTPUT:-} ]]; then + printf '%s_status=%s\n' "$language" "$rc" >> "$GITHUB_OUTPUT" || exit 2 + fi case $rc in 0) ;; 1) [[ $verdict == 2 ]] || verdict=1 ;; diff --git a/actions/secret-scan-check/test.sh b/actions/secret-scan-check/test.sh index 97956f7..9284286 100644 --- a/actions/secret-scan-check/test.sh +++ b/actions/secret-scan-check/test.sh @@ -47,13 +47,17 @@ fixture() { commit() { git -C "$SUBJECT" add .; git -C "$SUBJECT" commit -qm control; } expect() { local expected=$1 label=$2 rc=0 - HEAD_SHA=$(git -C "$SUBJECT" rev-parse HEAD) PR_HEAD=$(git -C "$SUBJECT" rev-parse HEAD) \ + : > "$tmp/receipt" + GITHUB_OUTPUT="$tmp/receipt" HEAD_SHA=$(git -C "$SUBJECT" rev-parse HEAD) PR_HEAD=$(git -C "$SUBJECT" rev-parse HEAD) \ PR_BASE=${PR_BASE:-$BASE_SHA} bash "$root/actions/secret-scan-check/check.sh" >"$tmp/output" 2>&1 || rc=$? if [[ $rc != "$expected" ]]; then cat "$tmp/output"; echo "FAIL $label: wanted $expected got $rc"; exit 1; fi echo "ok: $label" } fixture clean SCAN_MODE=full expect 0 'clean full history and tree' +grep -q '^rust_status=0$' "$tmp/receipt" +grep -q '^shell_status=0$' "$tmp/receipt" +grep -q "^subject_sha=$(git -C "$SUBJECT" rev-parse HEAD)$" "$tmp/receipt" SCAN_MODE=pr-range expect 0 'empty introduced range still scans the tree' # Build three PR commits. Final tree and BASE..HEAD file diff are clean. printf '%s_%s\n' ghp Q7mN2aR9vB4cD6eF8gH1jK3pL5sT0uV2wX4z >"$SUBJECT/key.txt" @@ -112,12 +116,14 @@ printf 'const AUTH_SECRET: &str = "%s%s";\n' N7cR2mQ9aB4dE6fG 8hJ1kL3pS5tV0wX2 > commit SCAN_MODE=full expect 1 'Rust supplemental finding contributes to composite verdict' grep -q 'supplement=rust exit=1' "$tmp/output" +grep -q '^rust_status=1$' "$tmp/receipt" fixture shellliteral printf 'export SERVICE_TOKEN="%s%s"\n' N7cR2mQ9aB4dE6fG 8hJ1kL3pS5tV0wX2 >"$SUBJECT/run.sh" echo '*.sh' >"$SUBJECT/.shell-secrets-ignore" commit GITHUB_EVENT_NAME=pull_request SCAN_MODE=pr-range expect 1 'PR shell finding cannot be waived by a head-added ignore' grep -q 'supplement=shell exit=1' "$tmp/output" +grep -q '^shell_status=1$' "$tmp/receipt" fixture wrongbase BASE_SHA=0000000000000000000000000000000000000000 SCAN_MODE=pr-range expect 2 'missing base refuses a verdict' echo mismatch >"$SUBJECT/.gitleaks-estate.toml" diff --git a/config/gitleaks/consumer-canaries.json b/config/gitleaks/consumer-canaries.json new file mode 100644 index 0000000..92c64b7 --- /dev/null +++ b/config/gitleaks/consumer-canaries.json @@ -0,0 +1,24 @@ +{ + "schema_version": 1, + "suite_sha": "41f8a325cf5ceae0f8ce213f991c86b95c987f9b", + "subjects": [ + { + "repository": "hyperpolymath/unified-dataset-vocab", + "sha": "12acc7881e12ffb52c9146dea7f6d43fce0c13e3", + "profile": "documentation and vocabulary", + "default_branch": "main" + }, + { + "repository": "hyperpolymath/idrisiser", + "sha": "c630c024c9be7852a42a6699f2394d10af0ee71b", + "profile": "Rust and shell scaffolding", + "default_branch": "main" + }, + { + "repository": "hyperpolymath/gitbot-fleet", + "sha": "83e35bcfd04d43960d635fec5397da8dfa344268", + "profile": "larger Rust estate tooling with local Gitleaks exceptions", + "default_branch": "main" + } + ] +} diff --git a/docs/SECRET-SCANNER-ROLLOUT.adoc b/docs/SECRET-SCANNER-ROLLOUT.adoc index 8023344..7d0df73 100644 --- a/docs/SECRET-SCANNER-ROLLOUT.adoc +++ b/docs/SECRET-SCANNER-ROLLOUT.adoc @@ -94,3 +94,69 @@ SonarCloud initially rejected the PR for an HTTPS-redirection concern in the installer. Added explicit HTTPS-only initial and redirect protocols; other annotations about positional arguments were also addressed. A new analysis must validate the correction; the quality gate has not been weakened. + +== Published implementation and read-only corpus canaries (2026-10-31) + +PR #64 landed as 41f8a325cf5ceae0f8ce213f991c86b95c987f9b. Its push-to-main +Secret Scanner, Supplemental secret checks, Bounded secret scanner controls, +Code Hygiene Self-Test and Central Estate CI/CD Audit runs all passed. + +The next caller advancement selects that published commit, including the +integrated supplementary verdict, rather than an intermediate PR SHA. +The separate supplementary jobs remain temporarily as comparison checks. + +`config/gitleaks/consumer-canaries.json` pins three public default-branch +snapshots: unified-dataset-vocab, idrisiser and gitbot-fleet. The consumer +canary workflow audits their full history and tree with the exact published +suite scripts and local consumer configuration. It never runs subject code, +changes a consumer, inherits consumer secrets or uploads finding artifacts. +All detector controls, findings and infrastructure errors remain enforcing. +Each audit has the same bounded scanner passes and a 15-minute job ceiling. + +These are offline audits of immutable published consumer snapshots, not PR +scans. The child scanner process clears the unrelated cicd-suite PR event +context; no consumer PR base/head is fabricated. This does not test PR-specific +configuration trust or the consumer's workflow admission/lock policy. The +integration fixtures cover the former; actual migration PRs must test the +latter. A successful offline audit is not a consumer deployment or a merge +readiness verdict for its existing open PRs. + +Only after results are inspected should consumer callers be proposed, with +`scan / gitleaks` preserved, least-privilege permissions, existing repository +triggers retained, and consumer-specific lock policy handled explicitly. +Any failed audits need rule/path diagnosis, not wider global exclusions. +The canary repository snapshots and suite SHA are recorded independently so +results are not silently reused after either changes. + +=== Fleet compatibility gate discovered before migration + +The live gitbot-fleet main rules require scan / gitleaks, scan / rust-secrets +and scan / shell-secrets (as well as unrelated build/security checks). +Replacing its caller with a reusable emitting only Gitleaks would strand +required contexts. Do not remove those requirements to ease rollout. + +The next reusable implementation therefore publishes rust-secrets and +shell-secrets compatibility results derived from the already-executed detector +results. Receipts include the actual scanned subject SHA and each detector's +exit status. The result jobs assert the expected SHA, completed upstream state +and explicit status; missing, mismatched, skipped, cancelled or error receipts +cannot pass. A component may report success when it really passed but another +component failed; the aggregate Gitleaks verdict remains failing in that case. +No second full-history scan or duplicate subject checkout is introduced. + +The exact result-job shell is tested from parsed YAML with missing/error/ +timeout/cancelled/skipped/wrong-subject controls. Integration fixtures verify +that actual scanner execution emits passing/failing component receipts. +These are empirical assertions, not cryptographic attestations or formal proofs. + +The published 41f8a3 version used for the current corpus audits predates these +additional result jobs. Do not deploy that version to fleet: first publish the +compatibility implementation, then pin a consumer canary to its main SHA and +verify all three actual contexts and fleet's transitive lock-sync gate. +The read-only corpus audit passing does not waive those rollout requirements. + +The corpus canary workflow runs only in pull-request context. An initial +manual-dispatch trigger was removed after CodeQL flagged dynamic checkouts +in a default-branch execution context as a potential cache-poisoning surface. +There is no need to grant that extra execution surface for these PR audits; +no suppression or quality-gate exception is used to accept the alert. diff --git a/tests/secret-scanner-receipts.sh b/tests/secret-scanner-receipts.sh new file mode 100644 index 0000000..4bb2196 --- /dev/null +++ b/tests/secret-scanner-receipts.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# SPDX-License-Identifier: MPL-2.0 +# Exercise the exact compatibility-job shell parsed from the workflow YAML. +set -euo pipefail +root=$(cd "$(dirname "$0")/.." && pwd) +command -v yq >/dev/null || { echo 'mikefarah yq v4 is required' >&2; exit 2; } +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT +program=$(yq -r '.jobs["supplemental-results"].steps[0].run' "$root/.github/workflows/secret-scanner-reusable.yml") +[[ -n $program && $program != null ]] || { echo 'receipt validator missing' >&2; exit 2; } +printf '%s\n' "$program" > "$tmp/validate.sh" +sha=0123456789abcdef0123456789abcdef01234567 +assert_receipt() { + local want=$1 label=$2 detector=$3 upstream=$4 actual=${5-$sha} expected=${6-$sha} rc=0 + DETECTOR_STATUS=$detector UPSTREAM_RESULT=$upstream SUBJECT_SHA=$actual EXPECTED_SHA=$expected \ + bash "$tmp/validate.sh" >"$tmp/output" 2>&1 || rc=$? + [[ $rc == "$want" ]] || { cat "$tmp/output"; echo "FAIL $label: wanted $want got $rc"; exit 1; } + echo "ok: $LANGUAGE $label" +} +for language in rust shell; do + export LANGUAGE=$language + assert_receipt 0 'completed passing detector' 0 success + assert_receipt 0 'passing detector despite another failed scan component' 0 failure + assert_receipt 1 'finding stays red' 1 failure + assert_receipt 2 'tool error stays non-passing' 2 failure + assert_receipt 2 'timeout stays non-passing' 124 failure + assert_receipt 2 'missing receipt is not green' '' success + assert_receipt 2 'cancelled upstream is not green' 0 cancelled + assert_receipt 2 'skipped upstream is not green' 0 skipped + assert_receipt 2 'wrong subject is not green' 0 success ffffffffffffffffffffffffffffffffffffffff + assert_receipt 2 'missing expected SHA is not green' 0 success "$sha" '' +done +LANGUAGE=unknown assert_receipt 2 'unknown detector is refused' 0 success