From 286ab5866f79a87fcccc4901bfb5b98677dd7282 Mon Sep 17 00:00:00 2001 From: Jonathan Haas <15969068+haasonsaas@users.noreply.github.com> Date: Wed, 29 Apr 2026 23:56:58 -0700 Subject: [PATCH 1/3] ci: add codeql-guard to keep CodeQL out of evalops/* --- .github/workflows/codeql-guard.yml | 91 ++++++++++++++++++++++++++++++ 1 file changed, 91 insertions(+) create mode 100644 .github/workflows/codeql-guard.yml diff --git a/.github/workflows/codeql-guard.yml b/.github/workflows/codeql-guard.yml new file mode 100644 index 0000000..1e97dac --- /dev/null +++ b/.github/workflows/codeql-guard.yml @@ -0,0 +1,91 @@ +name: codeql-guard + +on: + pull_request: + paths: + - ".github/workflows/**" + - ".github/workflow-templates/**" + push: + branches: [main] + paths: + - ".github/workflows/**" + - ".github/workflow-templates/**" + schedule: + # Daily org-wide drift sweep. + - cron: "17 9 * * *" + workflow_dispatch: + +permissions: + contents: read + issues: write + +jobs: + guard-self: + name: Forbid CodeQL in evalops/.github + if: ${{ github.event_name != 'schedule' }} + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 5 + steps: + - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 + + - name: Forbid github/codeql-action references + shell: bash + run: | + set -euo pipefail + shopt -s globstar nullglob + targets=(.github/workflows .github/workflow-templates) + existing=() + for d in "${targets[@]}"; do + [ -d "$d" ] && existing+=("$d") + done + if [ "${#existing[@]}" -eq 0 ]; then + echo "no workflow directories present" + exit 0 + fi + if grep -RIn --include='*.yml' --include='*.yaml' \ + 'github/codeql-action' "${existing[@]}" 2>/dev/null; then + echo "::error::EvalOps policy forbids github/codeql-action. See SECURITY.md (Code Scanning)." + exit 1 + fi + echo "ok: no codeql-action references in evalops/.github" + + guard-org: + name: Sweep evalops/* for CodeQL workflow drift + if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }} + runs-on: blacksmith-4vcpu-ubuntu-2404 + timeout-minutes: 15 + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + steps: + - name: Search org for github/codeql-action references + shell: bash + run: | + set -euo pipefail + mapfile -t hits < <( + gh api -X GET search/code \ + -f q='org:evalops "github/codeql-action" path:.github/workflows' \ + --jq '.items[] | "\(.repository.full_name)\t\(.path)"' \ + 2>/dev/null || true + ) + if [ "${#hits[@]}" -eq 0 ]; then + echo "ok: no CodeQL workflow files found in any evalops repo" + exit 0 + fi + { + echo "## codeql-guard tripped" + echo + echo "EvalOps does not run GitHub CodeQL (see \`SECURITY.md\` and the Blacksmith" + echo "code security configuration). The following workflow files reference" + echo "\`github/codeql-action\` and need to be removed or the policy amended:" + echo + for h in "${hits[@]}"; do + repo="${h%%$'\t'*}" + path="${h##*$'\t'}" + echo "- \`${repo}\` — \`${path}\`" + done + } > /tmp/body.md + gh issue create \ + --repo evalops/.github \ + --title "codeql-guard: CodeQL workflow drift detected" \ + --body-file /tmp/body.md + exit 1 From 11a107a4630a28b88439bc75852fec68b96eb7bf Mon Sep 17 00:00:00 2001 From: Jonathan Haas <15969068+haasonsaas@users.noreply.github.com> Date: Wed, 29 Apr 2026 23:57:16 -0700 Subject: [PATCH 2/3] docs(security): document CodeQL-disabled policy and codeql-guard --- SECURITY.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/SECURITY.md b/SECURITY.md index 70b6a92..1ee3229 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -22,3 +22,24 @@ We support security patches for the latest release of each actively maintained s ## Scope This policy applies to all repositories in the [evalops](https://github.com/evalops) GitHub organization. + +## Code Scanning + +EvalOps does not enable GitHub CodeQL. Every repository is attached to the +**EvalOps Blacksmith recommended** code security configuration +(`id=245233`), which sets `code_scanning_default_setup: disabled` and is the +default for new repositories. + +Equivalent static analysis lives elsewhere: + +- `semgrep`-based custom rules in service repos (see `.semgrep/` directories + and the `semgrep-custom` workflows). +- Service-specific gates such as `architecture-review`, `contract-skew-check`, + and `migration-check` in `evalops/platform`. +- The [`codeql-guard`](.github/workflows/codeql-guard.yml) workflow in this + repo enforces the policy: it rejects PRs that introduce + `github/codeql-action` here, and it sweeps every `evalops/*` repo daily, + opening an issue if a CodeQL workflow file appears anywhere in the org. + +To request a policy change, open a PR against this file and the guard +workflow. From fae51734c0b19228b08d974b5078354549e117a4 Mon Sep 17 00:00:00 2001 From: Cursor Agent Date: Thu, 30 Apr 2026 07:02:55 +0000 Subject: [PATCH 3/3] Fix codeql guard workflow checks --- .github/workflows/codeql-guard.yml | 32 ++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 11 deletions(-) diff --git a/.github/workflows/codeql-guard.yml b/.github/workflows/codeql-guard.yml index 1e97dac..8f319f6 100644 --- a/.github/workflows/codeql-guard.yml +++ b/.github/workflows/codeql-guard.yml @@ -42,7 +42,7 @@ jobs: echo "no workflow directories present" exit 0 fi - if grep -RIn --include='*.yml' --include='*.yaml' \ + if grep -RIn --include='*.yml' --include='*.yaml' --exclude='codeql-guard.yml' \ 'github/codeql-action' "${existing[@]}" 2>/dev/null; then echo "::error::EvalOps policy forbids github/codeql-action. See SECURITY.md (Code Scanning)." exit 1 @@ -55,22 +55,27 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 15 env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ github.token }} + ORG_CODE_SEARCH_TOKEN: ${{ secrets.EVALOPS_ORG_READ_TOKEN }} steps: - name: Search org for github/codeql-action references shell: bash run: | set -euo pipefail - mapfile -t hits < <( - gh api -X GET search/code \ + if [ -z "${ORG_CODE_SEARCH_TOKEN}" ]; then + echo "::error::Set secrets.EVALOPS_ORG_READ_TOKEN to a token with org-wide code search access." + exit 1 + fi + response="$( + GH_TOKEN="${ORG_CODE_SEARCH_TOKEN}" gh api -X GET search/code \ -f q='org:evalops "github/codeql-action" path:.github/workflows' \ --jq '.items[] | "\(.repository.full_name)\t\(.path)"' \ - 2>/dev/null || true - ) - if [ "${#hits[@]}" -eq 0 ]; then + )" + if [ -z "${response}" ]; then echo "ok: no CodeQL workflow files found in any evalops repo" exit 0 fi + mapfile -t hits <<< "${response}" { echo "## codeql-guard tripped" echo @@ -84,8 +89,13 @@ jobs: echo "- \`${repo}\` — \`${path}\`" done } > /tmp/body.md - gh issue create \ - --repo evalops/.github \ - --title "codeql-guard: CodeQL workflow drift detected" \ - --body-file /tmp/body.md + title="codeql-guard: CodeQL workflow drift detected" + if issue_number="$(gh issue list --repo evalops/.github --state open --search "\"${title}\" in:title" --limit 1 --json number --jq '.[0].number // empty')" && [ -n "${issue_number}" ]; then + echo "open tracking issue already exists: #${issue_number}" + else + gh issue create \ + --repo evalops/.github \ + --title "${title}" \ + --body-file /tmp/body.md + fi exit 1