From 2241702db0f60de3e2175b6e5407b21273dc4d84 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Tue, 14 Jul 2026 19:11:39 +0000 Subject: [PATCH 1/4] add language filtering for codeql --- .github/workflows/codeql.yml | 41 ++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 0a7aa09530a8..01b7044650e4 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -27,8 +27,49 @@ on: - cron: '25 10 * * 6' jobs: + detect-changes: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: read + outputs: + actions: ${{ steps.filter.outputs.actions }} + go: ${{ steps.filter.outputs.go }} + java-kotlin: ${{ steps.filter.outputs['java-kotlin'] }} + javascript-typescript: ${{ steps.filter.outputs['javascript-typescript'] }} + python: ${{ steps.filter.outputs.python }} + rust: ${{ steps.filter.outputs.rust }} + steps: + - uses: dorny/paths-filter@v3 + id: filter + with: + filters: | + actions: + - '.github/**' + go: + - 'sdks/go/**' + - 'playground/backend/**' + - 'learning/katas/go/**' + - 'learning/tour-of-beam/backend/**' + java-kotlin: + - 'sdks/java/**' + - 'runners/**' + - 'examples/java/**' + - 'examples/kotlin/**' + javascript-typescript: + - 'sdks/typescript/**' + python: + - 'sdks/python/**' + rust: + - '**/*.rs' + analyze: name: Analyze (${{ matrix.language }}) + needs: detect-changes + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + needs.detect-changes.outputs[matrix.language] == 'true' # Runner size impacts CodeQL analysis time. To learn more, please see: # - https://gh.io/recommended-hardware-resources-for-running-codeql # - https://gh.io/supported-runners-and-hardware-resources From 5d20cac0d42cc5324eed11e9d2044872aa00d974 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Tue, 14 Jul 2026 19:16:20 +0000 Subject: [PATCH 2/4] fix workflow file --- .github/workflows/codeql.yml | 53 +++++++++++++++++++----------------- 1 file changed, 28 insertions(+), 25 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 01b7044650e4..1564809720f4 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -33,12 +33,7 @@ jobs: contents: read pull-requests: read outputs: - actions: ${{ steps.filter.outputs.actions }} - go: ${{ steps.filter.outputs.go }} - java-kotlin: ${{ steps.filter.outputs['java-kotlin'] }} - javascript-typescript: ${{ steps.filter.outputs['javascript-typescript'] }} - python: ${{ steps.filter.outputs.python }} - rust: ${{ steps.filter.outputs.rust }} + matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - uses: dorny/paths-filter@v3 id: filter @@ -62,14 +57,36 @@ jobs: - 'sdks/python/**' rust: - '**/*.rs' + - name: Generate Matrix + id: set-matrix + run: | + ALL_MATRIX='[ + {"language": "actions", "build-mode": "none"}, + {"language": "go", "build-mode": "manual"}, + {"language": "java-kotlin", "build-mode": "autobuild"}, + {"language": "javascript-typescript", "build-mode": "none"}, + {"language": "python", "build-mode": "none"}, + {"language": "rust", "build-mode": "none"} + ]' + + if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then + echo "matrix=$ALL_MATRIX" >> $GITHUB_OUTPUT + else + FILTERED=$(echo "$ALL_MATRIX" | jq -c '[.[] | select( + (.language == "actions" and "${{ steps.filter.outputs.actions }}" == "true") or + (.language == "go" and "${{ steps.filter.outputs.go }}" == "true") or + (.language == "java-kotlin" and "${{ steps.filter.outputs['java-kotlin'] }}" == "true") or + (.language == "javascript-typescript" and "${{ steps.filter.outputs['javascript-typescript'] }}" == "true") or + (.language == "python" and "${{ steps.filter.outputs.python }}" == "true") or + (.language == "rust" and "${{ steps.filter.outputs.rust }}" == "true") + )]') + echo "matrix=$FILTERED" >> $GITHUB_OUTPUT + fi analyze: name: Analyze (${{ matrix.language }}) needs: detect-changes - if: | - github.event_name == 'schedule' || - github.event_name == 'workflow_dispatch' || - needs.detect-changes.outputs[matrix.language] == 'true' + if: needs.detect-changes.outputs.matrix != '[]' # Runner size impacts CodeQL analysis time. To learn more, please see: # - https://gh.io/recommended-hardware-resources-for-running-codeql # - https://gh.io/supported-runners-and-hardware-resources @@ -90,21 +107,7 @@ jobs: strategy: fail-fast: false matrix: - include: - - language: actions - build-mode: none - # - language: c-cpp - # build-mode: autobuild - - language: go - build-mode: manual - - language: java-kotlin - build-mode: autobuild - - language: javascript-typescript - build-mode: none - - language: python - build-mode: none - - language: rust - build-mode: none + include: ${{ fromJSON(needs.detect-changes.outputs.matrix) }} # CodeQL supports the following values keywords for 'language': 'actions', 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'rust', 'swift' # Use `c-cpp` to analyze code written in C, C++ or both # Use 'java-kotlin' to analyze code written in Java, Kotlin or both From 9e21adf27b508618f63ce89598788e3758723841 Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Tue, 14 Jul 2026 19:19:32 +0000 Subject: [PATCH 3/4] remove third party dorny action --- .github/workflows/codeql.yml | 81 +++++++++++++++++++++++------------- 1 file changed, 51 insertions(+), 30 deletions(-) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index 1564809720f4..a029e67f10d3 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -35,28 +35,11 @@ jobs: outputs: matrix: ${{ steps.set-matrix.outputs.matrix }} steps: - - uses: dorny/paths-filter@v3 - id: filter + - name: Checkout repository history + uses: actions/checkout@v4 with: - filters: | - actions: - - '.github/**' - go: - - 'sdks/go/**' - - 'playground/backend/**' - - 'learning/katas/go/**' - - 'learning/tour-of-beam/backend/**' - java-kotlin: - - 'sdks/java/**' - - 'runners/**' - - 'examples/java/**' - - 'examples/kotlin/**' - javascript-typescript: - - 'sdks/typescript/**' - python: - - 'sdks/python/**' - rust: - - '**/*.rs' + fetch-depth: 0 + - name: Generate Matrix id: set-matrix run: | @@ -71,18 +54,56 @@ jobs: if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "matrix=$ALL_MATRIX" >> $GITHUB_OUTPUT + exit 0 + fi + + if [ "${{ github.event_name }}" = "pull_request" ]; then + BASE_SHA="${{ github.event.pull_request.base.sha }}" + HEAD_SHA="${{ github.event.pull_request.head.sha }}" else - FILTERED=$(echo "$ALL_MATRIX" | jq -c '[.[] | select( - (.language == "actions" and "${{ steps.filter.outputs.actions }}" == "true") or - (.language == "go" and "${{ steps.filter.outputs.go }}" == "true") or - (.language == "java-kotlin" and "${{ steps.filter.outputs['java-kotlin'] }}" == "true") or - (.language == "javascript-typescript" and "${{ steps.filter.outputs['javascript-typescript'] }}" == "true") or - (.language == "python" and "${{ steps.filter.outputs.python }}" == "true") or - (.language == "rust" and "${{ steps.filter.outputs.rust }}" == "true") - )]') - echo "matrix=$FILTERED" >> $GITHUB_OUTPUT + BASE_SHA="${{ github.event.before }}" + HEAD_SHA="${{ github.event.after }}" + fi + + if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" = "0000000000000000000000000000000000000000" ]; then + CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || echo "") + else + CHANGED_FILES=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" 2>/dev/null || echo "") fi + check_path() { + local pattern="$1" + if echo "$CHANGED_FILES" | grep -q -E "$pattern"; then + echo "true" + else + echo "false" + fi + } + + ACTIONS_CHANGED=$(check_path "^\.github/") + GO_CHANGED=$(check_path "^sdks/go/|^playground/backend/|^learning/katas/go/|^learning/tour-of-beam/backend/") + JAVA_CHANGED=$(check_path "^sdks/java/|^runners/|^examples/java/|^examples/kotlin/|\.java$") + JS_CHANGED=$(check_path "^sdks/typescript/|\.js$|\.ts$") + PYTHON_CHANGED=$(check_path "^sdks/python/|\.py$") + RUST_CHANGED=$(check_path "\.rs$") + + FILTERED=$(echo "$ALL_MATRIX" | jq -c --arg actions "$ACTIONS_CHANGED" \ + --arg go "$GO_CHANGED" \ + --arg java "$JAVA_CHANGED" \ + --arg js "$JS_CHANGED" \ + --arg python "$PYTHON_CHANGED" \ + --arg rust "$RUST_CHANGED" \ + '[.[] | select( + (.language == "actions" and $actions == "true") or + (.language == "go" and $go == "true") or + (.language == "java-kotlin" and $java == "true") or + (.language == "javascript-typescript" and $js == "true") or + (.language == "python" and $python == "true") or + (.language == "rust" and $rust == "true") + )]') + + echo "matrix=$FILTERED" >> $GITHUB_OUTPUT + analyze: name: Analyze (${{ matrix.language }}) needs: detect-changes From eaef601a22ecbd62a653540ec25af80034c18f0a Mon Sep 17 00:00:00 2001 From: Derrick Williams Date: Tue, 14 Jul 2026 19:25:33 +0000 Subject: [PATCH 4/4] add comments --- .github/workflows/codeql.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml index a029e67f10d3..7922f7b2e857 100644 --- a/.github/workflows/codeql.yml +++ b/.github/workflows/codeql.yml @@ -43,6 +43,7 @@ jobs: - name: Generate Matrix id: set-matrix run: | + # Full matrix of all CodeQL supported languages and their build modes in Apache Beam. ALL_MATRIX='[ {"language": "actions", "build-mode": "none"}, {"language": "go", "build-mode": "manual"}, @@ -52,11 +53,13 @@ jobs: {"language": "rust", "build-mode": "none"} ]' + # On periodic scheduled runs (cron) or manual dispatches, scan all languages. if [[ "${{ github.event_name }}" == "schedule" || "${{ github.event_name }}" == "workflow_dispatch" ]]; then echo "matrix=$ALL_MATRIX" >> $GITHUB_OUTPUT exit 0 fi + # Determine the commit range to compare for changed files. if [ "${{ github.event_name }}" = "pull_request" ]; then BASE_SHA="${{ github.event.pull_request.base.sha }}" HEAD_SHA="${{ github.event.pull_request.head.sha }}" @@ -65,12 +68,14 @@ jobs: HEAD_SHA="${{ github.event.after }}" fi + # Fall back to comparing against HEAD~1 if base SHA is empty or zeroed. if [ -z "$BASE_SHA" ] || [ "$BASE_SHA" = "0000000000000000000000000000000000000000" ]; then CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD 2>/dev/null || echo "") else CHANGED_FILES=$(git diff --name-only "$BASE_SHA" "$HEAD_SHA" 2>/dev/null || echo "") fi + # Helper function to check if any changed file matches a given regex pattern. check_path() { local pattern="$1" if echo "$CHANGED_FILES" | grep -q -E "$pattern"; then @@ -80,6 +85,7 @@ jobs: fi } + # Check path changes for each language/module context. ACTIONS_CHANGED=$(check_path "^\.github/") GO_CHANGED=$(check_path "^sdks/go/|^playground/backend/|^learning/katas/go/|^learning/tour-of-beam/backend/") JAVA_CHANGED=$(check_path "^sdks/java/|^runners/|^examples/java/|^examples/kotlin/|\.java$") @@ -87,6 +93,7 @@ jobs: PYTHON_CHANGED=$(check_path "^sdks/python/|\.py$") RUST_CHANGED=$(check_path "\.rs$") + # Filter ALL_MATRIX entries using jq, keeping only entries for languages with path changes. FILTERED=$(echo "$ALL_MATRIX" | jq -c --arg actions "$ACTIONS_CHANGED" \ --arg go "$GO_CHANGED" \ --arg java "$JAVA_CHANGED" \