From bc88bcb91457e493ce38385143a8ab56402a2ad0 Mon Sep 17 00:00:00 2001 From: Even Stensberg Date: Mon, 24 Aug 2026 17:01:27 +0200 Subject: [PATCH 1/3] feat: add script to check CI org wide Co-Authored-By: Claude Opus 5 --- scripts/org-workflows-on-main.sh | 119 +++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100755 scripts/org-workflows-on-main.sh diff --git a/scripts/org-workflows-on-main.sh b/scripts/org-workflows-on-main.sh new file mode 100755 index 0000000..1a83f46 --- /dev/null +++ b/scripts/org-workflows-on-main.sh @@ -0,0 +1,119 @@ +#!/bin/bash + + +# usage: ./org-workflows-on-main.sh + +ORG="webpack" + +# Colors (disabled when not writing to a terminal) +if [ -t 1 ]; then + BOLD=$'\033[1m'; DIM=$'\033[2m'; RESET=$'\033[0m' + GREEN=$'\033[32m'; RED=$'\033[31m'; YELLOW=$'\033[33m'; CYAN=$'\033[36m' +else + BOLD=""; DIM=""; RESET=""; GREEN=""; RED=""; YELLOW=""; CYAN="" +fi + +# Check if logged into GitHub CLI +gh auth status || exit 1 + +# List all repositories in the organization, with their default branch +repos=$(gh repo list "$ORG" --limit 100 --json name,defaultBranchRef \ + --jq '.[] | "\(.name)\t\(.defaultBranchRef.name // "")"') + +failed=() +ok=0 + +while IFS=$'\t' read -r repo branch; do + [ -n "$repo" ] || continue + + # Empty repository: no default branch to check + if [ -z "$branch" ]; then + printf '%s•%s %-40s %s%-8s%s %s%s%s\n' \ + "$YELLOW" "$RESET" "$ORG/$repo" \ + "$YELLOW" "empty" "$RESET" \ + "$DIM" "no default branch" "$RESET" + continue + fi + + # Date of the last commit pushed to the default branch + updated=$(gh api "repos/$ORG/$repo/commits/$branch" 2>/dev/null | jq -r '.commit.committer.date // empty') + + # Legacy commit statuses + status_json=$(gh api "repos/$ORG/$repo/commits/$branch/status" 2>/dev/null) + s_count=0; s_state="" + if [ -n "$status_json" ]; then + s_count=$(jq -r '.total_count // 0' <<< "$status_json") + s_state=$(jq -r '.state // empty' <<< "$status_json") + fi + + # Workflow / check runs (GitHub Actions and other check-run apps) + checks_json=$(gh api "repos/$ORG/$repo/commits/$branch/check-runs" 2>/dev/null) + c_count=0; c_state="" + if [ -n "$checks_json" ]; then + c_count=$(jq -r '.total_count // 0' <<< "$checks_json") + c_state=$(jq -r ' + (.check_runs // []) as $r + | if ($r | length) == 0 then "" + elif [$r[] | select(.status != "completed")] | length > 0 then "pending" + elif [$r[] | select(.conclusion == "failure")] | length > 0 then "failure" + else "success" end' <<< "$checks_json") + fi + + if [ -z "$status_json" ] && [ -z "$checks_json" ]; then + # Nothing to report on at all (e.g. branch not reachable) + state="success" + label="null" + elif [ "$s_count" -eq 0 ] && [ "$c_count" -eq 0 ]; then + # No status checks and no workflow runs -> considered successful + state="success" + label="success" + elif [ "$s_state" = "failure" ] || [ "$s_state" = "error" ] || [ "$c_state" = "failure" ]; then + state="failure" + label="failure" + elif { [ "$s_count" -gt 0 ] && [ "$s_state" = "pending" ]; } || [ "$c_state" = "pending" ]; then + state="pending" + label="pending" + else + state="success" + label="success" + fi + + # 2026-08-20T14:33:02Z -> 2026-08-20 14:33 UTC + if [ -n "$updated" ]; then + when="${updated//T/ }" + when="last push to $branch ${when%:*Z} UTC" + else + when="no commits on $branch" + fi + + case "$state" in + success) + ok=$((ok + 1)) + color="$GREEN"; mark="✔" + ;; + failure|error) + failed+=("https://github.com/$ORG/$repo/commits/$branch") + color="$RED"; mark="✖" + ;; + *) + color="$YELLOW"; mark="•" + ;; + esac + + printf '%s%s%s %-40s %s%-8s%s %s%s%s\n' \ + "$color" "$mark" "$RESET" "$ORG/$repo" \ + "$color" "$label" "$RESET" \ + "$DIM" "$when" "$RESET" +done <<< "$repos" + +echo "" +if [ ${#failed[@]} -eq 0 ]; then + printf '%s%s✔ All %d repositories are green on their default branch%s\n' "$BOLD" "$GREEN" "$ok" "$RESET" +else + printf '%s%s✖ %d failing repositor%s%s\n' "$BOLD" "$RED" "${#failed[@]}" \ + "$([ ${#failed[@]} -eq 1 ] && echo y || echo ies)" "$RESET" + for url in "${failed[@]}"; do + printf ' %s%s%s\n' "$CYAN" "$url" "$RESET" + done + printf '%s%d passing%s\n' "$DIM" "$ok" "$RESET" +fi From 35685368f5fa3428ddd26e2d0ea0df59aa0ec1e5 Mon Sep 17 00:00:00 2001 From: Even Stensberg Date: Mon, 24 Aug 2026 20:44:52 +0200 Subject: [PATCH 2/3] feat: harden org-wide CI check script Only report a repository green when it can actually be verified: retry and detect API errors, paginate check runs, pin every request to one head SHA, keep the newest run per workflow/check, skip Dependabot dynamic runs, and report unverifiable repositories instead of rounding them up to a pass. Scan repositories in parallel and add ORG/JOBS/filter knobs. Co-Authored-By: Claude Opus 5 --- scripts/org-workflows-on-main.sh | 533 ++++++++++++++++++++++++++----- 1 file changed, 455 insertions(+), 78 deletions(-) diff --git a/scripts/org-workflows-on-main.sh b/scripts/org-workflows-on-main.sh index 1a83f46..29e8aca 100755 --- a/scripts/org-workflows-on-main.sh +++ b/scripts/org-workflows-on-main.sh @@ -1,9 +1,45 @@ #!/bin/bash +# +# Report CI health of every repository in a GitHub organisation, on its +# default branch. +# +# The checks are deliberately strict: a repository is only reported green when +# we could actually verify that it is green. Anything unverifiable -- an API +# error, a truncated response, an unknown check conclusion, a commit that no +# workflow ever ran on -- is reported as such instead of being rounded up to a +# pass. +# +# ORG=webpack ./org-workflows-on-main.sh +# +# Environment: +# ORG organisation to scan (default: webpack) +# INCLUDE_ARCHIVED also scan archived repositories (default: 0) +# INCLUDE_FORKS also scan forks (default: 0) +# CANCELLED_IS_FAIL count cancelled checks as failures (default: 0) +# INCLUDE_DYNAMIC_RUNS count Dependabot "dynamic/" runs (default: 0) +# JOBS repositories checked in parallel (default: 8) +# REPO_LIMIT max repositories to list (default: 1000) +# +# Exit status is 0 only when every scanned repository was verified green. +set -o pipefail -# usage: ./org-workflows-on-main.sh +ORG="${ORG:-webpack}" +INCLUDE_ARCHIVED="${INCLUDE_ARCHIVED:-0}" +INCLUDE_FORKS="${INCLUDE_FORKS:-0}" +CANCELLED_IS_FAIL="${CANCELLED_IS_FAIL:-0}" +INCLUDE_DYNAMIC_RUNS="${INCLUDE_DYNAMIC_RUNS:-0}" +JOBS="${JOBS:-8}" +REPO_LIMIT="${REPO_LIMIT:-1000}" -ORG="webpack" +# Absolute path to self, so the xargs workers can re-enter this script +# regardless of the caller's working directory. +SELF=$(cd "$(dirname "$0")" && pwd)/$(basename "$0") + +# Conclusions that mean the check genuinely failed. "cancelled" is handled +# separately because it usually means "superseded", not "broken". +FAIL_CONCLUSIONS='"failure","timed_out","startup_failure","action_required","stale"' +PASS_CONCLUSIONS='"success","neutral","skipped"' # Colors (disabled when not writing to a terminal) if [ -t 1 ]; then @@ -13,107 +49,448 @@ else BOLD=""; DIM=""; RESET=""; GREEN=""; RED=""; YELLOW=""; CYAN="" fi -# Check if logged into GitHub CLI -gh auth status || exit 1 +# --------------------------------------------------------------------------- +# API helper +# --------------------------------------------------------------------------- -# List all repositories in the organization, with their default branch -repos=$(gh repo list "$ORG" --limit 100 --json name,defaultBranchRef \ - --jq '.[] | "\(.name)\t\(.defaultBranchRef.name // "")"') +# gh_api [extra gh args...] +# +# Prints the response body and returns 0 only when the request really +# succeeded, 2 when the endpoint answered 404, 1 for anything else. gh writes the error body to stdout on a 4xx/5xx, so a plain +# `gh api ... 2>/dev/null` leaves jq looking at an error object and reading it +# as "zero checks, therefore fine" -- which is how unverifiable repositories +# used to be reported green. Transient failures are retried. +gh_api() { + local path="$1"; shift + local attempt out rc -failed=() -ok=0 + for attempt in 1 2 3; do + out=$(gh api "$path" "$@" 2>/dev/null) + rc=$? -while IFS=$'\t' read -r repo branch; do - [ -n "$repo" ] || continue + if [ $rc -eq 0 ] && [ -n "$out" ] && jq -e . >/dev/null 2>&1 <<< "$out"; then + # An error object gh happened to exit 0 on is still an error. + if jq -e '(if type == "array" then (.[0] // {}) else . end) + | type == "object" and has("message") and has("documentation_url")' \ + >/dev/null 2>&1 <<< "$out"; then + printf '%s' "$out" + api_is_404 "$out" && return 2 + return 1 + fi + printf '%s' "$out" + return 0 + fi + + # 4xx will not fix itself; only back off for empty or 5xx responses. + case "$out" in + *'"status": "4'*|*'"status":"4'*) break ;; + esac + sleep $(( attempt * 2 )) + done + + printf '%s' "$out" + api_is_404 "$out" && return 2 + return 1 +} + +# A 404 on an /actions/ endpoint means Actions is switched off for the +# repository, not that the repository is unverifiable. +api_is_404() { + case "$1" in + *'"status": "404"'*|*'"status":"404"'*|*'Not Found'*) return 0 ;; + esac + return 1 +} + +# Condense an error body into one line of explanation. +api_error() { + local msg + msg=$(jq -r '(if type == "array" then (.[0] // {}) else . end) | .message? // empty' \ + 2>/dev/null <<< "$1" | head -1) + printf '%s' "${msg:-unreachable}" +} + +# --------------------------------------------------------------------------- +# Per-repository check. Emits a single "\t