Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
22 changes: 15 additions & 7 deletions .github/copilot-instructions.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,19 @@ Each script follows the `github-<script-name>/github-<script-name>.sh` naming co

Provides reusable helpers sourced by scripts:

| Function | Purpose |
|----------|---------|
| `print_status` / `print_success` / `print_warning` / `print_error` | Colored output |
| `err <message>` | `print_error` + `exit 1` in one call |
| `require_env_var <VAR>` | Exit with message if variable unset/empty |
| `require_command <cmd>` | Exit if binary not in PATH |
| `configure_gh_auth [scope_hint]` | Bridge GITHUB_TOKEN→GH_TOKEN or verify gh auth session |
| `validate_github_token [bearer]` | Verify GITHUB_TOKEN via /user endpoint |
| `validate_slug <value> <label>` | Reject values with non-alphanumeric/hyphen/underscore chars |
| `gh_api <path> [--api-version V] [curl args...]` | Bearer-auth REST helper with 5-retry rate-limit handling; optional `--api-version` overrides the default `2022-11-28` header; returns literal `__404__` or `__422__` for those HTTP statuses — callers must check for these sentinels |
| `gh_api_paginate <path> [filter] [version]` | Paginated REST helper, follows Link headers, streams items; returns silently with empty output on 404/422 |
| `get_enterprise_orgs` | Three-tier enterprise org resolver (REST → GraphQL → /user/orgs) |
| `get_repo_page_count <url>` | Returns total pages for a paginated endpoint |

### Script Anatomy (Standardized Pattern)

Expand Down Expand Up @@ -84,12 +97,7 @@ API_URL_PREFIX=${API_URL_PREFIX:-'https://api.github.com'}

For org-level repo iteration (REST):
```bash
get_repo_pagination () {
repo_pages=$(curl -H "Authorization: token ${GITHUB_TOKEN}" -I "${API_URL_PREFIX}/orgs/${ORG}/repos?per_page=100" | grep -Eo '&page=[0-9]+' | grep -Eo '[0-9]+' | tail -1;)
echo "${repo_pages:-1}"
}

for PAGE in $(seq "$(get_repo_pagination)"); do
for PAGE in $(seq "$(get_repo_page_count "${API_URL_PREFIX}/orgs/${ORG}/repos?per_page=100")"); do
# Process repos on this page with per_page=100
done
```
Expand All @@ -101,7 +109,7 @@ For enterprise-level org iteration (GraphQL), use cursor-based pagination — se
Rate limiting delays are calibrated per operation type:
- **Repo-level operations** (permission grants, archival): `sleep 5` between each repository.
- **Code search** (`github-dockerfile-discovery`): configurable via `SEARCH_SLEEP` (default 2 s) and `CONTENT_SLEEP` (default 1 s).
- **gh_api helper** (lib): auto-retries up to 5 times on HTTP 403/429, sleeping 60 s before each retry. Returns the literal string `__404__` or `__422__` (exit 0) for those HTTP statuses instead of failing — every caller must check for these sentinels before passing the result to `jq`. `gh_api_paginate` returns silently with empty output on 404/422.
- **gh_api helper** (lib): auto-retries up to 5 times on HTTP 403/429, sleeping 60 s before each retry. Pass `--api-version VERSION` immediately after the URL to override the default `2022-11-28` header. Returns the literal string `__404__` or `__422__` (exit 0) for those HTTP statuses instead of failing — every caller must check for these sentinels before passing the result to `jq`. `gh_api_paginate` returns silently with empty output on 404/422.

### Authentication Headers

Expand Down
3 changes: 2 additions & 1 deletion AGENTS.md
Original file line number Diff line number Diff line change
Expand Up @@ -140,12 +140,13 @@ Always use `SCRIPT_DIR` to build the path to `lib/github-common.sh`. The library
| Function | Purpose |
|----------|---------|
| `print_status` / `print_success` / `print_warning` / `print_error` | Colored output |
| `err <message>` | `print_error` + `exit 1` in one call |
| `require_env_var <VAR>` | Exit with message if variable unset/empty |
| `require_command <cmd>` | Exit if binary not in PATH |
| `configure_gh_auth [scope_hint]` | Bridge GITHUB_TOKEN→GH_TOKEN or verify gh auth session |
| `validate_github_token [bearer]` | Verify GITHUB_TOKEN via /user endpoint |
| `validate_slug <value> <label>` | Reject values with non-alphanumeric/hyphen/underscore chars |
| `gh_api <path> [curl args...]` | Bearer-auth REST helper with 5-retry rate-limit handling; returns literal `__404__` or `__422__` for those HTTP statuses — callers must check for these sentinels |
| `gh_api <path> [--api-version V] [curl args...]` | Bearer-auth REST helper with 5-retry rate-limit handling; optional `--api-version` overrides the default `2022-11-28` header; returns literal `__404__` or `__422__` for those HTTP statuses — callers must check for these sentinels |
| `gh_api_paginate <path> [filter] [version]` | Paginated REST helper, follows Link headers, streams items; returns silently with empty output on 404/422 |
| `get_enterprise_orgs` | Three-tier enterprise org resolver (REST → GraphQL → /user/orgs) |
| `get_repo_page_count <url>` | Returns total pages for a paginated endpoint |
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,6 @@ set -euo pipefail
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=../../lib/github-common.sh
source "${SCRIPT_DIR}/../../lib/github-common.sh"
# Redirect status output to stderr — stdout is reserved for CSV data
print_status() { echo -e "${BLUE}[INFO]${NC} $1" >&2; }
print_success() { echo -e "${GREEN}[SUCCESS]${NC} $1" >&2; }
print_warning() { echo -e "${YELLOW}[WARNING]${NC} $1" >&2; }
print_error() { echo -e "${RED}[ERROR]${NC} $1" >&2; }

###
## GLOBAL VARIABLES
Expand Down Expand Up @@ -68,60 +63,6 @@ cleanup() {
}
trap cleanup EXIT

###
## Prerequisite checks
###
check_prerequisites() {
print_status "Checking prerequisites..."
require_command curl
require_command jq
require_command base64
require_env_var GITHUB_TOKEN "GITHUB_TOKEN"
print_success "Prerequisites OK"
}

###
## gh_api_link – like gh_api but also returns the Link header for pagination
## Writes body to stdout, sets global RESPONSE_LINK
###
RESPONSE_LINK=""
gh_api_link() {
local url="$1"
shift
[[ "${url}" == http* ]] || url="${API_URL_PREFIX}${url}"

local attempt
for attempt in 1 2 3 4 5; do
local response
response=$(curl -s -D - \
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
"$@" "${url}")

local http_code
http_code=$(echo "${response}" | head -1 | grep -oE '[0-9]{3}' | head -1)
RESPONSE_LINK=$(echo "${response}" | grep -i '^Link:' | tr -d '\r' | sed 's/Link: //' || true)
local body
body=$(echo "${response}" | awk 'BEGIN{p=0} /^\r?$/{p=1; next} p{print}')

if [[ "${http_code}" == "200" ]]; then
echo "${body}"
return 0
elif [[ "${http_code}" == "403" || "${http_code}" == "429" ]]; then
local retry_after=60
print_warning "Rate limited (HTTP ${http_code}). Sleeping ${retry_after}s..."
sleep "${retry_after}"
else
print_warning "HTTP ${http_code} for ${url} (attempt ${attempt}/5)"
sleep 5
fi
done

print_error "Failed to GET ${url} after 5 attempts"
return 1
}

###
## search_dockerfiles_in_org <org>
## Appends TSV rows (org TAB repo_full_name TAB path TAB html_url) to REFS_TEMP
Expand Down Expand Up @@ -383,7 +324,10 @@ build_text_summary() {
## MAIN
###
main() {
check_prerequisites
require_command curl
require_command jq
require_command base64
require_env_var GITHUB_TOKEN "GITHUB_TOKEN"
validate_github_token

mkdir -p "${REPORT_DIR}"
Expand Down
16 changes: 3 additions & 13 deletions enterprise/github-get-public-repos/github-get-public-repos.sh
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,6 @@ source "${SCRIPT_DIR}/../../lib/github-common.sh"
print_status() { echo -e "${BLUE}[INFO]${NC} $1" >&2; }
print_success() { echo -e "${GREEN}[SUCCESS]${NC} $1" >&2; }
print_warning() { echo -e "${YELLOW}[WARNING]${NC} $1" >&2; }
print_error() { echo -e "${RED}[ERROR]${NC} $1" >&2; }

###
## GLOBAL VARIABLES
Expand All @@ -60,17 +59,6 @@ cleanup() {
}
trap cleanup EXIT

###
## Prerequisite checks
###
check_prerequisites() {
print_status "Checking prerequisites..."
require_command curl
require_command jq
require_env_var GITHUB_TOKEN "GITHUB_TOKEN"
print_success "Prerequisites OK"
}

###
## resolve_orgs
## Returns the final deduplicated list of org logins to scan.
Expand Down Expand Up @@ -186,7 +174,9 @@ get_public_repos_for_org() {
## Main
###
main() {
check_prerequisites
require_command curl
require_command jq
require_env_var GITHUB_TOKEN "GITHUB_TOKEN"
validate_github_token

mkdir -p "${REPORT_DIR}"
Expand Down
15 changes: 11 additions & 4 deletions lib/github-common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,7 @@
# validate_github_token [bearer] — verify GITHUB_TOKEN via /user endpoint
# validate_token <VAR_NAME> — verify a secondary token variable
# validate_slug <value> [label] — exit if value contains unsafe chars
# gh_api <path|url> [curl args...] — Bearer-auth REST helper with retry;
# gh_api <path|url> [--api-version V] [curl args…] — Bearer-auth REST helper with retry;
# returns "__404__"/"__422__" (exit 0) for those codes
# gh_api_paginate <path> [filter] [version] — paginated REST, follows Link headers;
# silently returns empty output on 404/422
Expand Down Expand Up @@ -50,6 +50,7 @@ print_status() { echo -e "${BLUE}[INFO]${NC} $1"; }
print_success() { echo -e "${GREEN}[SUCCESS]${NC} $1"; }
print_warning() { echo -e "${YELLOW}[WARNING]${NC} $1"; }
print_error() { echo -e "${RED}[ERROR]${NC} $1" >&2; }
err() { print_error "$*"; exit 1; }

###
## require_env_var <VAR_NAME> [description]
Expand Down Expand Up @@ -167,15 +168,21 @@ validate_slug() {
}

###
## gh_api <path_or_full_url> [extra curl args...]
## gh_api <path_or_full_url> [--api-version VERSION] [extra curl args...]
## GitHub REST/GraphQL API helper with Bearer auth and rate-limit retries.
## Prepends API_URL_PREFIX when the first argument starts with /.
## Returns __404__ / __422__ for those status codes rather than failing.
## Any extra arguments after the URL are passed directly to curl (e.g. -X POST -d ...).
## Pass --api-version VERSION immediately after the URL to override the default
## API version (2022-11-28). Any remaining arguments are passed to curl.
###
gh_api() {
local url="$1"
local api_version="2022-11-28"
shift
if [[ "${1:-}" == "--api-version" ]]; then
api_version="$2"
shift 2
fi
[[ "${url}" == http* ]] || url="${API_URL_PREFIX}${url}"

local attempt
Expand All @@ -184,7 +191,7 @@ gh_api() {
body=$(curl -s -w "\n%{http_code}" \
-H "Authorization: Bearer ${GITHUB_TOKEN}" \
-H "Accept: application/vnd.github+json" \
-H "X-GitHub-Api-Version: 2022-11-28" \
-H "X-GitHub-Api-Version: ${api_version}" \
"$@" "${url}")
http_code=$(echo "${body}" | tail -1)
body=$(echo "${body}" | sed '$d')
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,16 +51,6 @@ usage() {
echo "Optional: PERMISSION=push REPO_EXCLUDE_REGEX=<regex> API_URL_PREFIX=<url>"
}

get_repo_pagination() {
local repo_pages
repo_pages=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" -I "${API_URL_PREFIX}/orgs/${ORG}/repos?per_page=100" | grep -Eo '&page=[0-9]+' | grep -Eo '[0-9]+' | tail -1)
echo "${repo_pages:-1}"
}

limit_repo_pagination() {
seq "$(get_repo_pagination)"
}

validate() {
require_env_var GITHUB_TOKEN "GitHub token"
require_env_var ORG "GitHub organization"
Expand All @@ -79,7 +69,7 @@ validate() {

get_matching_repos() {
local page repos_json
for page in $(limit_repo_pagination); do
for page in $(seq "$(get_repo_page_count "${API_URL_PREFIX}/orgs/${ORG}/repos?per_page=100")"); do
repos_json=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" -H "Accept: application/json" "${API_URL_PREFIX}/orgs/${ORG}/repos?type=all&per_page=100&page=${page}&sort=full_name")

if [ -n "${REPO_EXCLUDE_REGEX}" ]; then
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,15 +70,6 @@ if [ -n "${REPO_NAME_FILTER}" ]; then
print_status "Repository filter: ${REPO_NAME_FILTER}*"
fi

get_repo_pagination () {
repo_pages=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" -I "${API_URL_PREFIX}/orgs/${ORG}/repos?per_page=100" | grep -Eo '&page=[0-9]+' | grep -Eo '[0-9]+' | tail -1;)
echo "${repo_pages:-1}"
}

limit_repo_pagination () {
seq "$(get_repo_pagination)"
}

apply_team_permissions () {
local REPO_NAME=$1
local PERMISSION=$2
Expand Down Expand Up @@ -107,7 +98,7 @@ process_repos () {
local REPO
local repos_json

for PAGE in $(limit_repo_pagination); do
for PAGE in $(seq "$(get_repo_page_count "${API_URL_PREFIX}/orgs/${ORG}/repos?per_page=100")"); do
repos_json=$(curl -s -H "Authorization: token ${GITHUB_TOKEN}" "${API_URL_PREFIX}/orgs/${ORG}/repos?page=${PAGE}&per_page=100&sort=full_name")

if ! echo "${repos_json}" | jq -e 'type == "array"' > /dev/null 2>&1; then
Expand Down
Loading