From dd8114e84710c2181ecf8bad980b0ca69b217e72 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:21:35 -0500 Subject: [PATCH 1/6] feat(ci): file one issue per dead link the online link check finds maintenance-file-dead-links.yml runs on each completed run of the sync-managed online link check. A read job holding actions: read alone downloads the ci-workflows link-check-online-dead-links artifact and checks its shape; a file job holding issues: write alone files, updates or reopens one issue per URL, found by the hidden first-line marker , and points each at the attribution audit's demotion path for a dead pointer. Refs: #9759 Co-Authored-By: Claude Opus 5.5 --- .../api-budget-coverage.test.mjs | 6 + .../workflows/maintenance-file-dead-links.yml | 229 +++++++++++++++ scripts/maintenance-file-dead-links.test.sh | 263 ++++++++++++++++++ 3 files changed, 498 insertions(+) create mode 100644 .github/workflows/maintenance-file-dead-links.yml create mode 100755 scripts/maintenance-file-dead-links.test.sh diff --git a/.github/actions/resolve-config/api-budget-coverage.test.mjs b/.github/actions/resolve-config/api-budget-coverage.test.mjs index de5daf5e10b..7f6f0e22d71 100644 --- a/.github/actions/resolve-config/api-budget-coverage.test.mjs +++ b/.github/actions/resolve-config/api-budget-coverage.test.mjs @@ -45,6 +45,12 @@ const ALLOWED = new Map([ "intake-triage.yml:triage-split-children", `${NO_LOG}; step "Triage each sub-issue" calls gh without api_budget_wrap_gh`, ], + // issues: write alone, so no checkout to load the action from: one record + // listing and at most one write per dead link. + [ + "maintenance-file-dead-links.yml:file", + `${NO_LOG}; step "File or update one issue per dead link" calls gh without api_budget_wrap_gh`, + ], [ "post-merge-demotion-alert.yml:close-cleared", `${NO_LOG}; step "Close the records main cleared" calls gh without api_budget_wrap_gh`, diff --git a/.github/workflows/maintenance-file-dead-links.yml b/.github/workflows/maintenance-file-dead-links.yml new file mode 100644 index 00000000000..ef4bfaa6204 --- /dev/null +++ b/.github/workflows/maintenance-file-dead-links.yml @@ -0,0 +1,229 @@ +name: maintenance-file-dead-links + +# Routes the dead links the weekly online link check finds to issues, one per +# URL, so each is repaired by the demotion path in +# plugins/attribution/skills/audit/reference/dispositions.md ("The demotion +# path when a pointer later dies"). This repository owns that trigger; the +# plugin owns the repair. +# +# The link check (maintenance-check-links-online-claude-code-plugins.yml) is a +# sync-managed caller of the ci-workflows reusable workflow, which uploads the +# run's broken links as the link-check-online-dead-links artifact. A +# reusable workflow's outputs reach only its own caller, so this workflow reads +# the artifact instead, in two jobs that each hold one scope: +# +# - read: actions: read, to download the artifact from the link-check run. It +# checks the shape, keeps at most MAX_LINKS links, and hands them on through +# its outputs. A run that uploaded no artifact (a scheduled run the +# repository property turned off) files nothing. +# - file: issues: write alone, no checkout. For each link it finds the record +# by the hidden first-line marker +# `` on an +# issue github-actions[bot] filed. An open record has its body rewritten +# with this run's locations; a record closed as completed is reopened, since +# the link is dead again; one closed as not planned is left alone. A link +# with no record gets a new issue, at most MAX_NEW a run. Its token starts +# no workflow; the hourly triage sweep in maintenance-audit-lane-runs.yml +# brings the issue to intake. +# +# The trigger names the link-check workflow by its `name:`, which the +# standards component owns: a rename there disconnects this workflow. + +on: + # Runs this file from the default branch and executes no code from the + # link-check run: the artifact is data, checked before use. + workflow_run: # zizmor: ignore[dangerous-triggers] + workflows: [maintenance-check-links-online-claude-code-plugins] + types: [completed] + branches: [main] + +permissions: {} + +jobs: + read: + name: Read the dead links + if: >- + github.event.workflow_run.head_branch == 'main' + && github.event.workflow_run.head_repository.full_name == github.repository + && contains(fromJSON('["success", "failure"]'), github.event.workflow_run.conclusion) + runs-on: ubuntu-24.04 + timeout-minutes: 5 + permissions: + actions: read # download the link-check run's artifact + outputs: + links: ${{ steps.links.outputs.links }} + steps: + # A run the property gate skipped uploads no artifact; that is not an + # error, so the next step decides. + - name: Download the dead links + id: download + continue-on-error: true + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: link-check-online-dead-links + path: ${{ runner.temp }}/dead-links + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + + - name: Check the dead links + id: links + env: + DOWNLOAD: ${{ steps.download.outcome }} + DEAD_LINKS: ${{ runner.temp }}/dead-links/dead-links.json + RUN_CONCLUSION: ${{ github.event.workflow_run.conclusion }} + MAX_LINKS: 50 + run: | + set -euo pipefail + if [[ "$DOWNLOAD" != success || ! -s "$DEAD_LINKS" ]]; then + if [[ "$RUN_CONCLUSION" == failure ]]; then + echo "::warning::the red link-check run left no dead-links artifact: nothing to file" + else + echo "the link-check run left no dead-links artifact: nothing to file" + fi + exit 0 + fi + # Each URL a plain http(s) URL, each path free of backticks and line + # breaks, so both sit safely in a code span; anything else is dropped. + if ! links="$(jq -c --argjson max "$MAX_LINKS" ' + if type != "array" then error("not an array") else . end + | map(select((.url | type == "string" and length <= 2000 and test("^https?://[^\\s`<>]+$")) + and (.locations | type == "array"))) + | map({url, + status: (if (.status | type) == "number" then .status else null end), + detail: ((.detail // "") | tostring | gsub("[\\r\\n`]+"; " ") | .[:200]), + locations: ([.locations[] + | select(type == "object" and (.path | type == "string" and test("^[^`\\r\\n]{1,300}$"))) + | {path, line: (if (.line | type) == "number" then .line else null end)}][:20])}) + | map(select(.locations | length > 0)) + | .[:$max]' "$DEAD_LINKS")"; then + echo "::error::the dead-links artifact is not a JSON array" + exit 1 + fi + total="$(jq 'length' "$DEAD_LINKS")" + kept="$(jq 'length' <<<"$links")" + if ((kept < total)); then + echo "::warning::kept $kept of $total dead links (malformed entries dropped, at most $MAX_LINKS kept)" + fi + if ((kept == 0)); then + echo "no dead links to file" + exit 0 + fi + delim="EOF_$(openssl rand -hex 16)" + { + echo "links<<$delim" + echo "$links" + echo "$delim" + } >>"$GITHUB_OUTPUT" + echo "handed $kept dead links to the file job" + + file: + name: File the dead-link issues + needs: [read] + if: needs.read.outputs.links != '' + runs-on: ubuntu-24.04 + timeout-minutes: 10 + permissions: + issues: write # read, file, update and reopen the dead-link records + # Two link-check runs finishing close together must not both find no + # record and file two. + concurrency: + group: maintenance-file-dead-links + cancel-in-progress: false + steps: + - name: File or update one issue per dead link + env: + GH_TOKEN: ${{ github.token }} + LINKS: ${{ needs.read.outputs.links }} + RUN_URL: ${{ github.event.workflow_run.html_url }} + MAX_NEW: 10 + PACE_SECONDS: 3 + DEMOTION_PATH: https://github.com/melodic-software/claude-code-plugins/blob/main/plugins/attribution/skills/audit/reference/dispositions.md#the-demotion-path-when-a-pointer-later-dies + run: | + set -euo pipefail + # Every record github-actions[bot] filed, by URL hash; an open one + # wins over a closed one. A failed read files nothing, since a + # weekly duplicate per link is worse than a week's delay. + # shellcheck disable=SC2016 # jq string interpolation, not shell + if ! listing="$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues?state=all&creator=github-actions%5Bbot%5D&per_page=100" \ + --jq '.[] | select(.pull_request == null) + | ((.body // "") | sub("^\\s+"; "") | split("\n")[0] + | capture("^$").h) as $h + | "\($h) \(.number) \(.state) \(.state_reason // "none")"')"; then + echo "::error::could not read the dead-link records: nothing filed" + exit 1 + fi + declare -A records=() + while read -r h number state reason; do + [[ -n "$h" ]] || continue + if [[ -z "${records[$h]:-}" || "$state" == open ]]; then + records[$h]="$number $state $reason" + fi + done <<<"$listing" + + filed=0 updated=0 reopened=0 left=0 failed=0 + over=() + while IFS= read -r link; do + url="$(jq -r '.url' <<<"$link")" + hash="$(printf '%s' "$url" | sha256sum | cut -d' ' -f1)" + body="$(jq -r --arg hash "$hash" --arg run "$RUN_URL" --arg demote "$DEMOTION_PATH" ' + "\n\n" + + "The weekly online link check found this link dead.\n\n" + + "- URL: `\(.url)`\n" + + "- Status: \(.status // "no status code")\(if .detail != "" then " (\(.detail))" else "" end)\n" + + "- Run: \($run)\n\n" + + "Where it appears:\n\n" + + ([.locations[] | "- `\(.path)`" + (if .line then " line \(.line)" else "" end)] | join("\n")) + + "\n\nRepair each pointer by [the demotion path when a pointer later dies](\($demote)): " + + "point at a live source that holds the same fact, or demote the pointer to a stamped record " + + "or an archived-snapshot citation; never re-expand it into a copy. A page that is alive but " + + "refuses the checker gets an exact-URL `.lycheeignore` entry with a dated reason.\n" + ' <<<"$link")" + record="${records[$hash]:-}" + if [[ -z "$record" ]]; then + if ((filed >= MAX_NEW)); then + over+=("$url") + continue + fi + if gh api -X POST "repos/$GITHUB_REPOSITORY/issues" -f title="Dead link: ${url:0:180}" -f body="$body" >/dev/null; then + filed=$((filed + 1)) + else + echo "::error::could not file the dead-link issue for $url" + failed=$((failed + 1)) + fi + else + read -r number state reason <<<"$record" + if [[ "$state" == closed && "$reason" == not_planned ]]; then + echo "#$number for $url was closed as not planned: left alone" + left=$((left + 1)) + continue + fi + if gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/$number" -f state=open -f body="$body" >/dev/null; then + if [[ "$state" == open ]]; then + updated=$((updated + 1)) + else + reopened=$((reopened + 1)) + fi + else + echo "::error::could not update #$number for $url" + failed=$((failed + 1)) + fi + fi + sleep "$PACE_SECONDS" + done < <(jq -c '.[]' <<<"$LINKS") + + { + echo "# Dead-link issues" + echo + echo "Filed $filed, updated $updated, reopened $reopened, left $left closed as not planned, $failed failed." + if ((${#over[@]} > 0)); then + echo + echo "Over the cap of $MAX_NEW new issues, filed next run:" + echo + # shellcheck disable=SC2016 # markdown code span, not shell + printf -- '- `%s`\n' "${over[@]}" + fi + } >>"$GITHUB_STEP_SUMMARY" + echo "filed $filed, updated $updated, reopened $reopened, left $left, failed $failed, over the cap ${#over[@]}" + if ((failed > 0)); then + exit 1 + fi diff --git a/scripts/maintenance-file-dead-links.test.sh b/scripts/maintenance-file-dead-links.test.sh new file mode 100755 index 00000000000..18c42c121c7 --- /dev/null +++ b/scripts/maintenance-file-dead-links.test.sh @@ -0,0 +1,263 @@ +#!/usr/bin/env bash +# Runs the two step bodies of .github/workflows/maintenance-file-dead-links.yml +# under the shell Actions gives a step with no `shell:` (bash -e): `Check the +# dead links` against artifact fixtures, and `File or update one issue per +# dead link` against a stub `gh` that applies the step's own --jq filter to a +# fixture issue list and logs every write. Also checks the workflow's shape: +# the trigger names the link-check workflow, and each job holds one scope. +# test-scope: .github/workflows/maintenance-file-dead-links.yml +set -uo pipefail +unset GIT_DIR GIT_WORK_TREE GIT_CONFIG + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +WORKFLOW="$ROOT/.github/workflows/maintenance-file-dead-links.yml" +CALLER="$ROOT/.github/workflows/maintenance-check-links-online-claude-code-plugins.yml" + +# shellcheck source=lib/test-harness.sh +. "$ROOT/scripts/lib/test-harness.sh" + +TMP_ROOT="$(mktemp -d)" +trap 'rm -rf "$TMP_ROOT"' EXIT +export TMPDIR="$TMP_ROOT" + +# The `run: |` body of the step named $1, de-indented. +step_body() { + awk -v name=" - name: $1" '$0 == name { on = 1; next } + on && /^ - / { exit } + on { print }' "$WORKFLOW" | + awk '/^ run: \|$/ { on = 1; next } + on && /^ / { print substr($0, 11); next } + on && /^[[:blank:]]*$/ { print ""; next } + on { exit }' +} +step_body 'Check the dead links' >"$TMP_ROOT/read.sh" +step_body 'File or update one issue per dead link' >"$TMP_ROOT/file.sh" +for body in read file; do + if [[ ! -s "$TMP_ROOT/$body.sh" ]]; then + fail "no $body step with a run: | body in $WORKFLOW" + test_harness::report + exit 1 + fi +done + +# --- Workflow shape ----------------------------------------------------------- +job() { awk -v j=" $1:" '$0 == j { on = 1; next } on && /^ [a-z-]+:$/ { exit } on' "$WORKFLOW"; } +scopes() { grep -oE '^ +[a-z-]+: (read|write)( |$)' | sed 's/^ *//; s/ *$//' | sort | paste -sd ' '; } +caller_name="$(sed -n 's/^name: //p' "$CALLER")" +triggers="$(awk '/^ workflow_run:/ { on = 1; next } on && /^ [a-z]/ { exit } on' "$WORKFLOW")" +if [[ -n "$caller_name" && "$triggers" == *"workflows: [$caller_name]"* && "$triggers" == *"types: [completed]"* ]]; then + ok "workflow_run fires on completed $caller_name runs" +else + fail "workflow_run does not name the link-check workflow '$caller_name': '$triggers'" +fi +if grep -q '^permissions: {}$' "$WORKFLOW"; then + ok "the workflow grants nothing by default" +else + fail "the workflow-level permissions are not {}" +fi +if [[ "$(job read | scopes)" == 'actions: read' ]]; then + ok "the read job holds actions: read alone" +else + fail "the read job's permissions are not exactly actions: read: '$(job read | scopes)'" +fi +if [[ "$(job file | scopes)" == 'issues: write' ]] && ! job file | grep -qE 'uses:|actions/checkout'; then + ok "the file job holds issues: write alone and checks out nothing" +else + fail "the file job's permissions are not exactly issues: write, or it uses an action" +fi +if job read | grep -q 'name: link-check-online-dead-links$'; then + ok "the read job downloads the ci-workflows link-check-online-dead-links artifact" +else + fail "the read job does not download link-check-online-dead-links" +fi + +# --- Check the dead links ----------------------------------------------------- +run_read() { # [MAX_LINKS] + local out="$TMP_ROOT/read-output" fixture="$TMP_ROOT/dead-links.json" + : >"$out" + rm -f "$fixture" + [[ -n "$3" ]] && printf '%s' "$3" >"$fixture" + READ_LOG="$(env -i PATH="$PATH" DOWNLOAD="$1" RUN_CONCLUSION="$2" DEAD_LINKS="$fixture" \ + MAX_LINKS="${4:-50}" GITHUB_OUTPUT="$out" bash -e "$TMP_ROOT/read.sh" 2>&1)" + READ_STATUS=$? + READ_LINKS="$(awk '/^links<"$TMP_ROOT/bin/gh" <<'STUB' +#!/usr/bin/env bash +# Listing: apply the step's --jq filter to the fixture issues. Writes: log them. +if [[ "${GH_LIST_STATUS:-0}" != 0 && " $* " == *" --paginate "* ]]; then + exit "$GH_LIST_STATUS" +fi +args=("$@") filter='' method=GET path='' +for ((i = 0; i < ${#args[@]}; i++)); do + case "${args[i]}" in + --jq) filter="${args[i + 1]}" ;; + -X) method="${args[i + 1]}" ;; + repos/*) path="${args[i]}" ;; + esac +done +if [[ "$method" == GET ]]; then + jq -r "$filter" "$GH_ISSUES" + exit +fi +{ + printf '%s %s\n' "$method" "$path" + for ((i = 0; i < ${#args[@]}; i++)); do + [[ "${args[i]}" == -f ]] && printf 'F %s\n' "${args[i + 1]%%$'\n'*}" + done +} >>"$GH_LOG" +if [[ -n "${GH_WRITE_BODIES:-}" ]]; then + for ((i = 0; i < ${#args[@]}; i++)); do + [[ "${args[i]}" == -f && "${args[i + 1]}" == body=* ]] && printf '%s\n----\n' "${args[i + 1]#body=}" >>"$GH_WRITE_BODIES" + done +fi +exit "${GH_WRITE_STATUS:-0}" +STUB +chmod +x "$TMP_ROOT/bin/gh" + +sha() { printf '%s' "$1" | sha256sum | cut -d' ' -f1; } +link() { jq -cn --arg u "$1" '{url: $u, status: 404, detail: "Not Found", locations: [{path: "docs/a.md", line: 7}]}'; } +url_open='https://example.com/open' url_done='https://example.com/done' url_wont='https://example.com/wont' +url_new1='https://example.com/new1' url_new2='https://example.com/new2' +marker() { printf '' "$(sha "$1")"; } +jq -n --arg mo "$(marker "$url_open")" --arg md "$(marker "$url_done")" --arg mw "$(marker "$url_wont")" \ + --arg mn "$(marker "$url_new1")" --arg mp "$(marker "$url_new2")" '[ + {number: 11, state: "open", state_reason: null, body: ($mo + "\n\nold")}, + {number: 12, state: "closed", state_reason: "completed", body: ($md + "\n")}, + {number: 13, state: "closed", state_reason: "not_planned", body: ($mw + "\n")}, + {number: 14, state: "open", state_reason: null, body: ("intro\n" + $mn), pull_request: null}, + {number: 15, state: "open", state_reason: null, body: $mp, pull_request: {url: "x"}} +]' >"$TMP_ROOT/issues.json" + +run_file() { # [VAR=value ...] + local links="$1" + shift + : >"$TMP_ROOT/gh.log" + : >"$TMP_ROOT/bodies" + : >"$TMP_ROOT/summary" + FILE_LOG="$(env -i PATH="$TMP_ROOT/bin:$PATH" GH_ISSUES="$TMP_ROOT/issues.json" GH_LOG="$TMP_ROOT/gh.log" \ + GH_WRITE_BODIES="$TMP_ROOT/bodies" GITHUB_REPOSITORY=o/r GITHUB_STEP_SUMMARY="$TMP_ROOT/summary" \ + LINKS="$links" RUN_URL=https://github.com/o/r/actions/runs/1 MAX_NEW=1 PACE_SECONDS=0 \ + DEMOTION_PATH=https://example.com/dispositions "$@" bash -e "$TMP_ROOT/file.sh" 2>&1)" + FILE_STATUS=$? + GH_WRITES="$(cat "$TMP_ROOT/gh.log")" +} + +all="[$(link "$url_open"),$(link "$url_done"),$(link "$url_wont"),$(link "$url_new1"),$(link "$url_new2")]" +run_file "$all" +expected="PATCH repos/o/r/issues/11 +F state=open +F body=$(marker "$url_open") +PATCH repos/o/r/issues/12 +F state=open +F body=$(marker "$url_done") +POST repos/o/r/issues +F title=Dead link: $url_new1 +F body=$(marker "$url_new1")" +if ((FILE_STATUS == 0)) && [[ "$GH_WRITES" == "$expected" ]]; then + ok "an open record is updated, a completed one reopened, a not-planned one left, and new links filed up to MAX_NEW" +else + fail "the file step wrote the wrong records: status $FILE_STATUS, writes: +$GH_WRITES +log: $FILE_LOG" +fi +if [[ "$(cat "$TMP_ROOT/summary")" == *"Filed 1, updated 1, reopened 1, left 1 closed as not planned, 0 failed."* && + "$(cat "$TMP_ROOT/summary")" == *"- \`$url_new2\`"* ]]; then + ok "the summary counts each outcome and lists the link over the cap" +else + fail "the summary is wrong: $(cat "$TMP_ROOT/summary")" +fi +first_body="$(awk '/^----$/ { exit } { print }' "$TMP_ROOT/bodies")" +# shellcheck disable=SC2016 # a literal markdown code span +if [[ "$first_body" == *"- URL: \`$url_open\`"* && "$first_body" == *'- `docs/a.md` line 7'* && + "$first_body" == *"- Run: https://github.com/o/r/actions/runs/1"* && + "$first_body" == *"(https://example.com/dispositions)"* && "$first_body" == *"- Status: 404 (Not Found)"* ]]; then + ok "the body names the URL, status, run, each location and the demotion path" +else + fail "the issue body is missing a part: $first_body" +fi + +run_file "[$(link "$url_new1")]" MAX_NEW=5 +if [[ "$GH_WRITES" == "POST repos/o/r/issues"* ]]; then + ok "a marker that is not the first body line, or sits on a pull request, is not a record" +else + fail "a misplaced marker counted as a record: $GH_WRITES" +fi + +run_file "$all" GH_LIST_STATUS=1 +if ((FILE_STATUS == 1)) && [[ -z "$GH_WRITES" && "$FILE_LOG" == *"::error::could not read the dead-link records"* ]]; then + ok "a failed record read files nothing and fails" +else + fail "a failed record read did not stop the filer: $FILE_STATUS $GH_WRITES $FILE_LOG" +fi + +run_file "[$(link "$url_new1")]" GH_WRITE_STATUS=1 +if ((FILE_STATUS == 1)) && [[ "$FILE_LOG" == *"::error::could not file the dead-link issue for $url_new1"* ]]; then + ok "a failed write fails the job" +else + fail "a failed write did not fail the job: $FILE_STATUS $FILE_LOG" +fi + +test_harness::report From 391587dafc7993462a4c2477280e657ed8cae172 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:27:08 -0500 Subject: [PATCH 2/6] fix(ci): label new dead-link issues needs-triage and say when to close one An issue the GITHUB_TOKEN opens raises no event, so intake never adds the needs-triage floor label, and the hourly triage sweep leaves github-actions[bot] issues out: the filer adds the label itself. The body now says a close sticks only once the link check stops reporting the URL, and puts the server's status detail in a code span. Refs: #9759 Co-Authored-By: Claude Opus 5.5 --- .../workflows/maintenance-file-dead-links.yml | 17 +++++++++++------ scripts/maintenance-file-dead-links.test.sh | 8 +++++--- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/.github/workflows/maintenance-file-dead-links.yml b/.github/workflows/maintenance-file-dead-links.yml index ef4bfaa6204..9b3fb7495f0 100644 --- a/.github/workflows/maintenance-file-dead-links.yml +++ b/.github/workflows/maintenance-file-dead-links.yml @@ -22,9 +22,11 @@ name: maintenance-file-dead-links # issue github-actions[bot] filed. An open record has its body rewritten # with this run's locations; a record closed as completed is reopened, since # the link is dead again; one closed as not planned is left alone. A link -# with no record gets a new issue, at most MAX_NEW a run. Its token starts -# no workflow; the hourly triage sweep in maintenance-audit-lane-runs.yml -# brings the issue to intake. +# with no record gets a new issue, at most MAX_NEW a run, labeled +# needs-triage. Its token starts no workflow, so intake-triage.yml never +# adds that floor label on open, and the hourly triage sweep leaves +# github-actions[bot] issues out (triage-sweep.mjs): the label is what puts +# the issue in the untriaged queue. A reopened record keeps its labels. # # The trigger names the link-check workflow by its `name:`, which the # standards component owns: a rename there disconnects this workflow. @@ -169,14 +171,17 @@ jobs: "\n\n" + "The weekly online link check found this link dead.\n\n" + "- URL: `\(.url)`\n" - + "- Status: \(.status // "no status code")\(if .detail != "" then " (\(.detail))" else "" end)\n" + + "- Status: \(.status // "no status code")\(if .detail != "" then " `\(.detail)`" else "" end)\n" + "- Run: \($run)\n\n" + "Where it appears:\n\n" + ([.locations[] | "- `\(.path)`" + (if .line then " line \(.line)" else "" end)] | join("\n")) + "\n\nRepair each pointer by [the demotion path when a pointer later dies](\($demote)): " + "point at a live source that holds the same fact, or demote the pointer to a stamped record " + "or an archived-snapshot citation; never re-expand it into a copy. A page that is alive but " - + "refuses the checker gets an exact-URL `.lycheeignore` entry with a dated reason.\n" + + "refuses the checker gets an exact-URL `.lycheeignore` entry with a dated reason.\n\n" + + "Close this only once the link check no longer reports the URL: removed, replaced, or in " + + "`.lycheeignore`. A dead URL kept in a link, the marked pointer of a demoted record included, " + + "is found again and reopens this issue on the next run.\n" ' <<<"$link")" record="${records[$hash]:-}" if [[ -z "$record" ]]; then @@ -184,7 +189,7 @@ jobs: over+=("$url") continue fi - if gh api -X POST "repos/$GITHUB_REPOSITORY/issues" -f title="Dead link: ${url:0:180}" -f body="$body" >/dev/null; then + if gh api -X POST "repos/$GITHUB_REPOSITORY/issues" -f 'labels[]=needs-triage' -f title="Dead link: ${url:0:180}" -f body="$body" >/dev/null; then filed=$((filed + 1)) else echo "::error::could not file the dead-link issue for $url" diff --git a/scripts/maintenance-file-dead-links.test.sh b/scripts/maintenance-file-dead-links.test.sh index 18c42c121c7..2d7583492cc 100755 --- a/scripts/maintenance-file-dead-links.test.sh +++ b/scripts/maintenance-file-dead-links.test.sh @@ -214,10 +214,11 @@ PATCH repos/o/r/issues/12 F state=open F body=$(marker "$url_done") POST repos/o/r/issues +F labels[]=needs-triage F title=Dead link: $url_new1 F body=$(marker "$url_new1")" if ((FILE_STATUS == 0)) && [[ "$GH_WRITES" == "$expected" ]]; then - ok "an open record is updated, a completed one reopened, a not-planned one left, and new links filed up to MAX_NEW" + ok "an open record is updated, a completed one reopened, a not-planned one left, and new links filed needs-triage up to MAX_NEW" else fail "the file step wrote the wrong records: status $FILE_STATUS, writes: $GH_WRITES @@ -233,8 +234,9 @@ first_body="$(awk '/^----$/ { exit } { print }' "$TMP_ROOT/bodies")" # shellcheck disable=SC2016 # a literal markdown code span if [[ "$first_body" == *"- URL: \`$url_open\`"* && "$first_body" == *'- `docs/a.md` line 7'* && "$first_body" == *"- Run: https://github.com/o/r/actions/runs/1"* && - "$first_body" == *"(https://example.com/dispositions)"* && "$first_body" == *"- Status: 404 (Not Found)"* ]]; then - ok "the body names the URL, status, run, each location and the demotion path" + "$first_body" == *"(https://example.com/dispositions)"* && "$first_body" == *'- Status: 404 `Not Found`'* && + "$first_body" == *"Close this only once the link check no longer reports the URL"* ]]; then + ok "the body names the URL, status in a code span, run, each location, the demotion path and when to close" else fail "the issue body is missing a part: $first_body" fi From c77bb36beb3c5dff3cb23363aa654bc610cf9045 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:33:45 -0500 Subject: [PATCH 3/6] fix(ci): leave the server's text out of dead-link issue bodies The status detail is written by the remote server, outside the org, and intake's model reads these issue bodies, so the body keeps only the status code. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/maintenance-file-dead-links.yml | 2 +- scripts/maintenance-file-dead-links.test.sh | 5 +++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/maintenance-file-dead-links.yml b/.github/workflows/maintenance-file-dead-links.yml index 9b3fb7495f0..e504c8c4c3b 100644 --- a/.github/workflows/maintenance-file-dead-links.yml +++ b/.github/workflows/maintenance-file-dead-links.yml @@ -171,7 +171,7 @@ jobs: "\n\n" + "The weekly online link check found this link dead.\n\n" + "- URL: `\(.url)`\n" - + "- Status: \(.status // "no status code")\(if .detail != "" then " `\(.detail)`" else "" end)\n" + + "- Status: \(.status // "no status code") (the text the server sent is left out: it is written outside the org and intake reads this body)\n" + "- Run: \($run)\n\n" + "Where it appears:\n\n" + ([.locations[] | "- `\(.path)`" + (if .line then " line \(.line)" else "" end)] | join("\n")) diff --git a/scripts/maintenance-file-dead-links.test.sh b/scripts/maintenance-file-dead-links.test.sh index 2d7583492cc..e01b3e60689 100755 --- a/scripts/maintenance-file-dead-links.test.sh +++ b/scripts/maintenance-file-dead-links.test.sh @@ -234,9 +234,10 @@ first_body="$(awk '/^----$/ { exit } { print }' "$TMP_ROOT/bodies")" # shellcheck disable=SC2016 # a literal markdown code span if [[ "$first_body" == *"- URL: \`$url_open\`"* && "$first_body" == *'- `docs/a.md` line 7'* && "$first_body" == *"- Run: https://github.com/o/r/actions/runs/1"* && - "$first_body" == *"(https://example.com/dispositions)"* && "$first_body" == *'- Status: 404 `Not Found`'* && + "$first_body" == *"(https://example.com/dispositions)"* && "$first_body" == *'- Status: 404 (the text the server sent is left out'* && + "$first_body" != *"Not Found"* && "$first_body" == *"Close this only once the link check no longer reports the URL"* ]]; then - ok "the body names the URL, status in a code span, run, each location, the demotion path and when to close" + ok "the body names the URL, status code without the server's text, run, each location, the demotion path and when to close" else fail "the issue body is missing a part: $first_body" fi From 4dc48cb7e10f98e7ff864d1001b6aa284379e516 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:41:19 -0500 Subject: [PATCH 4/6] fix(ci): keep one rollup issue for dead links instead of one per link The owner chose one rollup issue. The file job now finds the issue github-actions[bot] filed whose first body line is and rewrites its task list of dead links (URL, status code, locations) each run. With no dead links it closes an open rollup as completed with a comment; a completed rollup is reopened when links die again, one closed as not planned is left alone, and a new one, labeled needs-triage, is opened only when none exists. The read job hands on an empty list so the rollup can close, drops the server's text, and counts the links over the 50-link cap for a "+N more" line. The per-link marker, the 10-new-issue cap and the write spacing are gone. Refs: #9759 Co-Authored-By: Claude Opus 5.5 --- .../api-budget-coverage.test.mjs | 6 +- .../workflows/maintenance-file-dead-links.yml | 201 ++++++++---------- scripts/maintenance-file-dead-links.test.sh | 193 ++++++++++------- 3 files changed, 205 insertions(+), 195 deletions(-) diff --git a/.github/actions/resolve-config/api-budget-coverage.test.mjs b/.github/actions/resolve-config/api-budget-coverage.test.mjs index 7f9db818ef7..60611717605 100644 --- a/.github/actions/resolve-config/api-budget-coverage.test.mjs +++ b/.github/actions/resolve-config/api-budget-coverage.test.mjs @@ -45,11 +45,11 @@ const ALLOWED = new Map([ "intake-triage.yml:triage-split-children", `${NO_LOG}; step "Triage each sub-issue" calls gh without api_budget_wrap_gh`, ], - // issues: write alone, so no checkout to load the action from: one record - // listing and at most one write per dead link. + // issues: write alone, so no checkout to load the action from: one rollup + // listing and at most two writes. [ "maintenance-file-dead-links.yml:file", - `${NO_LOG}; step "File or update one issue per dead link" calls gh without api_budget_wrap_gh`, + `${NO_LOG}; step "Rewrite the rollup issue" calls gh without api_budget_wrap_gh`, ], [ "post-merge-demotion-alert.yml:close-cleared", diff --git a/.github/workflows/maintenance-file-dead-links.yml b/.github/workflows/maintenance-file-dead-links.yml index e504c8c4c3b..4d6a507241b 100644 --- a/.github/workflows/maintenance-file-dead-links.yml +++ b/.github/workflows/maintenance-file-dead-links.yml @@ -1,7 +1,7 @@ name: maintenance-file-dead-links -# Routes the dead links the weekly online link check finds to issues, one per -# URL, so each is repaired by the demotion path in +# Routes the dead links the weekly online link check finds to one rollup +# issue, so each is repaired by the demotion path in # plugins/attribution/skills/audit/reference/dispositions.md ("The demotion # path when a pointer later dies"). This repository owns that trigger; the # plugin owns the repair. @@ -14,19 +14,19 @@ name: maintenance-file-dead-links # # - read: actions: read, to download the artifact from the link-check run. It # checks the shape, keeps at most MAX_LINKS links, and hands them on through -# its outputs. A run that uploaded no artifact (a scheduled run the -# repository property turned off) files nothing. -# - file: issues: write alone, no checkout. For each link it finds the record -# by the hidden first-line marker -# `` on an -# issue github-actions[bot] filed. An open record has its body rewritten -# with this run's locations; a record closed as completed is reopened, since -# the link is dead again; one closed as not planned is left alone. A link -# with no record gets a new issue, at most MAX_NEW a run, labeled -# needs-triage. Its token starts no workflow, so intake-triage.yml never +# its outputs, an empty list included. A run that uploaded no artifact (a +# scheduled run the repository property turned off) hands on nothing, so +# the rollup is left as it is. +# - file: issues: write alone, no checkout. It finds the rollup, an issue +# github-actions[bot] filed whose first body line is +# ``, and rewrites its list of dead +# links. With none it closes an open rollup as completed, with a comment. A +# rollup closed as completed is reopened when links are dead again; one +# closed as not planned is left alone. With no rollup it opens one, labeled +# needs-triage: its token starts no workflow, so intake-triage.yml never # adds that floor label on open, and the hourly triage sweep leaves -# github-actions[bot] issues out (triage-sweep.mjs): the label is what puts -# the issue in the untriaged queue. A reopened record keeps its labels. +# github-actions[bot] issues out (triage-sweep.mjs), so the label is what +# puts the issue in the untriaged queue. A reopened rollup keeps its labels. # # The trigger names the link-check workflow by its `name:`, which the # standards component owns: a rename there disconnects this workflow. @@ -53,7 +53,9 @@ jobs: permissions: actions: read # download the link-check run's artifact outputs: + found: ${{ steps.links.outputs.found }} links: ${{ steps.links.outputs.links }} + more: ${{ steps.links.outputs.more }} steps: # A run the property gate skipped uploads no artifact; that is not an # error, so the next step decides. @@ -86,149 +88,124 @@ jobs: fi # Each URL a plain http(s) URL, each path free of backticks and line # breaks, so both sit safely in a code span; anything else is dropped. - if ! links="$(jq -c --argjson max "$MAX_LINKS" ' + # The server's text is never kept: it is written outside the org and + # intake reads the issue body. + if ! valid="$(jq -c ' if type != "array" then error("not an array") else . end | map(select((.url | type == "string" and length <= 2000 and test("^https?://[^\\s`<>]+$")) and (.locations | type == "array"))) | map({url, status: (if (.status | type) == "number" then .status else null end), - detail: ((.detail // "") | tostring | gsub("[\\r\\n`]+"; " ") | .[:200]), locations: ([.locations[] | select(type == "object" and (.path | type == "string" and test("^[^`\\r\\n]{1,300}$"))) | {path, line: (if (.line | type) == "number" then .line else null end)}][:20])}) - | map(select(.locations | length > 0)) - | .[:$max]' "$DEAD_LINKS")"; then + | map(select(.locations | length > 0))' "$DEAD_LINKS")"; then echo "::error::the dead-links artifact is not a JSON array" exit 1 fi total="$(jq 'length' "$DEAD_LINKS")" - kept="$(jq 'length' <<<"$links")" - if ((kept < total)); then - echo "::warning::kept $kept of $total dead links (malformed entries dropped, at most $MAX_LINKS kept)" - fi - if ((kept == 0)); then - echo "no dead links to file" - exit 0 + count="$(jq 'length' <<<"$valid")" + if ((count < total)); then + echo "::warning::dropped $((total - count)) of $total malformed dead-link entries" fi + links="$(jq -c --argjson max "$MAX_LINKS" '.[:$max]' <<<"$valid")" + more=$((count > MAX_LINKS ? count - MAX_LINKS : 0)) delim="EOF_$(openssl rand -hex 16)" { + echo "found=true" + echo "more=$more" echo "links<<$delim" echo "$links" echo "$delim" } >>"$GITHUB_OUTPUT" - echo "handed $kept dead links to the file job" + echo "handed $((count - more)) dead links to the file job ($more more over the cap)" file: - name: File the dead-link issues + name: Update the dead-link rollup issue needs: [read] - if: needs.read.outputs.links != '' + if: needs.read.outputs.found == 'true' runs-on: ubuntu-24.04 timeout-minutes: 10 permissions: - issues: write # read, file, update and reopen the dead-link records + issues: write # read, open, rewrite, comment on, close and reopen the rollup # Two link-check runs finishing close together must not both find no - # record and file two. + # rollup and open two. concurrency: group: maintenance-file-dead-links cancel-in-progress: false steps: - - name: File or update one issue per dead link + - name: Rewrite the rollup issue env: GH_TOKEN: ${{ github.token }} LINKS: ${{ needs.read.outputs.links }} + MORE: ${{ needs.read.outputs.more }} RUN_URL: ${{ github.event.workflow_run.html_url }} - MAX_NEW: 10 - PACE_SECONDS: 3 + MARKER: DEMOTION_PATH: https://github.com/melodic-software/claude-code-plugins/blob/main/plugins/attribution/skills/audit/reference/dispositions.md#the-demotion-path-when-a-pointer-later-dies run: | set -euo pipefail - # Every record github-actions[bot] filed, by URL hash; an open one - # wins over a closed one. A failed read files nothing, since a - # weekly duplicate per link is worse than a week's delay. + [[ "$MORE" =~ ^[0-9]+$ ]] || MORE=0 + # The rollups github-actions[bot] filed: an open one wins, else the + # newest. A failed read writes nothing, so it never opens a second. # shellcheck disable=SC2016 # jq string interpolation, not shell if ! listing="$(gh api --paginate "repos/$GITHUB_REPOSITORY/issues?state=all&creator=github-actions%5Bbot%5D&per_page=100" \ --jq '.[] | select(.pull_request == null) - | ((.body // "") | sub("^\\s+"; "") | split("\n")[0] - | capture("^$").h) as $h - | "\($h) \(.number) \(.state) \(.state_reason // "none")"')"; then - echo "::error::could not read the dead-link records: nothing filed" + | select((.body // "") | sub("^\\s+"; "") | split("\n")[0] | rtrimstr("\r") == env.MARKER) + | "\(.number) \(.state) \(.state_reason // "none")"')"; then + echo "::error::could not read the dead-link rollup issues: nothing written" exit 1 fi - declare -A records=() - while read -r h number state reason; do - [[ -n "$h" ]] || continue - if [[ -z "${records[$h]:-}" || "$state" == open ]]; then - records[$h]="$number $state $reason" + number='' state='' reason='' + while read -r n s r; do + [[ "$n" =~ ^[0-9]+$ ]] || continue + if [[ -z "$number" || ( "$s" == open && "$state" != open ) || ( "$state" != open && "$s" != open && n -gt number ) ]]; then + number="$n" state="$s" reason="$r" fi done <<<"$listing" + count="$(jq 'length' <<<"$LINKS")" - filed=0 updated=0 reopened=0 left=0 failed=0 - over=() - while IFS= read -r link; do - url="$(jq -r '.url' <<<"$link")" - hash="$(printf '%s' "$url" | sha256sum | cut -d' ' -f1)" - body="$(jq -r --arg hash "$hash" --arg run "$RUN_URL" --arg demote "$DEMOTION_PATH" ' - "\n\n" - + "The weekly online link check found this link dead.\n\n" - + "- URL: `\(.url)`\n" - + "- Status: \(.status // "no status code") (the text the server sent is left out: it is written outside the org and intake reads this body)\n" - + "- Run: \($run)\n\n" - + "Where it appears:\n\n" - + ([.locations[] | "- `\(.path)`" + (if .line then " line \(.line)" else "" end)] | join("\n")) - + "\n\nRepair each pointer by [the demotion path when a pointer later dies](\($demote)): " - + "point at a live source that holds the same fact, or demote the pointer to a stamped record " - + "or an archived-snapshot citation; never re-expand it into a copy. A page that is alive but " - + "refuses the checker gets an exact-URL `.lycheeignore` entry with a dated reason.\n\n" - + "Close this only once the link check no longer reports the URL: removed, replaced, or in " - + "`.lycheeignore`. A dead URL kept in a link, the marked pointer of a demoted record included, " - + "is found again and reopens this issue on the next run.\n" - ' <<<"$link")" - record="${records[$hash]:-}" - if [[ -z "$record" ]]; then - if ((filed >= MAX_NEW)); then - over+=("$url") - continue - fi - if gh api -X POST "repos/$GITHUB_REPOSITORY/issues" -f 'labels[]=needs-triage' -f title="Dead link: ${url:0:180}" -f body="$body" >/dev/null; then - filed=$((filed + 1)) - else - echo "::error::could not file the dead-link issue for $url" - failed=$((failed + 1)) - fi - else - read -r number state reason <<<"$record" - if [[ "$state" == closed && "$reason" == not_planned ]]; then - echo "#$number for $url was closed as not planned: left alone" - left=$((left + 1)) - continue - fi - if gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/$number" -f state=open -f body="$body" >/dev/null; then - if [[ "$state" == open ]]; then - updated=$((updated + 1)) - else - reopened=$((reopened + 1)) - fi - else - echo "::error::could not update #$number for $url" - failed=$((failed + 1)) - fi + summary() { printf '# Dead-link rollup\n\n%s\n' "$1" >>"$GITHUB_STEP_SUMMARY"; echo "$1"; } + if ((count == 0)); then + if [[ "$state" != open ]]; then + summary "No dead links, and no open rollup issue: nothing to do." + exit 0 fi - sleep "$PACE_SECONDS" - done < <(jq -c '.[]' <<<"$LINKS") + gh api -X POST "repos/$GITHUB_REPOSITORY/issues/$number/comments" \ + -f body="The link check reported no dead links in $RUN_URL: closing." >/dev/null + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/$number" -f state=closed -f state_reason=completed >/dev/null + summary "No dead links: closed the rollup #$number as completed." + exit 0 + fi + if [[ "$state" == closed && "$reason" == not_planned ]]; then + summary "$count dead links, but the rollup #$number was closed as not planned: left alone. Reopen it to resume the list." + exit 0 + fi - { - echo "# Dead-link issues" - echo - echo "Filed $filed, updated $updated, reopened $reopened, left $left closed as not planned, $failed failed." - if ((${#over[@]} > 0)); then - echo - echo "Over the cap of $MAX_NEW new issues, filed next run:" - echo - # shellcheck disable=SC2016 # markdown code span, not shell - printf -- '- `%s`\n' "${over[@]}" - fi - } >>"$GITHUB_STEP_SUMMARY" - echo "filed $filed, updated $updated, reopened $reopened, left $left, failed $failed, over the cap ${#over[@]}" - if ((failed > 0)); then - exit 1 + body="$(jq -r --arg marker "$MARKER" --arg run "$RUN_URL" --arg demote "$DEMOTION_PATH" --argjson more "$MORE" ' + "\($marker)\n\n" + + "The weekly online link check found these links dead in \($run). Each run rewrites this list.\n\n" + + ([.[] | "- [ ] `\(.url)`, status \(.status // "none"): " + + ([.locations[] | "`\(.path)`" + (if .line then " line \(.line)" else "" end)] | join(", "))] + | join("\n")) + + (if $more > 0 then "\n- +\($more) more, see the run." else "" end) + + "\n\nThe status is the code alone: the text the server sent is left out, since it is written " + + "outside the org and intake reads this body.\n\n" + + "Repair each pointer by [the demotion path when a pointer later dies](\($demote)): " + + "point at a live source that holds the same fact, or demote the pointer to a stamped record " + + "or an archived-snapshot citation; never re-expand it into a copy. A page that is alive but " + + "refuses the checker gets an exact-URL `.lycheeignore` entry with a dated reason.\n\n" + + "Close this only once the link check no longer reports these URLs: removed, replaced, or in " + + "`.lycheeignore`. A run that reports none closes it. A dead URL kept in a link, the marked " + + "pointer of a demoted record included, is found again and reopens this issue on the next run.\n" + ' <<<"$LINKS")" + if [[ -z "$number" ]]; then + gh api -X POST "repos/$GITHUB_REPOSITORY/issues" -f 'labels[]=needs-triage' \ + -f title="Dead links found by the online link check" -f body="$body" >/dev/null + summary "Opened the rollup issue with $count dead links." + elif [[ "$state" == open ]]; then + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/$number" -f body="$body" >/dev/null + summary "Rewrote the rollup #$number with $count dead links." + else + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/$number" -f state=open -f body="$body" >/dev/null + summary "Reopened the rollup #$number with $count dead links." fi diff --git a/scripts/maintenance-file-dead-links.test.sh b/scripts/maintenance-file-dead-links.test.sh index e01b3e60689..561e420a222 100755 --- a/scripts/maintenance-file-dead-links.test.sh +++ b/scripts/maintenance-file-dead-links.test.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # Runs the two step bodies of .github/workflows/maintenance-file-dead-links.yml # under the shell Actions gives a step with no `shell:` (bash -e): `Check the -# dead links` against artifact fixtures, and `File or update one issue per -# dead link` against a stub `gh` that applies the step's own --jq filter to a -# fixture issue list and logs every write. Also checks the workflow's shape: -# the trigger names the link-check workflow, and each job holds one scope. +# dead links` against artifact fixtures, and `Rewrite the rollup issue` +# against a stub `gh` that applies the step's own --jq filter to a fixture +# issue list and logs every write. Also checks the workflow's shape: the +# trigger names the link-check workflow, and each job holds one scope. # test-scope: .github/workflows/maintenance-file-dead-links.yml set -uo pipefail unset GIT_DIR GIT_WORK_TREE GIT_CONFIG @@ -31,7 +31,7 @@ step_body() { on { exit }' } step_body 'Check the dead links' >"$TMP_ROOT/read.sh" -step_body 'File or update one issue per dead link' >"$TMP_ROOT/file.sh" +step_body 'Rewrite the rollup issue' >"$TMP_ROOT/file.sh" for body in read file; do if [[ ! -s "$TMP_ROOT/$body.sh" ]]; then fail "no $body step with a run: | body in $WORKFLOW" @@ -39,6 +39,8 @@ for body in read file; do exit 1 fi done +# The rollup marker as the workflow sets it. +MARKER="$(sed -n 's/^ MARKER: //p' "$WORKFLOW")" # --- Workflow shape ----------------------------------------------------------- job() { awk -v j=" $1:" '$0 == j { on = 1; next } on && /^ [a-z-]+:$/ { exit } on' "$WORKFLOW"; } @@ -70,6 +72,11 @@ if job read | grep -q 'name: link-check-online-dead-links$'; then else fail "the read job does not download link-check-online-dead-links" fi +if [[ "$MARKER" == '' ]]; then + ok "the rollup marker is the fixed link-check-dead-pointer rollup comment" +else + fail "the rollup marker is '$MARKER'" +fi # --- Check the dead links ----------------------------------------------------- run_read() { # [MAX_LINKS] @@ -80,6 +87,7 @@ run_read() { # [MAX_LINKS] READ_LOG="$(env -i PATH="$PATH" DOWNLOAD="$1" RUN_CONCLUSION="$2" DEAD_LINKS="$fixture" \ MAX_LINKS="${4:-50}" GITHUB_OUTPUT="$out" bash -e "$TMP_ROOT/read.sh" 2>&1)" READ_STATUS=$? + READ_OUT="$(cat "$out")" READ_LINKS="$(awk '/^links<"$TMP_ROOT/bin/gh" <<'STUB' #!/usr/bin/env bash @@ -168,99 +176,124 @@ fi [[ "${args[i]}" == -f ]] && printf 'F %s\n' "${args[i + 1]%%$'\n'*}" done } >>"$GH_LOG" -if [[ -n "${GH_WRITE_BODIES:-}" ]]; then - for ((i = 0; i < ${#args[@]}; i++)); do - [[ "${args[i]}" == -f && "${args[i + 1]}" == body=* ]] && printf '%s\n----\n' "${args[i + 1]#body=}" >>"$GH_WRITE_BODIES" - done -fi +for ((i = 0; i < ${#args[@]}; i++)); do + [[ "${args[i]}" == -f && "${args[i + 1]}" == body=* ]] && printf '%s\n' "${args[i + 1]#body=}" >>"$GH_BODY" +done exit "${GH_WRITE_STATUS:-0}" STUB chmod +x "$TMP_ROOT/bin/gh" -sha() { printf '%s' "$1" | sha256sum | cut -d' ' -f1; } -link() { jq -cn --arg u "$1" '{url: $u, status: 404, detail: "Not Found", locations: [{path: "docs/a.md", line: 7}]}'; } -url_open='https://example.com/open' url_done='https://example.com/done' url_wont='https://example.com/wont' -url_new1='https://example.com/new1' url_new2='https://example.com/new2' -marker() { printf '' "$(sha "$1")"; } -jq -n --arg mo "$(marker "$url_open")" --arg md "$(marker "$url_done")" --arg mw "$(marker "$url_wont")" \ - --arg mn "$(marker "$url_new1")" --arg mp "$(marker "$url_new2")" '[ - {number: 11, state: "open", state_reason: null, body: ($mo + "\n\nold")}, - {number: 12, state: "closed", state_reason: "completed", body: ($md + "\n")}, - {number: 13, state: "closed", state_reason: "not_planned", body: ($mw + "\n")}, - {number: 14, state: "open", state_reason: null, body: ("intro\n" + $mn), pull_request: null}, - {number: 15, state: "open", state_reason: null, body: $mp, pull_request: {url: "x"}} -]' >"$TMP_ROOT/issues.json" - +issues() { jq -n --arg m "$MARKER" "$1" >"$TMP_ROOT/issues.json"; } run_file() { # [VAR=value ...] local links="$1" shift : >"$TMP_ROOT/gh.log" - : >"$TMP_ROOT/bodies" + : >"$TMP_ROOT/body" : >"$TMP_ROOT/summary" FILE_LOG="$(env -i PATH="$TMP_ROOT/bin:$PATH" GH_ISSUES="$TMP_ROOT/issues.json" GH_LOG="$TMP_ROOT/gh.log" \ - GH_WRITE_BODIES="$TMP_ROOT/bodies" GITHUB_REPOSITORY=o/r GITHUB_STEP_SUMMARY="$TMP_ROOT/summary" \ - LINKS="$links" RUN_URL=https://github.com/o/r/actions/runs/1 MAX_NEW=1 PACE_SECONDS=0 \ + GH_BODY="$TMP_ROOT/body" GITHUB_REPOSITORY=o/r GITHUB_STEP_SUMMARY="$TMP_ROOT/summary" \ + LINKS="$links" MORE=0 RUN_URL=https://github.com/o/r/actions/runs/1 MARKER="$MARKER" \ DEMOTION_PATH=https://example.com/dispositions "$@" bash -e "$TMP_ROOT/file.sh" 2>&1)" FILE_STATUS=$? GH_WRITES="$(cat "$TMP_ROOT/gh.log")" + BODY="$(cat "$TMP_ROOT/body")" + SUMMARY="$(cat "$TMP_ROOT/summary")" } +two='[{"url":"https://example.com/a","status":404,"locations":[{"path":"docs/a.md","line":7},{"path":"docs/b.md","line":null}]},{"url":"https://example.com/b","status":null,"locations":[{"path":"README.md","line":2}]}]' -all="[$(link "$url_open"),$(link "$url_done"),$(link "$url_wont"),$(link "$url_new1"),$(link "$url_new2")]" -run_file "$all" -expected="PATCH repos/o/r/issues/11 -F state=open -F body=$(marker "$url_open") -PATCH repos/o/r/issues/12 -F state=open -F body=$(marker "$url_done") -POST repos/o/r/issues +# No rollup yet: a marker on a later line, or on a pull request, is not one. +issues '[{number: 4, state: "open", state_reason: null, body: ("intro\n" + $m)}, + {number: 5, state: "open", state_reason: null, body: $m, pull_request: {url: "x"}}]' +run_file "$two" +if ((FILE_STATUS == 0)) && [[ "$GH_WRITES" == "POST repos/o/r/issues F labels[]=needs-triage -F title=Dead link: $url_new1 -F body=$(marker "$url_new1")" -if ((FILE_STATUS == 0)) && [[ "$GH_WRITES" == "$expected" ]]; then - ok "an open record is updated, a completed one reopened, a not-planned one left, and new links filed needs-triage up to MAX_NEW" +F title=Dead links found by the online link check +F body=$MARKER" ]]; then + ok "with no rollup, one issue is opened, labeled needs-triage; a misplaced marker is no rollup" else - fail "the file step wrote the wrong records: status $FILE_STATUS, writes: + fail "no rollup did not open one issue: status $FILE_STATUS, writes: $GH_WRITES log: $FILE_LOG" fi -if [[ "$(cat "$TMP_ROOT/summary")" == *"Filed 1, updated 1, reopened 1, left 1 closed as not planned, 0 failed."* && - "$(cat "$TMP_ROOT/summary")" == *"- \`$url_new2\`"* ]]; then - ok "the summary counts each outcome and lists the link over the cap" +# shellcheck disable=SC2016 # literal markdown code spans +if [[ "$BODY" == *'- [ ] `https://example.com/a`, status 404: `docs/a.md` line 7, `docs/b.md`'* && + "$BODY" == *'- [ ] `https://example.com/b`, status none: `README.md` line 2'* && + "$BODY" == *"found these links dead in https://github.com/o/r/actions/runs/1"* && + "$BODY" == *"(https://example.com/dispositions)"* && + "$BODY" == *"Close this only once the link check no longer reports these URLs"* && + "$BODY" != *"more, see the run"* ]]; then + ok "the body lists each link as a task with its status and locations, the run, the demotion path and when to close" else - fail "the summary is wrong: $(cat "$TMP_ROOT/summary")" + fail "the rollup body is missing a part: $BODY" fi -first_body="$(awk '/^----$/ { exit } { print }' "$TMP_ROOT/bodies")" -# shellcheck disable=SC2016 # a literal markdown code span -if [[ "$first_body" == *"- URL: \`$url_open\`"* && "$first_body" == *'- `docs/a.md` line 7'* && - "$first_body" == *"- Run: https://github.com/o/r/actions/runs/1"* && - "$first_body" == *"(https://example.com/dispositions)"* && "$first_body" == *'- Status: 404 (the text the server sent is left out'* && - "$first_body" != *"Not Found"* && - "$first_body" == *"Close this only once the link check no longer reports the URL"* ]]; then - ok "the body names the URL, status code without the server's text, run, each location, the demotion path and when to close" + +run_file "$two" MORE=7 +if [[ "$BODY" == *$'\n- +7 more, see the run.\n'* ]]; then + ok "links over the cap get a +N more line" +else + fail "no overflow line: $BODY" +fi + +# An open rollup wins over an older closed one. +issues '[{number: 9, state: "closed", state_reason: "completed", body: $m}, + {number: 8, state: "open", state_reason: null, body: ("\n" + $m + "\n\nold")}]' +run_file "$two" +if ((FILE_STATUS == 0)) && [[ "$GH_WRITES" == "PATCH repos/o/r/issues/8 +F body=$MARKER" && "$SUMMARY" == *"Rewrote the rollup #8 with 2 dead links."* ]]; then + ok "an open rollup has its list rewritten" +else + fail "the open rollup was not rewritten: $FILE_STATUS $GH_WRITES $FILE_LOG" +fi + +run_file '[]' +if ((FILE_STATUS == 0)) && [[ "$GH_WRITES" == "POST repos/o/r/issues/8/comments +F body=The link check reported no dead links in https://github.com/o/r/actions/runs/1: closing. +PATCH repos/o/r/issues/8 +F state=closed +F state_reason=completed" ]]; then + ok "with no dead links, the open rollup gets a comment and closes as completed" +else + fail "the empty run did not close the rollup: $FILE_STATUS $GH_WRITES $FILE_LOG" +fi + +issues '[{number: 3, state: "closed", state_reason: "completed", body: $m}, + {number: 6, state: "closed", state_reason: "completed", body: $m}]' +run_file '[]' +if ((FILE_STATUS == 0)) && [[ -z "$GH_WRITES" ]]; then + ok "with no dead links and no open rollup, nothing is written" +else + fail "the empty run wrote with no open rollup: $GH_WRITES" +fi +run_file "$two" +if ((FILE_STATUS == 0)) && [[ "$GH_WRITES" == "PATCH repos/o/r/issues/6 +F state=open +F body=$MARKER" && "$SUMMARY" == *"Reopened the rollup #6"* ]]; then + ok "a rollup closed as completed, the newest, is reopened with the new list" else - fail "the issue body is missing a part: $first_body" + fail "the completed rollup was not reopened: $FILE_STATUS $GH_WRITES $FILE_LOG" fi -run_file "[$(link "$url_new1")]" MAX_NEW=5 -if [[ "$GH_WRITES" == "POST repos/o/r/issues"* ]]; then - ok "a marker that is not the first body line, or sits on a pull request, is not a record" +issues '[{number: 6, state: "closed", state_reason: "not_planned", body: $m}]' +run_file "$two" +if ((FILE_STATUS == 0)) && [[ -z "$GH_WRITES" && "$SUMMARY" == *"the rollup #6 was closed as not planned: left alone"* ]]; then + ok "a rollup closed as not planned is left alone, and the summary says so" else - fail "a misplaced marker counted as a record: $GH_WRITES" + fail "the not-planned rollup was touched: $FILE_STATUS $GH_WRITES $SUMMARY" fi -run_file "$all" GH_LIST_STATUS=1 -if ((FILE_STATUS == 1)) && [[ -z "$GH_WRITES" && "$FILE_LOG" == *"::error::could not read the dead-link records"* ]]; then - ok "a failed record read files nothing and fails" +run_file "$two" GH_LIST_STATUS=1 +if ((FILE_STATUS == 1)) && [[ -z "$GH_WRITES" && "$FILE_LOG" == *"::error::could not read the dead-link rollup issues"* ]]; then + ok "a failed rollup read writes nothing and fails" else - fail "a failed record read did not stop the filer: $FILE_STATUS $GH_WRITES $FILE_LOG" + fail "a failed rollup read did not stop the filer: $FILE_STATUS $GH_WRITES $FILE_LOG" fi -run_file "[$(link "$url_new1")]" GH_WRITE_STATUS=1 -if ((FILE_STATUS == 1)) && [[ "$FILE_LOG" == *"::error::could not file the dead-link issue for $url_new1"* ]]; then +issues '[]' +run_file "$two" GH_WRITE_STATUS=1 +if ((FILE_STATUS != 0)); then ok "a failed write fails the job" else - fail "a failed write did not fail the job: $FILE_STATUS $FILE_LOG" + fail "a failed write did not fail the job: $FILE_LOG" fi test_harness::report From 39210825fc3f3feaf3826bc45ee6085cafc21e86 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Thu, 5 Nov 2026 11:41:49 -0500 Subject: [PATCH 5/6] test(ci): disable SC2016 file-wide in the dead-link filer test Its jq programs and markdown code spans are literal single-quoted text. Refs: #9759 Co-Authored-By: Claude Opus 5.5 --- scripts/maintenance-file-dead-links.test.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/maintenance-file-dead-links.test.sh b/scripts/maintenance-file-dead-links.test.sh index 561e420a222..c93af360c5f 100755 --- a/scripts/maintenance-file-dead-links.test.sh +++ b/scripts/maintenance-file-dead-links.test.sh @@ -1,4 +1,5 @@ #!/usr/bin/env bash +# shellcheck disable=SC2016 # jq programs and markdown code spans are literal # Runs the two step bodies of .github/workflows/maintenance-file-dead-links.yml # under the shell Actions gives a step with no `shell:` (bash -e): `Check the # dead links` against artifact fixtures, and `Rewrite the rollup issue` From 1cb530c05098e76b4ca552b9014dc7684d809ee4 Mon Sep 17 00:00:00 2001 From: Kyle Sexton <153232337+kyle-sexton@users.noreply.github.com> Date: Thu, 5 Nov 2026 12:00:01 -0500 Subject: [PATCH 6/6] fix(ci): never close the dead-link rollup on bad data, and cap its size - The read job hands on an empty list only when the artifact itself is empty. An artifact whose every entry fails the shape check warns and hands on nothing, so a malformed artifact or a format change upstream never closes the rollup as fixed. - Each link keeps 5 locations and a count of the rest, shown as "+K more locations", and URLs and paths are length-capped, so 50 links stay under the 128 KiB one environment variable holds. - A rollup body over 60,000 characters moves links from the end into the "+N more, see the run" line, under the 65,536-character issue limit. - The test's two piped grep -q checks read here-strings instead. Refs: #9759 Co-Authored-By: Claude Opus 5.5 --- .../workflows/maintenance-file-dead-links.yml | 47 +++++++++++++++---- scripts/maintenance-file-dead-links.test.sh | 35 ++++++++++++-- 2 files changed, 69 insertions(+), 13 deletions(-) diff --git a/.github/workflows/maintenance-file-dead-links.yml b/.github/workflows/maintenance-file-dead-links.yml index 4d6a507241b..f84b7bfc300 100644 --- a/.github/workflows/maintenance-file-dead-links.yml +++ b/.github/workflows/maintenance-file-dead-links.yml @@ -89,17 +89,21 @@ jobs: # Each URL a plain http(s) URL, each path free of backticks and line # breaks, so both sit safely in a code span; anything else is dropped. # The server's text is never kept: it is written outside the org and - # intake reads the issue body. + # intake reads the issue body. Each link keeps 5 locations and a count + # of the rest, and URLs and paths are length-capped, so 50 links stay + # well under the 128 KiB a single environment variable can hold when + # the file job reads them as LINKS. if ! valid="$(jq -c ' if type != "array" then error("not an array") else . end - | map(select((.url | type == "string" and length <= 2000 and test("^https?://[^\\s`<>]+$")) + | map(select((.url | type == "string" and length <= 1000 and test("^https?://[^\\s`<>]+$")) and (.locations | type == "array"))) | map({url, status: (if (.status | type) == "number" then .status else null end), - locations: ([.locations[] - | select(type == "object" and (.path | type == "string" and test("^[^`\\r\\n]{1,300}$"))) - | {path, line: (if (.line | type) == "number" then .line else null end)}][:20])}) - | map(select(.locations | length > 0))' "$DEAD_LINKS")"; then + all: [.locations[] + | select(type == "object" and (.path | type == "string" and test("^[^`\\r\\n]{1,200}$"))) + | {path, line: (if (.line | type) == "number" then .line else null end)}]}) + | map(select(.all | length > 0) + | {url, status, locations: .all[:5], more_locations: ((.all | length) - 5 | if . < 0 then 0 else . end)})' "$DEAD_LINKS")"; then echo "::error::the dead-links artifact is not a JSON array" exit 1 fi @@ -108,6 +112,14 @@ jobs: if ((count < total)); then echo "::warning::dropped $((total - count)) of $total malformed dead-link entries" fi + # Only an artifact that is itself empty means no dead links. One whose + # every entry failed the shape check (a malformed artifact or a format + # change upstream) hands on nothing, so the rollup is never closed as + # fixed on bad data. + if ((total > 0 && count == 0)); then + echo "::warning::every dead-link entry failed the shape check: the rollup is left as it is" + exit 0 + fi links="$(jq -c --argjson max "$MAX_LINKS" '.[:$max]' <<<"$valid")" more=$((count > MAX_LINKS ? count - MAX_LINKS : 0)) delim="EOF_$(openssl rand -hex 16)" @@ -181,11 +193,17 @@ jobs: exit 0 fi - body="$(jq -r --arg marker "$MARKER" --arg run "$RUN_URL" --arg demote "$DEMOTION_PATH" --argjson more "$MORE" ' + # The body with the first $1 links, the rest counted in the +N more + # line. Each item shows at most 5 locations. + render() { + jq -r --arg marker "$MARKER" --arg run "$RUN_URL" --arg demote "$DEMOTION_PATH" \ + --argjson keep "$1" --argjson more "$((MORE + count - $1))" ' "\($marker)\n\n" + "The weekly online link check found these links dead in \($run). Each run rewrites this list.\n\n" - + ([.[] | "- [ ] `\(.url)`, status \(.status // "none"): " - + ([.locations[] | "`\(.path)`" + (if .line then " line \(.line)" else "" end)] | join(", "))] + + ([.[:$keep][] | "- [ ] `\(.url)`, status \(.status // "none"): " + + ([.locations[:5][] | "`\(.path)`" + (if .line then " line \(.line)" else "" end)] | join(", ")) + + (((.more_locations // 0) + ((.locations | length) - 5 | if . < 0 then 0 else . end)) as $k + | if $k > 0 then ", +\($k) more locations" else "" end)] | join("\n")) + (if $more > 0 then "\n- +\($more) more, see the run." else "" end) + "\n\nThe status is the code alone: the text the server sent is left out, since it is written " @@ -197,7 +215,16 @@ jobs: + "Close this only once the link check no longer reports these URLs: removed, replaced, or in " + "`.lycheeignore`. A run that reports none closes it. A dead URL kept in a link, the marked " + "pointer of a demoted record included, is found again and reopens this issue on the next run.\n" - ' <<<"$LINKS")" + ' <<<"$LINKS" + } + # An issue body holds at most 65,536 characters; past 60,000, links + # move from the end of the list into the +N more line. + keep="$count" + body="$(render "$keep")" + while ((${#body} > 60000 && keep > 0)); do + keep=$((keep - 1)) + body="$(render "$keep")" + done if [[ -z "$number" ]]; then gh api -X POST "repos/$GITHUB_REPOSITORY/issues" -f 'labels[]=needs-triage' \ -f title="Dead links found by the online link check" -f body="$body" >/dev/null diff --git a/scripts/maintenance-file-dead-links.test.sh b/scripts/maintenance-file-dead-links.test.sh index c93af360c5f..a46b201f467 100755 --- a/scripts/maintenance-file-dead-links.test.sh +++ b/scripts/maintenance-file-dead-links.test.sh @@ -63,12 +63,12 @@ if [[ "$(job read | scopes)" == 'actions: read' ]]; then else fail "the read job's permissions are not exactly actions: read: '$(job read | scopes)'" fi -if [[ "$(job file | scopes)" == 'issues: write' ]] && ! job file | grep -qE 'uses:|actions/checkout'; then +if [[ "$(job file | scopes)" == 'issues: write' ]] && ! grep -qE 'uses:|actions/checkout' <<<"$(job file)"; then ok "the file job holds issues: write alone and checks out nothing" else fail "the file job's permissions are not exactly issues: write, or it uses an action" fi -if job read | grep -q 'name: link-check-online-dead-links$'; then +if grep -q 'name: link-check-online-dead-links$' <<<"$(job read)"; then ok "the read job downloads the ci-workflows link-check-online-dead-links artifact" else fail "the read job does not download link-check-online-dead-links" @@ -110,7 +110,7 @@ if [[ "$READ_LOG" == *"::warning::dropped 3 of 5 malformed dead-link entries"* ] else fail "no warning for dropped links: $READ_LOG" fi -if [[ "$(jq -c '.[0]' <<<"$READ_LINKS")" == '{"url":"https://example.com/gone","status":404,"locations":[{"path":"docs/a.md","line":3},{"path":"docs/b c.md","line":null}]}' ]]; then +if [[ "$(jq -c '.[0]' <<<"$READ_LINKS")" == '{"url":"https://example.com/gone","status":404,"locations":[{"path":"docs/a.md","line":3},{"path":"docs/b c.md","line":null}],"more_locations":0}' ]]; then ok "the server's text is dropped and locations keep path and line" else fail "links are not normalized: $READ_LINKS" @@ -130,6 +130,13 @@ else fail "an empty list was not handed on: $READ_STATUS $READ_OUT" fi +run_read success failure "[$tick,$ftp,$badpath]" +if ((READ_STATUS == 0)) && [[ -z "$READ_OUT" && "$READ_LOG" == *"::warning::every dead-link entry failed the shape check"* ]]; then + ok "an artifact whose every entry is malformed hands on nothing, so the rollup is not closed" +else + fail "an all-malformed artifact handed on a list: $READ_STATUS $READ_OUT $READ_LOG" +fi + run_read failure success '' if ((READ_STATUS == 0)) && [[ -z "$READ_OUT" && "$READ_LOG" != *::warning::* ]]; then ok "a green run with no artifact (property off) hands on nothing, quietly" @@ -235,6 +242,28 @@ else fail "no overflow line: $BODY" fi +# 50 long URLs with 20 locations each on long paths: the read step keeps 5 +# locations a link and stays under the 128 KiB a single environment variable +# holds, and the file step keeps the body under the 65,536-character limit. +big="$(jq -cn '[range(53) as $i | {url: ("https://example.com/" + ("u" * 950) + "/\($i)"), status: 404, + locations: [range(20) as $j | {path: ("docs/" + ("p" * 185) + "/\($j).md"), line: $j}]}]')" +run_read success failure "$big" +if ((READ_STATUS == 0)) && [[ "$(jq -c '[length, (map(.locations | length) | unique), (map(.more_locations) | unique)]' <<<"$READ_LINKS")" == '[50,[5],[15]]' ]] && + [[ "$READ_OUT" == *$'\nmore=3\n'* ]] && ((${#READ_LINKS} < 131072)); then + ok "the read step keeps 5 locations a link, counts the rest, and hands on under 128 KiB (${#READ_LINKS} bytes)" +else + fail "the read step did not cap locations or size: status $READ_STATUS, ${#READ_LINKS} bytes, $READ_LOG" +fi +run_file "$READ_LINKS" MORE=3 +shown="$(grep -c '^- \[ \] ' <<<"$BODY")" +over="$(sed -n 's/^- +\([0-9]*\) more, see the run\.$/\1/p' <<<"$BODY")" +if ((FILE_STATUS == 0 && ${#BODY} <= 60001 && shown > 0 && shown < 50)) && [[ "$over" == "$((50 - shown + 3))" ]] && + [[ "$BODY" == *", +15 more locations"* ]] && ! grep -q '/5\.md' <<<"$BODY"; then + ok "a body over 60,000 characters moves links into the +N more line; each item shows 5 locations and counts the rest" +else + fail "the big body was not capped: status $FILE_STATUS, ${#BODY} characters, $shown shown, over '$over'" +fi + # An open rollup wins over an older closed one. issues '[{number: 9, state: "closed", state_reason: "completed", body: $m}, {number: 8, state: "open", state_reason: null, body: ("\n" + $m + "\n\nold")}]'