ci: a red nightly opens an issue, and a green one closes it (#973)

.github/scripts/nightly-report.sh

@@ -0,0 +1,175 @@
+#!/usr/bin/env bash
+# Make a red nightly FINDABLE WITHOUT KNOWING TO LOOK (#973).
+#
+# The deep gate ran red two nights and nothing surfaced it; before that, 25
+# consecutive nights. Three properties compound and each is reasonable alone: a
+# scheduled run has no pull request to be red on, the failing job is named like a
+# reporting step rather than a gate, and GitHub notifies the *actor* of a
+# schedule event -- whoever last touched the workflow file, not whoever broke it.
+#
+# So this puts the verdict where somebody already looks: one issue, opened on the
+# first red, updated on every red after it, and CLOSED BY THE NEXT GREEN.
+#
+# ONE ISSUE, NOT ONE PER RUN. A notifier that files a new issue nightly is a
+# notifier people filter, and a filtered notifier is the state this replaces.
+# The issue is found by its exact title, which is why the title carries no run
+# number, no date and no job name -- everything variable goes in the body.
+#
+# IT CLOSES ITSELF. An alert that must be closed by hand becomes an alert nobody
+# closes, and then a stale one nobody believes. The green path is as load-bearing
+# as the red path and has its own arm.
+#
+# DRY RUN IS THE TESTED PATH, not a debugging aid: PGC_NIGHTLY_REPORT_DRYRUN=1
+# prints the verdict, the title and the body to stdout and calls no `gh`. The
+# selftest drives exactly this, so what the arms read is what CI composes.
+set -euo pipefail
+
+TITLE="${PGC_NIGHTLY_REPORT_TITLE:-nightly deep gate is red}"
+
+# THE VERDICT IS DERIVED HERE, FROM THE `needs` CONTEXT, AND THAT IS THE POINT.
+#
+# The first version computed it in the workflow from four named environment
+# variables, which made TWO lists: `needs: [...]`, guarded by a set-equality arm
+# in selftest 450, and the env block, guarded by nothing. A contributor adding a
+# gate was COMPELLED by the arm to update the guarded list and told nothing about
+# the one the verdict actually came from -- so the new gate could burn while this
+# reported green. #973 reintroduced inside the fix for #973, found by
+# @OffgridwithJD, who added a fifth job to a copy of the branch and got a fully
+# green tree with a nightly whose failure the reporter could not see.
+#
+# `toJSON(needs)` carries every entry, so there is one list and nothing to keep in
+# agreement -- which beats a guard asserting that two lists agree.
+#
+# AND IT MAKES THE DECISION TESTABLE. The workflow can only be read; this can be
+# DRIVEN, so selftest 450 feeds it tuples and the fire drill exercises the real
+# decision path instead of overriding the answer.
+verdict_from_needs() {	# verdict_from_needs <json> -> failure|success
+	local json="$1" results r
+	# `skipped` is not a failure: a fork run, and a fire drill, skip every gate.
+	results="$(printf '%s' "$json" | python3 -c '
+import json, sys
+d = json.load(sys.stdin)
+if not isinstance(d, dict) or not d:
+    sys.exit(3)
+for v in d.values():
+    print((v or {}).get("result", "missing"))
+' 2>/dev/null)" || return 3
+	[ -n "$results" ] || return 3
+	for r in $results; do
+		case "$r" in
+			failure|cancelled|timed_out) echo failure; return 0 ;;
+			success|skipped) ;;
+			# A result this does not know is NOT a pass. It means the platform
+			# grew a state while this did not, and treating it as green restores
+			# the silence #973 is about.
+			*) echo failure; return 0 ;;
+		esac
+	done
+	echo success
+}
+
+if [ "${1:-}" = "--from-needs" ]; then
+	# AN EMPTY OR UNPARSEABLE CONTEXT IS REFUSED, not defaulted. `needs` empty
+	# means the job list changed shape; reporting green on that is the failure
+	# mode this whole file exists to remove.
+	if ! verdict="$(verdict_from_needs "${2:?usage: --from-needs <toJSON(needs)>}")"; then
+		echo "nightly-report: the needs context was empty or unparseable" >&2
+		exit 3
+	fi
+	shift 2
+else
+	verdict="${1:?usage: nightly-report.sh <failure|success|--from-needs JSON> [job ...]}"
+	shift || true
+fi
+failed=("$@")
+
+run_url="${PGC_NIGHTLY_RUN_URL:-${GITHUB_SERVER_URL:-https://github.com}/${GITHUB_REPOSITORY:-}/actions/runs/${GITHUB_RUN_ID:-}}"
+
+case "$verdict" in
+	failure|success) ;;
+	*)
+		# NOT a silent default. A verdict this script does not understand means the
+		# caller changed and this did not; treating it as success would restore the
+		# exact silence the issue is about.
+		echo "nightly-report: verdict [$verdict] is neither failure nor success" >&2
+		exit 2
+		;;
+esac
+
+body_failure() {
+	printf '%s\n' "The nightly deep gate failed."
+	printf '\n'
+	printf 'Run: %s\n' "$run_url"
+	printf '\n'
+	if [ "${#failed[@]}" -gt 0 ]; then
+		printf 'Jobs that did not succeed:\n\n'
+		printf '  - %s\n' "${failed[@]}"
+	else
+		# The caller reported a failure and named no job. Say so rather than
+		# printing an empty list, which reads as "nothing failed".
+		printf 'No job name was reported with this failure, which is itself worth looking at.\n'
+	fi
+	printf '\n'
+	printf 'This issue is opened by the nightly on its first red, updated on each red\n'
+	printf 'after it, and closed automatically by the next green run.\n'
+}
+
+body_success() {
+	printf '%s\n' "The nightly deep gate is green again."
+	printf '\n'
+	printf 'Run: %s\n' "$run_url"
+	printf '\n'
+	printf 'Closing automatically. An alert that must be closed by hand becomes one\n'
+	printf 'nobody closes, and then a stale one nobody believes.\n'
+}
+
+if [ "${PGC_NIGHTLY_REPORT_DRYRUN:-0}" = 1 ]; then
+	printf 'verdict: %s\n' "$verdict"
+	printf 'title: %s\n' "$TITLE"
+	printf -- '--- body ---\n'
+	if [ "$verdict" = failure ]; then body_failure; else body_success; fi
+	exit 0
+fi
+
+# EXACT TITLE MATCH, not a search-relevance match. `gh issue list --search` is a
+# full-text query and would find any issue mentioning these words -- including
+# the ones the two of us have filed ABOUT this mechanism.
+# A HIGH LIMIT ON THE PLAIN LIST, NOT A SEARCH, AND BOTH HALVES OF THAT WERE
+# MEASURED RATHER THAN CHOSEN.
+#
+# `--limit 100` alone silently misses the target once more than a hundred issues
+# are open: the lookup returns empty and every red opens a NEW issue, which is
+# precisely "a notifier people filter" -- the thing this design rests on not being
+# (@OffgridwithJD).
+#
+# `--search` fixes that and introduces a worse one. GitHub's search index is
+# EVENTUALLY CONSISTENT: measured here, immediately after closing an issue the
+# search still reported it open, and caught up seconds later. A lag in the other
+# direction misses a freshly-opened issue and opens a duplicate -- the same
+# failure, arriving by a different route, and this mechanism can run twice in
+# quick succession because `concurrency: nightly` queues rather than cancels.
+#
+# `gh issue list` without `--search` reads the REST collection, which is strongly
+# consistent, and gh paginates past 100 on its own. So a high limit has neither
+# problem. The exact match stays client-side: `--search` would also have matched
+# the issues the two of us have filed ABOUT this mechanism.
+existing="$(gh issue list --state open --limit 1000 --json number,title \
+	--jq "map(select(.title == \"$TITLE\")) | .[0].number // empty")"
+
+if [ "$verdict" = failure ]; then
+	if [ -n "$existing" ]; then
+		gh issue comment "$existing" --body "$(body_failure)"
+		echo "nightly-report: commented on #$existing"
+	else
+		gh issue create --title "$TITLE" --body "$(body_failure)"
+		echo "nightly-report: opened a new issue"
+	fi
+else
+	if [ -n "$existing" ]; then
+		gh issue comment "$existing" --body "$(body_success)"
+		gh issue close "$existing" --reason completed
+		echo "nightly-report: closed #$existing"
+	else
+		echo "nightly-report: green, and nothing open to close"
+	fi
+fi

.github/workflows/nightly.yml

@@ -32,6 +32,11 @@ on:
   schedule:
     - cron: "0 6 * * *" # nightly, 06:00 UTC
   workflow_dispatch:
+    inputs:
+      fire_drill:
+        description: "Exercise the red-nightly reporter without breaking anything"
+        type: boolean
+        default: false
 
 permissions:
   contents: read
@@ -47,7 +52,12 @@ jobs:
     name: suites (PG ${{ matrix.pg }}, ${{ matrix.runner == 'ubuntu-24.04-arm' && 'aarch64' || 'x86_64' }})
     runs-on: ${{ matrix.runner }}
     timeout-minutes: 60
-    if: github.event_name == 'workflow_dispatch' || github.repository == 'commandprompt/pgcolumnar'
+    # A FIRE DRILL SKIPS THE HEAVY GATES. The drill exists to prove the reporter
+    # fires, and a proof that costs a full deep gate is one nobody re-runs -- which
+    # is how a notifier goes back to never having been seen to fire. `skipped` is
+    # not a failure to the reporter, so the drill's verdict comes from the input.
+    if: (github.event_name == 'workflow_dispatch' || github.repository == 'commandprompt/pgcolumnar')
+      && github.event.inputs.fire_drill != 'true'
     strategy:
       fail-fast: false
       # The full packaged matrix on x86_64, and the current major on aarch64.
@@ -202,7 +212,12 @@ jobs:
     name: sanitizer gate (ASAN+UBSAN)
     runs-on: ubuntu-latest
     timeout-minutes: 120
-    if: github.event_name == 'workflow_dispatch' || github.repository == 'commandprompt/pgcolumnar'
+    # A FIRE DRILL SKIPS THE HEAVY GATES. The drill exists to prove the reporter
+    # fires, and a proof that costs a full deep gate is one nobody re-runs -- which
+    # is how a notifier goes back to never having been seen to fire. `skipped` is
+    # not a failure to the reporter, so the drill's verdict comes from the input.
+    if: (github.event_name == 'workflow_dispatch' || github.repository == 'commandprompt/pgcolumnar')
+      && github.event.inputs.fire_drill != 'true'
     env:
       PG_VERSION: "18.4"
       SAN_PREFIX: /home/runner/pg_san
@@ -329,7 +344,12 @@ jobs:
     name: coverage report (PG 18)
     runs-on: ubuntu-latest
     timeout-minutes: 90
-    if: github.event_name == 'workflow_dispatch' || github.repository == 'commandprompt/pgcolumnar'
+    # A FIRE DRILL SKIPS THE HEAVY GATES. The drill exists to prove the reporter
+    # fires, and a proof that costs a full deep gate is one nobody re-runs -- which
+    # is how a notifier goes back to never having been seen to fire. `skipped` is
+    # not a failure to the reporter, so the drill's verdict comes from the input.
+    if: (github.event_name == 'workflow_dispatch' || github.repository == 'commandprompt/pgcolumnar')
+      && github.event.inputs.fire_drill != 'true'
     steps:
       - uses: actions/checkout@v4
 
@@ -488,7 +508,12 @@ jobs:
     name: extension upgrade guard (PG 18)
     runs-on: ubuntu-latest
     timeout-minutes: 45
-    if: github.event_name == 'workflow_dispatch' || github.repository == 'commandprompt/pgcolumnar'
+    # A FIRE DRILL SKIPS THE HEAVY GATES. The drill exists to prove the reporter
+    # fires, and a proof that costs a full deep gate is one nobody re-runs -- which
+    # is how a notifier goes back to never having been seen to fire. `skipped` is
+    # not a failure to the reporter, so the drill's verdict comes from the input.
+    if: (github.event_name == 'workflow_dispatch' || github.repository == 'commandprompt/pgcolumnar')
+      && github.event.inputs.fire_drill != 'true'
     # Named ONCE. Two steps consume it, and a tag spelled separately in each is
     # the kind of duplication that goes stale in one place and not the other.
     env:
@@ -575,3 +600,84 @@ jobs:
             echo "::error::the guard skipped for want of an old source; in this job that is a failure"
           fi
           exit "$rc"
+
+  # ---- make a red nightly findable without knowing to look (#973) -----------
+  #
+  # The deep gate ran red two nights and nothing surfaced it; before that, 25
+  # consecutive nights. A scheduled run has no pull request to be red on, the job
+  # that failed is named like a reporting step rather than a gate, and GitHub
+  # notifies the ACTOR of a schedule event -- whoever last touched this file, not
+  # whoever broke it.
+  #
+  # `if: always()` and not `if: failure()`, because the GREEN path is half the
+  # mechanism: an alert that must be closed by hand becomes one nobody closes.
+  #
+  # THE VERDICT COMES FROM needs.*.result, WHICH CANNOT BE LATE. The failing job
+  # NAMES come from the run's own API, because `needs` keys say `coverage` where a
+  # reader needs `coverage report (PG 18)` -- and if that call fails the verdict
+  # still stands and the body says no name was reported, rather than reporting a
+  # green.
+  red-nightly:
+    name: report a red nightly
+    runs-on: ubuntu-latest
+    needs: [suites, sanitizer, coverage, upgrade-guard]
+    if: always() && (github.event_name == 'workflow_dispatch' || github.repository == 'commandprompt/pgcolumnar')
+    permissions:
+      contents: read
+      issues: write
+    steps:
+      - uses: actions/checkout@v4
+      - name: open, update or close the nightly's issue
+        env:
+          GH_TOKEN: ${{ github.token }}
+          # ONE LIST, NOT TWO. An earlier version named the four results in four
+          # environment variables, which meant `needs:` was guarded by selftest 450
+          # and the verdict was computed from a second list guarded by nothing --
+          # so a contributor adding a gate was compelled to update the guarded list
+          # and told nothing about the one that decided the verdict
+          # (@OffgridwithJD, who added a fifth job and got a fully green tree with
+          # a nightly whose failure this could not see).
+          #
+          # toJSON(needs) carries every entry, so there is nothing to keep in
+          # agreement.
+          NEEDS_JSON: ${{ toJSON(needs) }}
+          DRILL: ${{ github.event.inputs.fire_drill }}
+        run: |
+          set -euo pipefail
+          # THE DRILL SUBSTITUTES THE INPUT, NOT THE ANSWER. An earlier version set
+          # verdict=failure directly, which meant the drill proved delivery and
+          # never touched the decision -- and the decision is the half that can
+          # fail the #973 way. It now feeds a synthetic context through the same
+          # derivation every real night uses.
+          # THE REAL CONTEXT IS PRINTED FIRST, ALWAYS, INCLUDING ON A DRILL.
+          # The whole fix rests on toJSON(needs) carrying every entry, and that was
+          # an assumption about the platform rather than something anyone had seen.
+          # Printing it makes every run -- including the cheap drill -- an
+          # observation of it: a drill shows all four gates as `skipped`, so a gate
+          # missing from this line is visible without waiting for a real red.
+          echo "needs context as given: $NEEDS_JSON"
+          needs_json="$NEEDS_JSON"
+          if [ "${DRILL:-false}" = "true" ]; then
+            needs_json='{"fire-drill":{"result":"failure"}}'
+            echo "fire drill: substituting a failing context through the same derivation"
+            # AND THE ISSUE MUST SAY IT IS A DRILL. Drill 3 did not: rewriting this
+            # step dropped the marker, the job list came back empty, and the body
+            # read exactly like a genuine red. An alert that cries wolf teaches a
+            # reader to discount it, which is the state #973 exists to leave.
+            drill_name="fire drill for #973: no gate actually failed"
+          fi
+          echo "verdict computed from: $needs_json"
+
+          names=()
+          while IFS= read -r n; do [ -n "$n" ] && names+=("$n"); done < <(
+            # SELECT WHAT FAILED, NOT WHAT DID NOT SUCCEED. The first version
+            # filtered negatively and the first fire drill caught it: this job is
+            # still RUNNING when it asks, so it did not match "success" and listed
+            # ITSELF as a failing job. A positive list cannot admit a state nobody
+            # enumerated -- and where it OMITS one (neutral, action_required,
+            # stale), the reporter says no job name was reported rather than
+            # printing an empty list.
+            gh run view "$GITHUB_RUN_ID" --json jobs --jq '.jobs[] | select(.conclusion == "failure" or .conclusion == "cancelled" or .conclusion == "timed_out") | .name' 2>/dev/null || true)
+
+          [ -n "${drill_name:-}" ] && names+=("$drill_name")
+          .github/scripts/nightly-report.sh --from-needs "$needs_json" ${names+"${names[@]}"}

docs/testing.md

@@ -348,6 +348,10 @@ instrumented PostgreSQL, and the coverage report. It also runs the
 extension-upgrade guard. That guard builds the previous release on PostgreSQL
 18, loads data into it, and upgrades it in place. The sanitizer build stays in a cache, because it takes longer to build
 than to run the suites against it.
+A final job, red-nightly, opens an issue when any of those fail. The next green
+run closes it. A scheduled run has no pull request to be red on, so nothing else
+surfaces a failure here. This gate once ran red for 25 consecutive nights behind
+a green per-PR gate.
 
 The aarch64 run executes the suites rather than only building them. Misaligned
 reads, the class most often expected to differ by architecture, are already