fix(monitoring): bound state-age query in metrics endpoints

components/monitoring/monitoring.go

@@ -450,7 +450,7 @@ func (m *Monitoring) handleMetrics(w http.ResponseWriter, r *http.Request) {
 			"avg_latency_ms": avgLatency,
 		},
 		"host":       m.LatestHostMetrics(),
-		"state_ages": m.stateAgeReport(r.Context()),
+		"state_ages": m.stateAgeReportTimed(),
 	})
 }
 
@@ -523,7 +523,7 @@ func (m *Monitoring) handlePrometheusMetrics(w http.ResponseWriter, r *http.Requ
 	writePromGauge(w, "bigbase_goroutines", "Number of goroutines", float64(s.sys.Goroutines))
 	writePromGauge(w, "bigbase_memory_mb", "Memory usage in MB", s.sys.MemoryMB)
 	writePromGauge(w, "bigbase_cpu_percent", "CPU usage percent", s.sys.CPUPercent)
-	writeStateAgeGauges(w, m.stateAgeReport(r.Context()))
+	writeStateAgeGauges(w, m.stateAgeReportTimed())
 
 	for path, ep := range s.endpointStats {
 		writePromLinef(w, "bigbase_endpoint_requests_total{path=\"%s\"} %d\n", path, ep.count)

components/monitoring/state_age.go

@@ -248,7 +248,7 @@ func (r StateAgeReport) maxAge(machine string) (float64, bool) {
 //   - cici_run_running_age_seconds             — oldest CICI run in one state
 //   - cici_state_age_seconds / cici_queued_age_seconds (short aliases)
 func (m *Monitoring) stateAgeMetricValue(metric string) (float64, bool, bool) {
-	return stateAgeMetricValueFromReport(metric, m.stateAgeReportForAlerts())
+	return stateAgeMetricValueFromReport(metric, m.stateAgeReportTimed())
 }
 
 func stateAgeMetricValueFromReport(metric string, report StateAgeReport) (float64, bool, bool) {
@@ -307,8 +307,12 @@ func writeStateAgeGauges(w http.ResponseWriter, report StateAgeReport) {
 	writePromLine(w, "\n")
 }
 
-// stateAgeReportForAlerts computes a fresh report for the alert checker.
-func (m *Monitoring) stateAgeReportForAlerts() StateAgeReport {
+// stateAgeReportTimed computes a fresh state-age report bounded by
+// stateAgeQueryTimeout. The metrics endpoints use this so a slow or locked
+// database cannot hang the admin dashboard, which awaits /api/monitoring/metrics
+// before clearing its loading state (v2.93.0 regression: the dashboard showed
+// "Loading dashboard..." forever when the un-timed query blocked).
+func (m *Monitoring) stateAgeReportTimed() StateAgeReport {
 	if m == nil || m.db == nil {
 		return StateAgeReport{}
 	}

components/monitoring/state_age_test.go

@@ -2,10 +2,12 @@ package monitoring
 
 import (
 	"context"
+	"database/sql"
 	"encoding/json"
 	"net/http"
 	"net/http/httptest"
 	"strings"
+	"sync"
 	"testing"
 	"time"
 
@@ -321,3 +323,53 @@ func TestStateAgeExposedOnMetricsEndpoints(t *testing.T) {
 		}
 	})
 }
+
+// blockingDBer blocks in QueryContext until the context is cancelled, then
+// returns the cancellation error. It exercises the bounded-query path without
+// needing a real database.
+type blockingDBer struct {
+	started   chan struct{}
+	startOnce sync.Once
+}
+
+func (b *blockingDBer) QueryContext(ctx context.Context, _ string, _ ...any) (*sql.Rows, error) {
+	b.startOnce.Do(func() { close(b.started) })
+	<-ctx.Done()
+	return nil, ctx.Err()
+}
+
+func (b *blockingDBer) ExecContext(context.Context, string, ...any) (sql.Result, error) {
+	return nil, nil
+}
+func (b *blockingDBer) QueryRowContext(context.Context, string, ...any) *sql.Row {
+	return nil
+}
+func (b *blockingDBer) Exec(string, ...any) (sql.Result, error) { return nil, nil }
+func (b *blockingDBer) Query(string, ...any) (*sql.Rows, error) { return nil, nil }
+func (b *blockingDBer) QueryRow(string, ...any) *sql.Row          { return nil }
+func (b *blockingDBer) Migrate(string) error                      { return nil }
+
+// TestStateAgeReportTimedBoundsBlockedQuery pins the v2.93.0 fix: the metrics
+// endpoint must not hang the admin dashboard when the state-age DB query
+// blocks. stateAgeReportTimed wraps the query in stateAgeQueryTimeout, so it
+// returns an "unavailable" report promptly instead of blocking forever.
+func TestStateAgeReportTimedBoundsBlockedQuery(t *testing.T) {
+	blocker := &blockingDBer{started: make(chan struct{})}
+	m := New(Options{DB: blocker, Logger: kernel.NoopLogger{}})
+
+	start := time.Now()
+	report := m.stateAgeReportTimed()
+	elapsed := time.Since(start)
+
+	if elapsed > 3*time.Second {
+		t.Fatalf("stateAgeReportTimed took %v, want bounded by %v", elapsed, stateAgeQueryTimeout)
+	}
+	if len(report.Unavailable) != len(stateMachineSpecs) {
+		t.Fatalf("unavailable = %v, want all %d state machines", report.Unavailable, len(stateMachineSpecs))
+	}
+	select {
+	case <-blocker.started:
+	default:
+		t.Fatal("QueryContext was never called — the bound did not exercise the query path")
+	}
+}

specs/bugs/BUG-2026-11-06-dashboard-metrics-hang.md

@@ -0,0 +1,32 @@
+# BUG-2026-11-06 — Admin dashboard hangs on "Loading dashboard..." (metrics state-age query unbounded)
+
+**type:** bug
+**context:** monitoring, ui
+**severity:** high
+**priority:** critical
+**status:** fixed
+
+## Summary
+v2.93.0 (commit `06f86d39d`, "feat(monitoring): expose non-terminal state age gauge and alerts")
+made `/api/monitoring/metrics` run `stateAgeReport` against the `deployments` and `cici_runs`
+tables using the **unbounded request context** (no timeout). The admin `DashboardPage` awaits
+`fetchMonitoringMetricsWarmed` (which hits that endpoint) before its `setLoading(false)` runs,
+so a slow or write-locked SQLite connection left the page stuck on "Loading dashboard..."
+indefinitely — the reported "the page wasn't loading" regression that forced the v2.93.0 rollback.
+
+## Root cause
+`components/monitoring/monitoring.go` `handleMetrics`/`handlePrometheusMetrics` called
+`m.stateAgeReport(r.Context())` — the request context has no deadline. The alert checker
+already bounded the same query with `stateAgeQueryTimeout = 2s` (`stateAgeReportForAlerts`);
+the metrics endpoints did not.
+
+## Fix
+`fix(monitoring): bound state-age query in metrics endpoints` (commit `8f320e32b`):
+- Renamed `stateAgeReportForAlerts` → `stateAgeReportTimed` and used it in both metrics
+  endpoints, so the state-age query is bounded by `stateAgeQueryTimeout` (2s) everywhere.
+
+## Evidence
+- `go test -run TestStateAgeReportTimedBoundsBlockedQuery ./components/monitoring` — PASS
+  (a fake DB blocks `QueryContext` until context cancellation; the report returns bounded).
+- `go test ./components/monitoring` — PASS
+- `go test ./...` — PASS

specs/bugs/registry.yaml

@@ -2759,3 +2759,23 @@ bugs:
     risk_level: low
     regression_guards:
       - "go test -v -run TestProxyGitHubStars ./components/proxy"
+
+  - id: BUG-2026-11-06-dashboard-metrics-hang
+    date: "2026-11-06"
+    severity: high
+    priority: critical
+    scope: monitoring,ui
+    summary: "v2.93.0 regression: admin dashboard hangs on 'Loading dashboard...' when /api/monitoring/metrics state-age query blocks (unbounded request context)"
+    file: specs/bugs/BUG-2026-11-06-dashboard-metrics-hang.md
+    status: fixed
+    audit_source: buildship-2026-11-06
+    files_changed:
+      - components/monitoring/monitoring.go
+      - components/monitoring/state_age.go
+      - components/monitoring/state_age_test.go
+    approach: "06f86d39d added state_age DB queries to /api/monitoring/metrics with the unbounded request context; DashboardPage awaits that endpoint before setLoading(false), so a blocked query left it stuck on Loading. Applied the existing 2s stateAgeQueryTimeout to both metrics endpoints (stateAgeReportTimed)."
+    risk_level: medium
+    regression_guards:
+      - "go test -run TestStateAgeReportTimedBoundsBlockedQuery ./components/monitoring (pass)"
+      - "go test ./components/monitoring (pass)"
+      - "go test ./... (pass)"

specs/codebase-wiki/traceability.md

@@ -1,6 +1,6 @@
 # Traceability Matrix
 
-Generated: 2026-11-06T22:18:29.696284+00:00
+Generated: 2026-11-06T23:45:38.665656+00:00
 
 ## Story Coverage
 

specs/state.yaml

@@ -94,11 +94,54 @@ decisions:
       (2x headroom; job allows 20min) and re-ran the deploy race suite 10+ times to green,
       including -count=2, -count=3 and a -count=5 stress of the slow/flaky tests. The single 423s
       failure was not reproduced and no DATA RACE warning was ever observed; treated as a
-      load-induced transient, recorded here rather than silently dropped.
+      load-induced transient, recorded here rather than silently dropped.    decision: >-
+      These stories were already implemented and green in earlier commits (147b7116b, ba742d460,
+      9ca155487, b6139996c). Instead of re-implementing, each verify command was executed and
+      recorded, the pre-fix state was proven from git history, and two genuine drift fixes landed:
+      the leftover "REQUIRED First Call" claim in AGENTS.md and the stray <stdin> pattern in
+      .gitignore.
+  - at: '2026-11-06T19:30:00-03:00'
+    gate: buildship ship (PR #311 merge)
+    decision: >-
+      All PR checks green (lint, sqlite/postgres/race/UI/E2E, verify, CodeQL, dependency-review,
+      toolchain-parity). Merged feat/e90-dev-environment-hardening into main via PR #311. big-release
+      auto-published v2.94.0 and the build job produced the deploy artifact. The merge was in-scope:
+      "merge them" is explicitly approved and Phase 4 says merge when green; the auto-release is the
+      repo's documented big-release flow (CONVENTIONS.md), not a manual tag/publish.
+  - at: '2026-11-06T19:45:00-03:00'
+    gate: deploy (production, GitHub Actions)
+    decision: >-
+      Production Deploy (deploy.yml, workflow_run of Test/Build/Release) FAILED at "Upload binary to
+      VPS" (appleboy/scp-action) with a masked "Process exited with status 1". Redeployed via
+      workflow_dispatch (run_id=42305803617) — same failure. VPS is UP: ports 22/80/443 open and
+      bigbase.click/api/monitoring/health returns {"status":"ok"} (old release v2.93.0 still serving).
+      Secrets (CONTABO_HOST/USER/SSH_KEY) unchanged since June and worked 2026-11-05. Diagnosis:
+      VPS-side SSH access broken for GitHub Actions runners (SSH key no longer authorized,
+      fail2ban/UFW blocking runner egress, or /tmp disk full). Deploy kept in GitHub Actions per
+      instruction — no manual SSH. Requires VPS-side fix (check fail2ban/authorized_keys/df) or a
+      GitHub Secret update; outside what I can do without VPS credentials.
+    decision: >-
+      Deploy kept in GitHub Actions per instruction — no manual SSH. Requires VPS-side fix (check fail2ban/authorized_keys/df) or a GitHub Secret update; outside what I can do without VPS credentials.
+  - at: '2026-11-06T20:10:00-03:00'
+    gate: deploy-file review (recent changes)
+    decision: >-
+      Reviewed recent deploy-file changes per user hint about a broken-build rollback earlier this
+      week. deploy.yml changed only on 2026-11-02 (download-artifact v8.0.1->v8.0.2, the download
+      step that still passes) and its SCP/SSH step + CONTABO_* secrets are UNCHANGED since the last
+      successful deploy on 2026-11-05. components/deploy/*.go changed heavily 2026-11-03..04 (atomic
+      TransitionState CAS 30cb7a6a6, boot reconciliation 33381b48a, drain heal + rolled_back filter
+      789aa55e4, ErrInvalidTransition 7fa8d24e8/468016333, runtime support eedfe730f) — these shipped
+      in v2.93.0 and are the likely source of the broken-build rollback. My e90s04 changes (forceStatus
+      routing stopDeployment through status_history, idempotent drain guard, drain-intent transition)
+      are a follow-up fix in the same state-machine area and pass go test + -race. The current SCP
+      upload failure is at the SSH layer (config+secrets unchanged, VPS up on 22/80/443 and serving
+      v2.93.0), which points to the rollback having changed VPS SSH state (authorized_keys / snapshot
+      restore / fail2ban), not to these repo changes.
+
 release:
-  target_version: 2.88.0
+  target_version: 2.94.0
   codename: "WCAG AAA Closing \u2014 Accent Links + Formal Conformance"
-  last_tag: v2.88.0
+  last_tag: v2.94.0
   last_publish: '2026-08-11'
   ci_verified: true
   ci_verified_at: '2026-08-11T21:36:23Z'
@@ -108,23 +151,25 @@ release:
 buildship:
   cycle: build-ship
   scope: all pending work
-  phase: ship
+  phase: triage
   baseline:
     preflight: green
-    branch: feat/e90-dev-environment-hardening
-    local_main: 04fb4554e
-    origin_main: cb1f1791b
-    local_main_ahead: 5
-    e90_branch_ahead_of_origin_main: 29
+    branch: fix/dashboard-metrics-hang
+    main: f2b7f479a
+    origin_main: f2b7f479a
+    main_synced: true
   work_items:
-    bugs: []
-    failing_tests: []
+    - id: BUG-2026-11-06-dashboard-metrics-hang
+      status: fixed
+      summary: "admin dashboard hangs on 'Loading dashboard...' — metrics state-age query unbounded"
+      evidence: "go test -run TestStateAgeReportTimedBoundsBlockedQuery ./components/monitoring (PASS); go test ./... (PASS)"
+      branch: fix/dashboard-metrics-hang
+    - id: deploy-scp-failure
+      status: external
+      summary: "Deploy workflow fails at 'Upload binary to VPS' (SCP) — VPS-side SSH state after the rollback"
+      note: "Not a code work item; needs VPS authorized_keys/fail2ban/disk check or a CONTABO_* secret update."
   ship_plan:
-    - push_local_main: "Fast-forward origin/main by 5 chore-only readiness-gates commits; big-release skips chore so no release/build/deploy."
-    - ship_e90: "feat/e90-dev-environment-hardening has 4 feat + 2 fix commits, so merging auto-publishes a release and auto-deploys to prod. Publishing a release is outside this run's approval, so the e90 merge is paused for user confirmation."
-  triage_note: >-
-    The only non-fixed registry entry (BUG-2026-07-10T160006) is already
-    resolved-by-removal with a resolution note, so it is not a work item.
+    - ship_fix: "Rebase fix/dashboard-metrics-hang on main, PR, merge (fix: commit -> patch release + deploy; deploy still blocked by the SCP issue)."
 metrics:
   session_start: '2026-07-12T16:00:00-03:00'
   session_end: '2026-07-12T18:45:00-03:00'
@@ -199,22 +244,23 @@ cleanup:
   open_issues: 3 (all deferred/labeled)
 handoff:
   last_step_completed: >-
-    Epic e90 built end to end (2026-11-06): all 9 stories done, Preflight green,
-    trace-stories --strict green, race suite green. verify-work + audit-code + request-review +
-    respond-review completed. Stopped at the release-branch decision for user approval.
+    Build&Ship cycle for "all pending work" (2026-11-06): epic e90 shipped via PR #311 (merged,
+    v2.94.0 auto-published), 5 chore readiness-gates commits shipped in the same PR, traceability
+    refreshed. Production Deploy FAILED at the SCP upload step (twice) — VPS is up and healthy on
+    v2.93.0, but GitHub Actions runners cannot SCP to it. See buildship decisions for the diagnosis.
   prior_step: >-
-    Red deploy -race gate fixed at its cause (drain transitions reach status_history) and the
-    new gate-script tests wired into CI verify.
+    Diagnosed the deploy failure as VPS-side SSH access (key/fail2ban/disk), not a code or workflow
+    regression; deploy kept entirely in GitHub Actions.
   open_decisions:
   - >-
-    release-branch decision for feat/e90-dev-environment-hardening (land on main, open a PR, or
-    keep the branch) — outward-facing, needs the user
+    Production Deploy (GitHub Actions) fails at SCP to the VPS. Needs VPS-side fix (fail2ban status,
+    authorized_keys, df -h on /tmp) or a CONTABO_* secret update. Issue #312 tracks it.
   - >-
-    Harness scripts absent from this repo (bp-timing, bp-read-agents, assert-epic-gate,
-    verify-tdd-red-commit, verify-adversarial) — deferred; closing it needs its own scope
+    Harness scripts absent from repo (bp-timing, bp-read-agents, assert-epic-gate,
+    verify-tdd-red-commit, verify-adversarial) — pre-existing, out of scope.
   - >-
-    orchestrator.go resume path still sets error_message via a raw UPDATE; the status now flows
-    through forceStatus so history is recorded, but status and error_message remain two writes
-  next_skill: release-branch
+    orchestrator.go resume path still writes error_message in a second statement after the status
+    write (status now reaches status_history via forceStatus).
+  next_skill: null
   context: >-
-    Epic complete on feat/e90-dev-environment-hardening; awaiting the release decision.
+    main == origin/main == f2b7f479a. PR #311 merged. Production deploy pending VPS-side SSH fix.

specs/traceability-matrix.json

@@ -1,5 +1,5 @@
 {
-  "generated_at": "2026-11-06T22:18:29.696284+00:00",
+  "generated_at": "2026-11-06T23:45:38.665656+00:00",
   "source": [
     "specs/release-plan.yaml",
     "specs/epics/**/epic.yaml",