diff --git a/docs/experiments/E1-fix-outcome-replay-2026-10-20.adoc b/docs/experiments/E1-fix-outcome-replay-2026-10-20.adoc index 55a1ca37..65149291 100644 --- a/docs/experiments/E1-fix-outcome-replay-2026-10-20.adoc +++ b/docs/experiments/E1-fix-outcome-replay-2026-10-20.adoc @@ -109,7 +109,7 @@ They went in commit `8225e55f` (2026-02-27, "Auto-commit: Sync changes", owner). ** `recipe-annotate-believe-me` is the canonical id. `recipe-remove-believe-me` stays as a registry alias, listed under `deprecated_recipe_ids`, because Hypatia still emits it. `dispatch-runner.sh` reports outcomes under the canonical id, so a verified annotation never raises the removal recipe's confidence. ** The script's predicate measures what it actually does: un-annotated `believe_me` calls. E1 above measured the claimed effect (removal). * *Reporting.* `learning-monitor.sh` counted a `.success` field that no row has. It now reports per verdict, with pre-verdict rows shown as legacy. -* *Replay.* `scripts/audit-fix-outcomes.sh` reruns this experiment. It reads the rows of all three writers. It excludes the fixers' own third-party directories (`lib/third-party-excludes.sh`), and reports NA, not 0, when `git grep` cannot run. Rerun with those changes, it reproduces `E1-replay-2026-10-20.tsv` byte for byte. It measures `recipe-remove-believe-me` by removal and `recipe-annotate-believe-me` by annotation, as each claims. `E1-replay-2026-10-20.tsv` is its output for the rows above. The HEAD column moves as the repos do. +* *Replay.* `scripts/audit-fix-outcomes.sh` reruns this experiment. It reads the rows of all three writers. It excludes the fixers' own third-party directories (`lib/third-party-excludes.sh`), and reports NA, not 0, when `git grep` cannot run. A row that names one `file` is skipped (and counted on stderr): the predicates are repo-wide, as the fixers run. Every replayed row here is repo-scoped. Rerun with those changes, it reproduces `E1-replay-2026-10-20.tsv` byte for byte. It measures `recipe-remove-believe-me` by removal and `recipe-annotate-believe-me` by annotation, as each claims. `E1-replay-2026-10-20.tsv` is its output for the rows above. The HEAD column moves as the repos do. All of this sits behind the automation quarantine (exit 78), which this change does not lift. It is a precondition for lifting it. diff --git a/run-fleet.sh b/run-fleet.sh index de146364..855938e2 100755 --- a/run-fleet.sh +++ b/run-fleet.sh @@ -346,6 +346,10 @@ cmd_fix() { local ftype fpattern ftype=$(echo "$finding" | jq -r '.type // ""') fpattern=$(echo "$finding" | jq -r '.pattern // ""') + # A deprecated recipe id is looked up and logged as its + # replacement, so a verified annotation is never recorded under + # the removal recipe it was renamed from. + fpattern=$(canonical_recipe_id "$REGISTRY" "$fpattern") local fseverity fseverity=$(echo "$finding" | jq -r '.severity // "unknown"') diff --git a/scripts/audit-fix-outcomes.sh b/scripts/audit-fix-outcomes.sh index 32b954a5..be987cd0 100755 --- a/scripts/audit-fix-outcomes.sh +++ b/scripts/audit-fix-outcomes.sh @@ -18,6 +18,10 @@ # Predicates mirror the fixers' own (their --check modes, applied with # `git grep` to a revision instead of a working tree). Recipes without one # print NA. +# +# Rows are replayed repo-wide, as the fixers run: a row naming one `file` is +# a narrower claim than the predicate measures, so it is skipped (and counted +# on stderr) rather than refuted by a target in some other file. set -euo pipefail LOG=$1; shift # The fixers' own exclusion list (lib/third-party-excludes.sh), as glob @@ -59,7 +63,9 @@ pred() { # repo rev pattern -> count of remaining hits, or NA printf 'repo\tpattern\toutcome\tfixed_at\trev_at\thits_at\thits_plus7d\thits_head\n' for repo in "$@"; do name=$(basename "$repo"); head=$(git -C "$repo" rev-parse origin/main) - jq -r --arg r "$name" 'select(.repo==$r) | [.pattern // .recipe_id // .pattern_id, .outcome, .fixed_at // .timestamp] | select(.[2] != null) | @tsv' "$LOG" | sort -u | + skipped=$(jq -c --arg r "$name" 'select(.repo==$r and (.file // "") != "")' "$LOG" | wc -l) + ((skipped == 0)) || echo "audit: $name: $skipped file-scoped row(s) skipped" >&2 + jq -r --arg r "$name" 'select(.repo==$r and (.file // "") == "") | [.pattern // .recipe_id // .pattern_id, .outcome, .fixed_at // .timestamp] | select(.[2] != null) | @tsv' "$LOG" | sort -u | while IFS=$'\t' read -r pat out ts; do at=$(git -C "$repo" rev-list -1 --before="$ts" origin/main || true) p7=$(git -C "$repo" rev-list -1 --before="$(date -u -d "$ts + 7 days" +%FT%TZ)" origin/main || true) diff --git a/scripts/dispatch-runner.sh b/scripts/dispatch-runner.sh index ef7ff6ae..1248d93a 100755 --- a/scripts/dispatch-runner.sh +++ b/scripts/dispatch-runner.sh @@ -96,6 +96,33 @@ SKIPPED=0 FAILED=0 SUCCEEDED=0 +# --- Run counters --- +# Under --parallel, execute_entry runs in a background subshell, where a plain +# ((FAILED++)) is lost when the job exits: the summary and the heartbeat then +# read 0 failures over real ones. A job (IN_JOB=1) appends the counter's name +# to a tally file instead (one short O_APPEND write, atomic), and the parent +# folds the tally in after `wait`. +count() { + if [[ "${IN_JOB:-0}" == 1 ]]; then + printf '%s\n' "$1" >> "$PARALLEL_TMPDIR/tally" + else + # shellcheck disable=SC2004 # $1 names the counter; the $ is the indirection + (( $1++ )) || true + fi +} + +fold_counts() { + local name + [[ -f "$PARALLEL_TMPDIR/tally" ]] || return 0 + while IFS= read -r name; do + case "$name" in + # shellcheck disable=SC2004 # $name names the counter + EXECUTED|SKIPPED|FAILED|SUCCEEDED) (( $name++ )) || true ;; + esac + done < "$PARALLEL_TMPDIR/tally" +} +# --- end run counters --- + # --- Parse arguments --- while [[ $# -gt 0 ]]; do case "$1" in @@ -357,7 +384,7 @@ execute_entry() { # Validate repo name (prevent directory traversal) if ! validate_safe_name "$repo" "repo"; then echo " SKIP: $pattern_id (unsafe repo name)" - ((SKIPPED++)) || true + count SKIPPED return fi repo_path="$REPOS_BASE/$repo" @@ -378,7 +405,7 @@ execute_entry() { # Double-check path stays within REPOS_BASE if ! validate_path_within "$repo_path" "$REPOS_BASE"; then echo " SKIP: $pattern_id (repo path escapes base)" - ((SKIPPED++)) || true + count SKIPPED return fi @@ -387,7 +414,7 @@ execute_entry() { for tp in "${THIRD_PARTY_PATHS[@]}"; do if [[ "$rel_repo_path" == "$tp" || "$rel_repo_path" == "$tp/"* ]]; then echo " SKIP: $repo (third-party path: $tp)" - ((SKIPPED++)) || true + count SKIPPED return fi done @@ -416,7 +443,7 @@ execute_entry() { echo " REFUSED: $pattern_id → $repo (license/SPDX auto-edits forbidden per estate policy 2026-06-02)" echo " recipe_id=$recipe_id category=$category" echo " Owner must approve and apply licence changes manually, per-file." - ((SKIPPED++)) || true + count SKIPPED record_outcome "$pattern_id" "$recipe_id" "$repo" "" "refused_license_policy" return ;; @@ -425,7 +452,7 @@ execute_entry() { # Skip if repo doesn't exist locally if [[ ! -d "$repo_path" ]]; then echo " SKIP: $repo (not found at $repo_path)" - ((SKIPPED++)) || true + count SKIPPED return fi @@ -477,9 +504,9 @@ execute_entry() { verdict=$(outcome_verdict "$rc" "$before" "$after") echo " $verdict (exit $rc, targets $before -> $after)" if [[ "$verdict" == failed || "$verdict" == incomplete ]]; then - ((FAILED++)) || true + count FAILED else - ((SUCCEEDED++)) || true + count SUCCEEDED fi record_outcome "$pattern_id" "$recipe_id" "$repo" "" "$verdict" "$rc" "$before" "$after" @@ -497,16 +524,16 @@ execute_entry() { verdict=$(outcome_verdict "$rc" NA NA) echo " $verdict (exit $rc)" if [[ "$verdict" == failed ]]; then - ((FAILED++)) || true + count FAILED else - ((SUCCEEDED++)) || true + count SUCCEEDED fi record_outcome "$pattern_id" "$recipe_id" "$repo" "" "$verdict" "$rc" else echo " SKIP: No fix mechanism available (no fix_script, no $RRA_BIN)" - ((SKIPPED++)) || true + count SKIPPED fi - ((EXECUTED++)) || true + count EXECUTED ;; review) @@ -526,7 +553,7 @@ execute_entry() { local finding_file="$findings_dir/${repo}--${safe_pattern_id}.json" echo "$entry" | jq '. + {dispatch_strategy: "review", needs_pr: true}' > "$finding_file" echo " Written: $finding_file" - ((EXECUTED++)) || true + count EXECUTED ;; report_only) @@ -542,12 +569,12 @@ execute_entry() { local advisory_file="$advisory_dir/$(date -u +%Y-%m).jsonl" echo "$entry" | jq '. + {dispatch_strategy: "report_only"}' >> "$advisory_file" - ((EXECUTED++)) || true + count EXECUTED ;; *) echo " UNKNOWN strategy: $strategy" - ((SKIPPED++)) || true + count SKIPPED ;; esac } @@ -589,7 +616,7 @@ run_entry() { wait -n 2>/dev/null || true ((ACTIVE_JOBS--)) || true done - execute_entry "$line" & + IN_JOB=1 execute_entry "$line" & ((ACTIVE_JOBS++)) || true fi } @@ -641,6 +668,7 @@ done < "$MANIFEST_PATH" # Wait for all background jobs to finish if [[ "$PARALLEL" -gt 1 ]]; then wait + fold_counts fi rm -rf "$PARALLEL_TMPDIR" 2>/dev/null || true diff --git a/tests/outcome-verdict-test.sh b/tests/outcome-verdict-test.sh index 8ce803f7..dd931988 100755 --- a/tests/outcome-verdict-test.sh +++ b/tests/outcome-verdict-test.sh @@ -18,6 +18,7 @@ # 6. Two parallel jobs on one repo record one fix, not two (repo_lock). # 7. The E1 replay (audit-fix-outcomes.sh) counts as the fixers do, and # reads NA when git grep cannot run. +# 8. Run counters from background jobs reach the parent (--parallel). set -euo pipefail @@ -194,8 +195,9 @@ git -C "$ar" update-ref refs/remotes/origin/main HEAD printf '%s\n' \ '{"repo":"ar","pattern":"recipe-shell-quote-vars","outcome":"success","fixed_at":"2026-02-01T00:00:00Z"}' \ '{"repo":"ar","pattern_id":"PA004","recipe_id":"recipe-annotate-believe-me","outcome":"success","timestamp":"2026-02-01T00:00:00Z"}' \ + '{"repo":"ar","pattern":"recipe-remove-believe-me","file":"A.idr","outcome":"success","fixed_at":"2026-02-01T00:00:00Z"}' \ > "$TMP/audit/log.jsonl" -audit() { "$ROOT/scripts/audit-fix-outcomes.sh" "$TMP/audit/log.jsonl" "$ar" | awk -F'\t' 'NR > 1 { print $2 "=" $6 }' | sort | paste -sd' '; } +audit() { "$ROOT/scripts/audit-fix-outcomes.sh" "$TMP/audit/log.jsonl" "$ar" 2>/dev/null | awk -F'\t' 'NR > 1 { print $2 "=" $6 }' | sort | paste -sd' '; } check "replay counts as the fixers do" \ "recipe-annotate-believe-me=$(recipe_check_count "$ROOT/scripts/fix-believe-me.sh" "$ar") recipe-shell-quote-vars=$(recipe_check_count "$fixer" "$ar")" \ "$(audit)" @@ -203,6 +205,26 @@ printf '#!/bin/sh\nfor a; do [ "$a" = grep ] && exit 128; done\nexec %s "$@"\n' chmod +x "$TMP/shim-git/git" check "replay predicate failure is NA" "recipe-annotate-believe-me=NA recipe-shell-quote-vars=NA" \ "$(PATH="$TMP/shim-git:$PATH" audit)" +# A row naming one file is a narrower claim than a repo-wide predicate: it is +# skipped (above, no recipe-remove-believe-me row) and the skip is reported. +check "file-scoped row is skipped, and reported" "audit: ar: 1 file-scoped row(s) skipped" \ + "$("$ROOT/scripts/audit-fix-outcomes.sh" "$TMP/audit/log.jsonl" "$ar" 2>&1 >/dev/null)" + +# --- 8. run counters survive --parallel ----------------------------------- +# execute_entry runs in a background subshell under --parallel; its counts +# reach the parent through the tally file (extracted, as in section 4). +awk '/^# --- Run counters ---/,/^# --- end run counters ---/' "$ROOT/scripts/dispatch-runner.sh" > "$TMP/counters.sh" +# shellcheck disable=SC1091 +source "$TMP/counters.sh" +PARALLEL_TMPDIR="$TMP/par-counters" +mkdir -p "$PARALLEL_TMPDIR" +EXECUTED=0 SUCCEEDED=0 FAILED=0 SKIPPED=0 +job_counts() { count EXECUTED; count "$1"; } +IN_JOB=1 job_counts FAILED & IN_JOB=1 job_counts SUCCEEDED & IN_JOB=1 job_counts FAILED & wait +count SKIPPED +fold_counts +check "background counts reach the parent" "EXECUTED=3 SUCCEEDED=1 FAILED=2 SKIPPED=1" \ + "EXECUTED=$EXECUTED SUCCEEDED=$SUCCEEDED FAILED=$FAILED SKIPPED=$SKIPPED" echo "" echo "Results: $pass passed, $fail failed"