From 6dec80f135acbfc6b906861fe157c5d82d86f1ae Mon Sep 17 00:00:00 2001 From: nivSwisa1 Date: Mon, 26 Oct 2026 12:44:29 +0200 Subject: [PATCH 1/4] LINBEE-30104 | fix: hold edits until the verdict line is printed (v1.4.7) Co-Authored-By: Claude Opus 5.5 --- .../.claude-plugin/plugin.json | 2 +- plugins/agentic-advisor/CHANGELOG.md | 8 +++ plugins/agentic-advisor/README.md | 2 +- .../hooks/agentic-advisor-trigger.sh | 49 ++++++++++--------- tests/agentic-advisor/hooks_test.sh | 20 ++++++-- 5 files changed, 51 insertions(+), 30 deletions(-) diff --git a/plugins/agentic-advisor/.claude-plugin/plugin.json b/plugins/agentic-advisor/.claude-plugin/plugin.json index b3e0d77..b101870 100644 --- a/plugins/agentic-advisor/.claude-plugin/plugin.json +++ b/plugins/agentic-advisor/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "agentic-advisor", - "version": "1.4.6", + "version": "1.4.7", "description": "Before writing code, grades how fragile the target is from LinearB health signals + local git history (rework, incidents, fix/revert density, code ownership) and holds the agent to a matching effort level (LOW/MEDIUM/HIGH) — lean on calm repos, defensive on fragile ones. Auto-triggers via a bundled hook and can report each effort decision to your LinearB org as usage telemetry.", "author": { "name": "LinearB", diff --git a/plugins/agentic-advisor/CHANGELOG.md b/plugins/agentic-advisor/CHANGELOG.md index e89a165..99c3f85 100644 --- a/plugins/agentic-advisor/CHANGELOG.md +++ b/plugins/agentic-advisor/CHANGELOG.md @@ -1,5 +1,13 @@ # Changelog +## [1.4.7] - 2026-10-26 + +### Fixed + +- The grade-before-edit hold now repeats until the verdict line is printed (at most 3 holds per repo and session, so it never walls). A prose summary of the grade, or switching to another file, no longer gets past it. +- Edits and subagent spawns go to that hold instead of a one-time reminder that let the edit through. +- The hook messages name the skill's SKILL.md file, so the agent can still grade when the skill is missing from its skill list. + ## [1.4.6] - 2026-10-21 ### Added diff --git a/plugins/agentic-advisor/README.md b/plugins/agentic-advisor/README.md index 9ae32ac..e14708d 100644 --- a/plugins/agentic-advisor/README.md +++ b/plugins/agentic-advisor/README.md @@ -34,7 +34,7 @@ The agent prints a one-line verdict before writing code, e.g.: ## Triggers -Auto-fires (once per repo, per session) on a Jira ticket in the prompt, code-task keywords (fix / implement / refactor…), or the first source-file edit or subagent spawn in a git repo (read-only agents like Explore don't count). Skips questions, doc/text edits, and files outside a git repo. If the grade was decided but never printed (common at lower effort levels, where the model keeps it in its thinking), the agent gets a one-time reminder (not a block) on its next file read or edit to record the verdict line. +Auto-fires (once per repo, per session) on a Jira ticket in the prompt, code-task keywords (fix / implement / refactor…), or the first source-file edit or subagent spawn in a git repo (read-only agents like Explore don't count). Skips questions, doc/text edits, and files outside a git repo. If the grade was decided but never printed (common at lower effort levels, where the model keeps it in its thinking or writes it as prose), the agent gets a reminder on its next file read, and its source edits and subagent spawns are held until the verdict line exists (at most 3 times per repo and session, so it never walls). ## Usage telemetry (on by default) diff --git a/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh b/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh index ee830e0..289f8d5 100755 --- a/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh +++ b/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh @@ -96,7 +96,8 @@ set_marker() { # $1 = a directory inside the repo } # Imperative, silent guidance — no permission question, no announcement. -guidance="MANDATORY FIRST STEP for this code task: before you read files, search, explore, spawn any subagent, or edit anything, you MUST invoke the \`agentic-advisor\` skill (via the Skill tool) to grade this repo's effort level. Do not start the task, do not touch code, until you have run it and printed its verdict line verbatim in this shape: > LinearB: — effort () — (a hook parses that line; a prose summary is not recorded). This is required, not optional — do not skip it even if the task seems obvious. Invoke it directly (no need to ask the user or announce it). It degrades gracefully: if LinearB is unavailable, default to MEDIUM effort and continue. If the task will not change any code, print > LinearB: — skipped (not a code task) instead of a grade." +skill_file="$(cd "$(dirname "$0")/.." 2>/dev/null && pwd)/skills/agentic-advisor/SKILL.md" +guidance="MANDATORY FIRST STEP for this code task: before you read files, search, explore, spawn any subagent, or edit anything, you MUST invoke the \`agentic-advisor\` skill (via the Skill tool) to grade this repo's effort level. Do not start the task, do not touch code, until you have run it and printed its verdict line verbatim in this shape: > LinearB: — effort () — (a hook parses that line; a prose summary is not recorded). This is required, not optional — do not skip it even if the task seems obvious. Invoke it directly (no need to ask the user or announce it). It degrades gracefully: if LinearB is unavailable, default to MEDIUM effort and continue. If the task will not change any code, print > LinearB: — skipped (not a code task) instead of a grade. If the skill is not in your skill list, read $skill_file and follow it." # Mark handled (this session+repo) and emit additionalContext for the event. emit() { # $1 = hookEventName, $2 = context text @@ -149,10 +150,12 @@ case "$event" in ;; PreToolUse) - # Verdict reminder: once per advisor run, if the first non-shell tool comes before that run's - # verdict line, attach a note asking for it (the call still runs). Keyed by the run count so a - # second repo's run is checked too; the marker stops re-parsing once a run is settled. - if skill_ran; then + tool="$(jq -r '.tool_name // empty' <<<"$input" 2>/dev/null || true)" + # Verdict reminder: once per advisor run, if a read comes before that run's verdict line, attach a + # note asking for it (the call still runs). Edits and spawns skip it: the hold below blocks them + # until the line exists. Keyed by the run count so a second repo's run is checked too. + case "$tool" in Read|Grep|Glob) is_read=1 ;; *) is_read=0 ;; esac + if [ "$is_read" = 1 ] && skill_ran; then # Count only assistant Skill calls (same thing verdict_state indexes), not echoes elsewhere. runs="$(grep '"role":"assistant"' "$transcript" 2>/dev/null | grep -cE '"skill"[[:space:]]*:[[:space:]]*"[^"]*'"agentic-advisor"'[^"]*"')" vdone="$marker_dir/${session_id:-default}.${runs:-0}.vdone" @@ -160,11 +163,10 @@ case "$event" in state="$(verdict_state)" if [ -n "$state" ] && : > "$vdone" 2>/dev/null && [ "$state" = "pending" ]; then dbg "PreToolUse: reminded the agent to print the verdict line" - note "LinearB: your effort grade isn't recorded yet. It's only recorded from the verdict line, not from thinking, so print it now: > LinearB: — effort () — . If you aren't writing text, run: printf '%s\\n' '> LinearB: — effort () — '. If no code will change, print > LinearB: — skipped (not a code task) instead." + note "LinearB: your effort grade isn't recorded yet. It's only recorded from the verdict line, not from thinking, so print it now (a prose summary doesn't count): > LinearB: — effort () — . If you aren't writing text, run: printf '%s\\n' '> LinearB: — effort () — '. If no code will change, print > LinearB: — skipped (not a code task) instead." fi fi fi - tool="$(jq -r '.tool_name // empty' <<<"$input" 2>/dev/null || true)" case "$tool" in Read|Grep|Glob) exit 0 ;; Task|Agent) @@ -189,13 +191,10 @@ case "$event" in ;; esac - # ONE-TIME grade-before-work HOLD: the FIRST source edit or subagent spawn in a repo this - # session is denied (permissionDecision:deny) so the skill grades before code is written. - # Held at most ONCE per repo/session (via its own dedicated .held marker — NOT the - # UserPromptSubmit nudge (.done) marker, so a prior prompt-nudge can't satisfy the check - # and bypass the hold); after that one hold, work proceeds even if the skill still hasn't - # run (we don't wall). Scoped to a git repo; fails OPEN on any uncertainty - # (missing file path, non-git, missing transcript, or marker write failure). + # GRADE-before-work HOLD: a source edit or subagent spawn in a repo with no verdict line this session + # is denied (permissionDecision:deny) until one exists, at most 3 times per repo/session (its own .held + # counter, not the nudge's .done marker), so prose or a file switch can't skip the grade and a broken + # setup never walls. Scoped to a git repo; fails OPEN on any uncertainty. git -C "$dir" rev-parse --is-inside-work-tree >/dev/null 2>&1 || exit 0 # Skill already graded this session -> allow the edit. A "skipped (not a code task)" line alone doesn't count: editing now is code work. @@ -217,16 +216,18 @@ case "$event" in # UserPromptSubmit nudge can't satisfy this check and bypass the hold. set_marker "$dir" held="${marker%.done}.held" - [ -f "$held" ] && exit 0 # already held once this repo/session -> allow - - # First source edit or spawn here + skill hasn't run -> HOLD it once, via a structured - # permissionDecision:deny. Same one-time block as before, but a much cleaner UI - # than `exit 2`: it shows just the reason (no "PreToolUse hook error: [path]" - # prefix) and feeds that reason to the model. FAIL OPEN if we can't record the - # marker (don't wedge). - : > "$held" 2>/dev/null || exit 0 - dbg "PreToolUse: held $tool until the skill runs" - reason="One quick first step before ${what}: run the \`agentic-advisor\` skill to grade this repo's effort (just this once per repo, this session). Print its verdict line (> LinearB: — effort (…) — …), then make the same call again — it'll go right through, and later calls won't pause. If LinearB isn't available it just defaults to MEDIUM." + n="$(cat "$held" 2>/dev/null)"; case "$n" in ''|*[!0-9]*) n=0 ;; esac + [ "$n" -ge 3 ] && exit 0 # held 3 times this repo/session -> allow (never wall) + + # Source edit or spawn here + no verdict line yet -> HOLD it via permissionDecision:deny (shows + # just the reason and feeds it to the model). FAIL OPEN if we can't record the count (don't wedge). + printf '%s' "$((n + 1))" > "$held" 2>/dev/null || exit 0 + dbg "PreToolUse: held $tool until a verdict line exists (hold $((n + 1)))" + if skill_ran; then + reason="Your effort grade isn't recorded yet: print the verdict line itself, not a prose summary — > LinearB: — effort () — (or run: printf '%s\\n' '> LinearB: …'). A skipped (not a code task) line doesn't count once you change code. Then make the same call again." + else + reason="One quick first step before ${what}: run the \`agentic-advisor\` skill to grade this repo's effort (once per repo, this session). If it is not in your skill list, read $skill_file and follow it. Print its verdict line (> LinearB: — effort (…) — …), then make the same call again. If LinearB isn't available it just defaults to MEDIUM." + fi [ "$what" = "this edit" ] || reason="$reason Tell the subagent the effort level in its prompt, so the delegated work follows it." deny "$reason" ;; diff --git a/tests/agentic-advisor/hooks_test.sh b/tests/agentic-advisor/hooks_test.sh index 9bcbc7c..9217ea2 100755 --- a/tests/agentic-advisor/hooks_test.sh +++ b/tests/agentic-advisor/hooks_test.sh @@ -124,11 +124,25 @@ check "trigger: LINEARB_AGENTIC_ADVISOR_ENABLED=1 keeps the hold" denied teardown # --- trigger: PreToolUse grade-before-edit hold --- +names_skill_file() { jq -e '.hookSpecificOutput.permissionDecisionReason | contains("/skills/agentic-advisor/SKILL.md")' <<<"$out" >/dev/null; } +wants_line() { jq -e '.hookSpecificOutput.permissionDecisionReason | contains("not a prose summary")' <<<"$out" >/dev/null; } setup run "$TRIGGER" PreToolUse "$(tool Edit app.js)" check "trigger: first source edit is held" denied +check "trigger: the hold names the SKILL.md to read if the skill isn't listed" names_skill_file +run "$TRIGGER" PreToolUse "$(tool Edit lib.js)" +check "trigger: switching files doesn't get past the hold" denied run "$TRIGGER" PreToolUse "$(tool Edit app.js)" -check "trigger: hold happens only once" silent +check "trigger: still held without a verdict (3rd)" denied +run "$TRIGGER" PreToolUse "$(tool Edit app.js)" +check "trigger: after 3 holds the edit goes through (never walls)" silent +teardown + +setup +add_skill +printf '%s\n' '{"message":{"role":"assistant","content":[{"type":"text","text":"LinearB shows the repo is calm, so I will rate this MEDIUM effort."}]}}' >> "$TRANSCRIPT" +run "$TRIGGER" PreToolUse "$(tool Edit app.js)" +check "trigger: a prose-only grade is held, asking for the verdict line" wants_line teardown setup @@ -144,9 +158,7 @@ run "$TRIGGER" PreToolUse "$(spawn Agent general-purpose)" check "trigger: first subagent spawn is held" denied check "trigger: spawn hold asks to pass the effort on" pass_effort run "$TRIGGER" PreToolUse "$(spawn Agent general-purpose)" -check "trigger: spawn hold happens only once" silent -run "$TRIGGER" PreToolUse "$(tool Edit app.js)" -check "trigger: a held spawn uses up the edit hold too" silent +check "trigger: a second spawn without a verdict is held again" denied teardown setup From 6956bb1bddd5feb3f836092117986f2cd2e9fea0 Mon Sep 17 00:00:00 2001 From: nivSwisa1 Date: Mon, 26 Oct 2026 12:52:35 +0200 Subject: [PATCH 2/4] fix: address gitstream/Copilot review (round 1) Co-Authored-By: Claude Opus 5.5 --- plugins/agentic-advisor/CHANGELOG.md | 2 +- .../hooks/agentic-advisor-report.sh | 9 ++++---- .../hooks/agentic-advisor-trigger.sh | 14 ++++++++----- tests/agentic-advisor/hooks_test.sh | 21 +++++++++++++++++++ 4 files changed, 36 insertions(+), 10 deletions(-) diff --git a/plugins/agentic-advisor/CHANGELOG.md b/plugins/agentic-advisor/CHANGELOG.md index 99c3f85..3c29e83 100644 --- a/plugins/agentic-advisor/CHANGELOG.md +++ b/plugins/agentic-advisor/CHANGELOG.md @@ -6,7 +6,7 @@ - The grade-before-edit hold now repeats until the verdict line is printed (at most 3 holds per repo and session, so it never walls). A prose summary of the grade, or switching to another file, no longer gets past it. - Edits and subagent spawns go to that hold instead of a one-time reminder that let the edit through. -- The hook messages name the skill's SKILL.md file, so the agent can still grade when the skill is missing from its skill list. +- The hook messages name the skill's SKILL.md file (only when it exists), so the agent can still grade when the skill is missing from its skill list. Reading that file counts as running the skill for the hold and the report. ## [1.4.6] - 2026-10-21 diff --git a/plugins/agentic-advisor/hooks/agentic-advisor-report.sh b/plugins/agentic-advisor/hooks/agentic-advisor-report.sh index 7ae34fe..781215e 100755 --- a/plugins/agentic-advisor/hooks/agentic-advisor-report.sh +++ b/plugins/agentic-advisor/hooks/agentic-advisor-report.sh @@ -116,7 +116,7 @@ touch "$state" 2>/dev/null || true # per-turn Stop that has nothing new to report costs only a grep or two, not a full # transcript parse. (The whole graded path requires the skill to have actually run.) if [ "$label" = "graded" ]; then - grep -qE '"skill"[[:space:]]*:[[:space:]]*"[^"]*agentic-advisor[^"]*"' "$transcript" 2>/dev/null || { [ "$event" = "SessionEnd" ] && dbg "SessionEnd: skip: skill never ran in this session"; exit 0; } + { grep -qE '"skill"[[:space:]]*:[[:space:]]*"[^"]*agentic-advisor[^"]*"' "$transcript" 2>/dev/null || grep -qE '"file_path"[[:space:]]*:[[:space:]]*"[^"]*/skills/[^"/]*agentic-advisor[^"/]*/SKILL\.md"' "$transcript" 2>/dev/null; } || { [ "$event" = "SessionEnd" ] && dbg "SessionEnd: skip: skill never ran in this session"; exit 0; } if [ "$event" = "Stop" ]; then # Has the set of verdict lines changed since we last processed a Stop? SKILL.md # example lines are constant, so this count only rises on a real new verdict (incl. @@ -237,8 +237,9 @@ for line in open(sys.argv[1]): for x in content: if not isinstance(x, dict): continue typ = x.get("type") - if typ == "tool_use" and x.get("name") == "Skill" \ - and "agentic-advisor" in str((x.get("input") or {}).get("skill", "")): + # The skill starts with its Skill call, or with a Read of its SKILL.md (the fallback). + if typ == "tool_use" and ((x.get("name") == "Skill" and "agentic-advisor" in str((x.get("input") or {}).get("skill", ""))) + or (x.get("name") == "Read" and re.search(r"/skills/[^/]*agentic-advisor[^/]*/SKILL\.md$", str((x.get("input") or {}).get("file_path", ""))))): if skill is None: skill = te # Same docs/text skip as the trigger: a docs-only edit is not a code session. elif typ == "tool_use" and x.get("name") in ("Edit", "Write") and first_edit is None \ @@ -283,7 +284,7 @@ if [ "$label" = "baseline" ]; then # should never run. If it ran anyway (grep finds a real Skill invocation), flag it — # a contaminated baseline must be excluded from the control arm. contaminated="false" - grep -qE '"skill"[[:space:]]*:[[:space:]]*"[^"]*agentic-advisor[^"]*"' "$transcript" 2>/dev/null && contaminated="true" + { grep -qE '"skill"[[:space:]]*:[[:space:]]*"[^"]*agentic-advisor[^"]*"' "$transcript" 2>/dev/null || grep -qE '"file_path"[[:space:]]*:[[:space:]]*"[^"]*/skills/[^"/]*agentic-advisor[^"/]*/SKILL\.md"' "$transcript" 2>/dev/null; } && contaminated="true" payload="$(jq -n \ --arg m "$METRIC_NAME" --arg src "$SOURCE" --argjson ts "$ts" \ --arg repo "$repo" --arg email "$email" --arg repourl "$repo_url" --arg branch "$branch" \ diff --git a/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh b/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh index 289f8d5..2118877 100755 --- a/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh +++ b/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh @@ -47,7 +47,8 @@ transcript="$(jq -r '.transcript_path // empty' <<<"$input" 2>/dev/null || true) # the first source edit so the effort grade is computed BEFORE code is written. skill_ran() { [ -n "$transcript" ] && [ -f "$transcript" ] || return 1 - grep -qE '"skill"[[:space:]]*:[[:space:]]*"[^"]*agentic-advisor[^"]*"' "$transcript" 2>/dev/null + grep -qE '"skill"[[:space:]]*:[[:space:]]*"[^"]*agentic-advisor[^"]*"' "$transcript" 2>/dev/null \ + || grep -qE '"file_path"[[:space:]]*:[[:space:]]*"[^"]*/skills/[^"/]*agentic-advisor[^"/]*/SKILL\.md"' "$transcript" 2>/dev/null } # Is the LATEST advisor Skill call still missing its verdict (assistant text, or the @@ -57,8 +58,10 @@ GRADED_RE='LinearB: .+ (LOW|MEDIUM|HIGH) effort' verdict_state() { # $1 = the verdict regex required (default: a grade or a skip) jq -rs --arg n "agentic-advisor" --arg re "${1:-$VERDICT_RE}" ' [.[] | select(.message.role? == "assistant") | .message.content? // [] | .[]?] as $c - | [$c | to_entries[] | select(.value.type? == "tool_use" and .value.name? == "Skill" - and ((.value.input.skill? // "") | tostring | contains($n))) | .key] as $sk + | [$c | to_entries[] | select(.value.type? == "tool_use" and ( + (.value.name? == "Skill" and ((.value.input.skill? // "") | tostring | contains($n))) + or (.value.name? == "Read" and ((.value.input.file_path? // "") | tostring + | test("/skills/[^/]*" + $n + "[^/]*/SKILL\\.md$"))))) | .key] as $sk | if ($sk | length) == 0 then "ok" elif ($c[($sk[-1] + 1):] | any( (.type? == "text" and ((.text? // "") | test($re))) @@ -97,7 +100,8 @@ set_marker() { # $1 = a directory inside the repo # Imperative, silent guidance — no permission question, no announcement. skill_file="$(cd "$(dirname "$0")/.." 2>/dev/null && pwd)/skills/agentic-advisor/SKILL.md" -guidance="MANDATORY FIRST STEP for this code task: before you read files, search, explore, spawn any subagent, or edit anything, you MUST invoke the \`agentic-advisor\` skill (via the Skill tool) to grade this repo's effort level. Do not start the task, do not touch code, until you have run it and printed its verdict line verbatim in this shape: > LinearB: — effort () — (a hook parses that line; a prose summary is not recorded). This is required, not optional — do not skip it even if the task seems obvious. Invoke it directly (no need to ask the user or announce it). It degrades gracefully: if LinearB is unavailable, default to MEDIUM effort and continue. If the task will not change any code, print > LinearB: — skipped (not a code task) instead of a grade. If the skill is not in your skill list, read $skill_file and follow it." +skill_hint=""; [ -f "$skill_file" ] && skill_hint=" If the skill is not in your skill list, read $skill_file and follow it." +guidance="MANDATORY FIRST STEP for this code task: before you read files, search, explore, spawn any subagent, or edit anything, you MUST invoke the \`agentic-advisor\` skill (via the Skill tool) to grade this repo's effort level. Do not start the task, do not touch code, until you have run it and printed its verdict line verbatim in this shape: > LinearB: — effort () — (a hook parses that line; a prose summary is not recorded). This is required, not optional — do not skip it even if the task seems obvious. Invoke it directly (no need to ask the user or announce it). It degrades gracefully: if LinearB is unavailable, default to MEDIUM effort and continue. If the task will not change any code, print > LinearB: — skipped (not a code task) instead of a grade.$skill_hint" # Mark handled (this session+repo) and emit additionalContext for the event. emit() { # $1 = hookEventName, $2 = context text @@ -226,7 +230,7 @@ case "$event" in if skill_ran; then reason="Your effort grade isn't recorded yet: print the verdict line itself, not a prose summary — > LinearB: — effort () — (or run: printf '%s\\n' '> LinearB: …'). A skipped (not a code task) line doesn't count once you change code. Then make the same call again." else - reason="One quick first step before ${what}: run the \`agentic-advisor\` skill to grade this repo's effort (once per repo, this session). If it is not in your skill list, read $skill_file and follow it. Print its verdict line (> LinearB: — effort (…) — …), then make the same call again. If LinearB isn't available it just defaults to MEDIUM." + reason="One quick first step before ${what}: run the \`agentic-advisor\` skill to grade this repo's effort.$skill_hint Print its verdict line (> LinearB: — effort (…) — …), then make the same call again; this hold repeats until it's printed. If LinearB isn't available it just defaults to MEDIUM." fi [ "$what" = "this edit" ] || reason="$reason Tell the subagent the effort level in its prompt, so the delegated work follows it." deny "$reason" diff --git a/tests/agentic-advisor/hooks_test.sh b/tests/agentic-advisor/hooks_test.sh index 9217ea2..ec97443 100755 --- a/tests/agentic-advisor/hooks_test.sh +++ b/tests/agentic-advisor/hooks_test.sh @@ -55,6 +55,8 @@ add_skill() { printf '%s\n' '{"timestamp":"2026-01-01T00:00:00Z","message":{"r add_verdict() { printf '%s\n' '{"timestamp":"2026-01-01T00:00:05Z","message":{"role":"assistant","model":"m","usage":{"output_tokens":20},"content":[{"type":"text","text":"> LinearB: widgets — MEDIUM effort (rework 6.5%) — focused diff."}]}}' >> "$TRANSCRIPT"; } # The loaded SKILL.md lands in the transcript as a user message, example verdict line included. add_skill_body() { printf '%s\n' '{"type":"user","isMeta":true,"message":{"role":"user","content":[{"type":"text","text":"Example: > LinearB: api-service — HIGH effort (rework 3.4%) — small steps."}]}}' >> "$TRANSCRIPT"; } +# The fallback: the agent reads the skill's SKILL.md instead of calling the Skill tool. +add_skill_read() { printf '%s\n' '{"timestamp":"2026-01-01T00:00:00Z","message":{"role":"assistant","model":"m","usage":{"output_tokens":10},"content":[{"type":"tool_use","name":"Read","input":{"file_path":"/x/plugins/agentic-advisor/skills/agentic-advisor/SKILL.md"}}]}}' >> "$TRANSCRIPT"; } add_skipped() { printf '%s\n' '{"message":{"role":"assistant","content":[{"type":"text","text":"> LinearB: widgets — skipped (not a code task)"}]}}' >> "$TRANSCRIPT"; } add_verdict_for() { printf '{"message":{"role":"assistant","content":[{"type":"text","text":"> LinearB: %s — MEDIUM effort (x) — y."}]}}\n' "$1" >> "$TRANSCRIPT"; } # Reporter posts from a detached subshell; wait briefly for the stub to record calls. @@ -145,6 +147,18 @@ run "$TRIGGER" PreToolUse "$(tool Edit app.js)" check "trigger: a prose-only grade is held, asking for the verdict line" wants_line teardown +setup +add_skill_read; add_skill_body; add_verdict +run "$TRIGGER" PreToolUse "$(tool Edit app.js)" +check "trigger: grading via the SKILL.md fallback (no Skill call) releases the edit" silent +teardown + +setup +add_skill_read; add_skill_body +run "$TRIGGER" PreToolUse "$(tool Edit app.js)" +check "trigger: a fallback read without a verdict line is still held" wants_line +teardown + setup run "$TRIGGER" PreToolUse "$(tool Edit README.md)" check "trigger: docs edit is never held" silent @@ -232,6 +246,13 @@ sleep 0.5 # a wrong post would arrive from a detached subshell check "report: a skipped (not a code task) session reports nothing" no_calls teardown +setup +add_skill_read; add_verdict +run "$REPORT" Stop +await_calls 1 +check "report: a grade made via the SKILL.md fallback is reported" one_decision +teardown + # Both exit before any background post, so no wait is needed. setup add_skill; add_verdict From 62fdca9b8ad778e294a2bd75e68040b95c8d3e83 Mon Sep 17 00:00:00 2001 From: nivSwisa1 Date: Mon, 26 Oct 2026 12:58:38 +0200 Subject: [PATCH 3/4] fix: address gitstream/Copilot review (round 2) Co-Authored-By: Claude Opus 5.5 --- plugins/agentic-advisor/hooks/agentic-advisor-report.sh | 4 ++-- plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh | 4 ++-- tests/agentic-advisor/hooks_test.sh | 7 +++++++ 3 files changed, 11 insertions(+), 4 deletions(-) diff --git a/plugins/agentic-advisor/hooks/agentic-advisor-report.sh b/plugins/agentic-advisor/hooks/agentic-advisor-report.sh index 781215e..d3a3c3b 100755 --- a/plugins/agentic-advisor/hooks/agentic-advisor-report.sh +++ b/plugins/agentic-advisor/hooks/agentic-advisor-report.sh @@ -116,7 +116,7 @@ touch "$state" 2>/dev/null || true # per-turn Stop that has nothing new to report costs only a grep or two, not a full # transcript parse. (The whole graded path requires the skill to have actually run.) if [ "$label" = "graded" ]; then - { grep -qE '"skill"[[:space:]]*:[[:space:]]*"[^"]*agentic-advisor[^"]*"' "$transcript" 2>/dev/null || grep -qE '"file_path"[[:space:]]*:[[:space:]]*"[^"]*/skills/[^"/]*agentic-advisor[^"/]*/SKILL\.md"' "$transcript" 2>/dev/null; } || { [ "$event" = "SessionEnd" ] && dbg "SessionEnd: skip: skill never ran in this session"; exit 0; } + { grep -qE '"skill"[[:space:]]*:[[:space:]]*"[^"]*agentic-advisor[^"]*"' "$transcript" 2>/dev/null || grep '"role":"assistant"' "$transcript" 2>/dev/null | grep -E '"name"[[:space:]]*:[[:space:]]*"Read"' | grep -qE '"file_path"[[:space:]]*:[[:space:]]*"[^"]*/skills/[^"/]*agentic-advisor[^"/]*/SKILL\.md"'; } || { [ "$event" = "SessionEnd" ] && dbg "SessionEnd: skip: skill never ran in this session"; exit 0; } if [ "$event" = "Stop" ]; then # Has the set of verdict lines changed since we last processed a Stop? SKILL.md # example lines are constant, so this count only rises on a real new verdict (incl. @@ -284,7 +284,7 @@ if [ "$label" = "baseline" ]; then # should never run. If it ran anyway (grep finds a real Skill invocation), flag it — # a contaminated baseline must be excluded from the control arm. contaminated="false" - { grep -qE '"skill"[[:space:]]*:[[:space:]]*"[^"]*agentic-advisor[^"]*"' "$transcript" 2>/dev/null || grep -qE '"file_path"[[:space:]]*:[[:space:]]*"[^"]*/skills/[^"/]*agentic-advisor[^"/]*/SKILL\.md"' "$transcript" 2>/dev/null; } && contaminated="true" + { grep -qE '"skill"[[:space:]]*:[[:space:]]*"[^"]*agentic-advisor[^"]*"' "$transcript" 2>/dev/null || grep '"role":"assistant"' "$transcript" 2>/dev/null | grep -E '"name"[[:space:]]*:[[:space:]]*"Read"' | grep -qE '"file_path"[[:space:]]*:[[:space:]]*"[^"]*/skills/[^"/]*agentic-advisor[^"/]*/SKILL\.md"'; } && contaminated="true" payload="$(jq -n \ --arg m "$METRIC_NAME" --arg src "$SOURCE" --argjson ts "$ts" \ --arg repo "$repo" --arg email "$email" --arg repourl "$repo_url" --arg branch "$branch" \ diff --git a/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh b/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh index 2118877..5852650 100755 --- a/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh +++ b/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh @@ -48,7 +48,7 @@ transcript="$(jq -r '.transcript_path // empty' <<<"$input" 2>/dev/null || true) skill_ran() { [ -n "$transcript" ] && [ -f "$transcript" ] || return 1 grep -qE '"skill"[[:space:]]*:[[:space:]]*"[^"]*agentic-advisor[^"]*"' "$transcript" 2>/dev/null \ - || grep -qE '"file_path"[[:space:]]*:[[:space:]]*"[^"]*/skills/[^"/]*agentic-advisor[^"/]*/SKILL\.md"' "$transcript" 2>/dev/null + || grep '"role":"assistant"' "$transcript" 2>/dev/null | grep -E '"name"[[:space:]]*:[[:space:]]*"Read"' | grep -qE '"file_path"[[:space:]]*:[[:space:]]*"[^"]*/skills/[^"/]*agentic-advisor[^"/]*/SKILL\.md"' } # Is the LATEST advisor Skill call still missing its verdict (assistant text, or the @@ -161,7 +161,7 @@ case "$event" in case "$tool" in Read|Grep|Glob) is_read=1 ;; *) is_read=0 ;; esac if [ "$is_read" = 1 ] && skill_ran; then # Count only assistant Skill calls (same thing verdict_state indexes), not echoes elsewhere. - runs="$(grep '"role":"assistant"' "$transcript" 2>/dev/null | grep -cE '"skill"[[:space:]]*:[[:space:]]*"[^"]*'"agentic-advisor"'[^"]*"')" + runs="$(grep '"role":"assistant"' "$transcript" 2>/dev/null | grep -cE '"skill"[[:space:]]*:[[:space:]]*"[^"]*'"agentic-advisor"'[^"]*"|"name"[[:space:]]*:[[:space:]]*"Read".*/skills/[^"/]*'"agentic-advisor"'[^"/]*/SKILL\.md"')" vdone="$marker_dir/${session_id:-default}.${runs:-0}.vdone" if [ ! -f "$vdone" ]; then state="$(verdict_state)" diff --git a/tests/agentic-advisor/hooks_test.sh b/tests/agentic-advisor/hooks_test.sh index ec97443..1a4b984 100755 --- a/tests/agentic-advisor/hooks_test.sh +++ b/tests/agentic-advisor/hooks_test.sh @@ -159,6 +159,13 @@ run "$TRIGGER" PreToolUse "$(tool Edit app.js)" check "trigger: a fallback read without a verdict line is still held" wants_line teardown +setup +printf '%s\n' '{"message":{"role":"assistant","content":[{"type":"tool_use","name":"Edit","input":{"file_path":"/x/plugins/agentic-advisor/skills/agentic-advisor/SKILL.md"}}]}}' >> "$TRANSCRIPT" +add_verdict +run "$TRIGGER" PreToolUse "$(tool Edit app.js)" +check "trigger: an Edit of SKILL.md is not a fallback read, so a source edit is still held" denied +teardown + setup run "$TRIGGER" PreToolUse "$(tool Edit README.md)" check "trigger: docs edit is never held" silent From d5b49997c909a179e79a6081bd095ea4a777be8e Mon Sep 17 00:00:00 2001 From: nivSwisa1 Date: Mon, 26 Oct 2026 13:49:49 +0200 Subject: [PATCH 4/4] fix: address gitstream/Copilot review (round 3) Co-Authored-By: Claude Opus 5.5 --- .../hooks/agentic-advisor-report.sh | 9 +++++++-- .../hooks/agentic-advisor-trigger.sh | 16 +++++++++++----- tests/agentic-advisor/hooks_test.sh | 13 +++++++++++++ 3 files changed, 31 insertions(+), 7 deletions(-) diff --git a/plugins/agentic-advisor/hooks/agentic-advisor-report.sh b/plugins/agentic-advisor/hooks/agentic-advisor-report.sh index d3a3c3b..24f932e 100755 --- a/plugins/agentic-advisor/hooks/agentic-advisor-report.sh +++ b/plugins/agentic-advisor/hooks/agentic-advisor-report.sh @@ -75,6 +75,11 @@ input="$(cat)" event="$(jq -r '.hook_event_name // empty' <<<"$input" 2>/dev/null || true)" session_id="$(jq -r '.session_id // empty' <<<"$input" 2>/dev/null || true)" transcript="$(jq -r '.transcript_path // empty' <<<"$input" 2>/dev/null || true)" +read_ran() { # the agent read the skill's SKILL.md (the fallback when the Skill tool lists no skill) + [ -n "$(jq -r --arg re '/skills/[^/]*agentic-advisor[^/]*/SKILL\.md$' 'select(.message.role? == "assistant") | .message.content? // [] | .[]? + | select(.type? == "tool_use" and .name? == "Read" and ((.input.file_path? // "") | tostring | test($re))) | "y"' \ + "$transcript" 2>/dev/null | head -n 1)" ] +} cwd="$(jq -r '.cwd // empty' <<<"$input" 2>/dev/null || true)" effort_actual="$(jq -r '.effort.level // empty' <<<"$input" 2>/dev/null || true)" if [ -z "$transcript" ] || [ ! -f "$transcript" ]; then @@ -116,7 +121,7 @@ touch "$state" 2>/dev/null || true # per-turn Stop that has nothing new to report costs only a grep or two, not a full # transcript parse. (The whole graded path requires the skill to have actually run.) if [ "$label" = "graded" ]; then - { grep -qE '"skill"[[:space:]]*:[[:space:]]*"[^"]*agentic-advisor[^"]*"' "$transcript" 2>/dev/null || grep '"role":"assistant"' "$transcript" 2>/dev/null | grep -E '"name"[[:space:]]*:[[:space:]]*"Read"' | grep -qE '"file_path"[[:space:]]*:[[:space:]]*"[^"]*/skills/[^"/]*agentic-advisor[^"/]*/SKILL\.md"'; } || { [ "$event" = "SessionEnd" ] && dbg "SessionEnd: skip: skill never ran in this session"; exit 0; } + { grep -qE '"skill"[[:space:]]*:[[:space:]]*"[^"]*agentic-advisor[^"]*"' "$transcript" 2>/dev/null || read_ran; } || { [ "$event" = "SessionEnd" ] && dbg "SessionEnd: skip: skill never ran in this session"; exit 0; } if [ "$event" = "Stop" ]; then # Has the set of verdict lines changed since we last processed a Stop? SKILL.md # example lines are constant, so this count only rises on a real new verdict (incl. @@ -284,7 +289,7 @@ if [ "$label" = "baseline" ]; then # should never run. If it ran anyway (grep finds a real Skill invocation), flag it — # a contaminated baseline must be excluded from the control arm. contaminated="false" - { grep -qE '"skill"[[:space:]]*:[[:space:]]*"[^"]*agentic-advisor[^"]*"' "$transcript" 2>/dev/null || grep '"role":"assistant"' "$transcript" 2>/dev/null | grep -E '"name"[[:space:]]*:[[:space:]]*"Read"' | grep -qE '"file_path"[[:space:]]*:[[:space:]]*"[^"]*/skills/[^"/]*agentic-advisor[^"/]*/SKILL\.md"'; } && contaminated="true" + { grep -qE '"skill"[[:space:]]*:[[:space:]]*"[^"]*agentic-advisor[^"]*"' "$transcript" 2>/dev/null || read_ran; } && contaminated="true" payload="$(jq -n \ --arg m "$METRIC_NAME" --arg src "$SOURCE" --argjson ts "$ts" \ --arg repo "$repo" --arg email "$email" --arg repourl "$repo_url" --arg branch "$branch" \ diff --git a/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh b/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh index 5852650..943b9bb 100755 --- a/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh +++ b/plugins/agentic-advisor/hooks/agentic-advisor-trigger.sh @@ -45,10 +45,15 @@ transcript="$(jq -r '.transcript_path // empty' <<<"$input" 2>/dev/null || true) # Has the agentic-advisor skill ACTUALLY run in this session? (a real Skill # tool_use in the transcript, not just text that quotes it). Used to hard-gate # the first source edit so the effort grade is computed BEFORE code is written. +read_ran() { # the agent read the skill's SKILL.md (the fallback when the Skill tool lists no skill) + [ -n "$(jq -r --arg re '/skills/[^/]*agentic-advisor[^/]*/SKILL\.md$' 'select(.message.role? == "assistant") | .message.content? // [] | .[]? + | select(.type? == "tool_use" and .name? == "Read" and ((.input.file_path? // "") | tostring | test($re))) | "y"' \ + "$transcript" 2>/dev/null | head -n 1)" ] +} skill_ran() { [ -n "$transcript" ] && [ -f "$transcript" ] || return 1 grep -qE '"skill"[[:space:]]*:[[:space:]]*"[^"]*agentic-advisor[^"]*"' "$transcript" 2>/dev/null \ - || grep '"role":"assistant"' "$transcript" 2>/dev/null | grep -E '"name"[[:space:]]*:[[:space:]]*"Read"' | grep -qE '"file_path"[[:space:]]*:[[:space:]]*"[^"]*/skills/[^"/]*agentic-advisor[^"/]*/SKILL\.md"' + || read_ran } # Is the LATEST advisor Skill call still missing its verdict (assistant text, or the @@ -58,10 +63,11 @@ GRADED_RE='LinearB: .+ (LOW|MEDIUM|HIGH) effort' verdict_state() { # $1 = the verdict regex required (default: a grade or a skip) jq -rs --arg n "agentic-advisor" --arg re "${1:-$VERDICT_RE}" ' [.[] | select(.message.role? == "assistant") | .message.content? // [] | .[]?] as $c - | [$c | to_entries[] | select(.value.type? == "tool_use" and ( - (.value.name? == "Skill" and ((.value.input.skill? // "") | tostring | contains($n))) - or (.value.name? == "Read" and ((.value.input.file_path? // "") | tostring - | test("/skills/[^/]*" + $n + "[^/]*/SKILL\\.md$"))))) | .key] as $sk + | [$c | to_entries[] | select(.value.type? == "tool_use" and .value.name? == "Skill" + and ((.value.input.skill? // "") | tostring | contains($n))) | .key] as $calls + | [$c | to_entries[] | select(.value.type? == "tool_use" and .value.name? == "Read" + and ((.value.input.file_path? // "") | tostring | test("/skills/[^/]*" + $n + "[^/]*/SKILL\\.md$"))) | .key] as $reads + | (if ($calls | length) > 0 then $calls else $reads[0:1] end) as $sk | if ($sk | length) == 0 then "ok" elif ($c[($sk[-1] + 1):] | any( (.type? == "text" and ((.text? // "") | test($re))) diff --git a/tests/agentic-advisor/hooks_test.sh b/tests/agentic-advisor/hooks_test.sh index 1a4b984..cc5e637 100755 --- a/tests/agentic-advisor/hooks_test.sh +++ b/tests/agentic-advisor/hooks_test.sh @@ -166,6 +166,19 @@ run "$TRIGGER" PreToolUse "$(tool Edit app.js)" check "trigger: an Edit of SKILL.md is not a fallback read, so a source edit is still held" denied teardown +setup +printf '%s\n' '{"message":{"role":"assistant","content":[{"type":"tool_use","name":"Read","input":{"file_path":"/x/app.js"}},{"type":"tool_use","name":"Edit","input":{"file_path":"/x/plugins/agentic-advisor/skills/agentic-advisor/SKILL.md"}}]}}' >> "$TRANSCRIPT" +add_verdict +run "$TRIGGER" PreToolUse "$(tool Edit app.js)" +check "trigger: a Read of another file batched with an Edit of SKILL.md is not a fallback read" denied +teardown + +setup +add_skill; add_verdict; add_skill_read +run "$TRIGGER" PreToolUse "$(tool Edit app.js)" +check "trigger: re-reading SKILL.md after the verdict keeps the repo graded" silent +teardown + setup run "$TRIGGER" PreToolUse "$(tool Edit README.md)" check "trigger: docs edit is never held" silent