diff --git a/CHANGELOG.md b/CHANGELOG.md index 83148c8f..ec2f6ca1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -40,6 +40,15 @@ Releases `activeagent` and `actionagent` 1.6.3 from one tag. takes on no telemetry dependency and `require "active_agent/evals"` still loads on its own. Hand the object to `Runner.new(around_evaluation:)` directly; a plain lambda there keeps working unchanged. +- A `Judge` block that accepts `kind:` is told which of the judge's three calls + it is serving — `:score`, `:recommend` or `:verdict` — so a host can trace, + budget or model them separately. Previously the only signal was the + `instructions` string, so hosts matched against the gem's own + `RECOMMEND_INSTRUCTIONS` / `VERDICT_INSTRUCTIONS` constants; rewording one + then sent every such host quietly down its `else` branch, mislabelling traces + rather than failing. The keyword reaches only a block that names it or + collects `**`, so judges taking `instructions:` and `prompt:` are unaffected. + (#462) ### Fixed diff --git a/lib/active_agent/evals/judge.rb b/lib/active_agent/evals/judge.rb index ed058c34..91aa9d5c 100644 --- a/lib/active_agent/evals/judge.rb +++ b/lib/active_agent/evals/judge.rb @@ -10,6 +10,21 @@ module Evals # RubyLLM.chat(model: "claude-opus-5").with_instructions(instructions).ask(prompt).content # end # + # A judge serves three different calls, and a block that accepts `kind:` is + # told which one it is serving — `:score`, `:recommend` or `:verdict` — so a + # host can trace them apart, budget them apart, or score with a cheaper model + # than it writes the verdict with: + # + # Judge.new(label: "claude-opus-5") do |instructions:, prompt:, kind:| + # model = kind == :score ? "claude-haiku-4-5" : "claude-opus-5" + # RubyLLM.chat(model: model).with_instructions(instructions).ask(prompt).content + # end + # + # The keyword is passed only to a block that names it (or collects `**`), so + # a two-keyword block written before this is unaffected. Without it the only + # signal is the `instructions` string, which means matching on the gem's own + # prose — and a reworded constant then mislabels silently instead of failing. + # # Every method returns nil when the judge fails or answers unusably, so an # evaluation degrades to rule scoring rather than aborting. class Judge @@ -23,6 +38,8 @@ class Judge # @param label [String] how reports name the judge (usually its model) # @yieldparam instructions [String] the system prompt # @yieldparam prompt [String] the user prompt + # @yieldparam kind [Symbol] which call this is — `:score`, `:recommend` or + # `:verdict`. Passed only to a block that accepts it. # @yieldreturn [String] the completion text # How much of a scenario's notes the judge reads. Where a suite's notes # are its grading rubric, a "Must not…" clause tends to come last, and a @@ -41,7 +58,7 @@ def score_criterion(criterion:, prompt:, answer:) return nil if answer.blank? guidance = criterion.dig("config", "prompt").presence || criterion["key"].to_s.humanize - parse_score(ask(SCORE_INSTRUCTIONS, <<~PROMPT)) + parse_score(ask(SCORE_INSTRUCTIONS, <<~PROMPT, :score)) Criterion: #{guidance} The user asked: @@ -64,7 +81,7 @@ def score_criterion(criterion:, prompt:, answer:) def score_task(scenario:, answer:) return nil if answer.blank? - parse_score(ask(SCORE_INSTRUCTIONS, <<~PROMPT)) + parse_score(ask(SCORE_INSTRUCTIONS, <<~PROMPT, :score)) A user asked an assistant: --- #{scenario.prompt} @@ -90,7 +107,7 @@ def recommend(scenario:, replay:, diagnosis:, available_tools: {}, instructions: "- #{call['name']}#{' (errored)' if call['error']}: #{call['arguments'].to_json.truncate(200)}" end.join("\n") - parsed = parse_object(ask(RECOMMEND_INSTRUCTIONS, <<~PROMPT)) + parsed = parse_object(ask(RECOMMEND_INSTRUCTIONS, <<~PROMPT, :recommend)) An AI agent failed one evaluation scenario. Recommend the fix. Agent instructions: @@ -146,7 +163,7 @@ def verdict(summaries, instructions: nil) "#{", faults: #{faults}" if faults.present?}" end - parsed = parse_object(ask(VERDICT_INSTRUCTIONS, <<~PROMPT)) + parsed = parse_object(ask(VERDICT_INSTRUCTIONS, <<~PROMPT, :verdict)) An AI agent ran the same scenarios under several models. Its goals: --- #{instructions.to_s.truncate(1_000).presence || '(no instructions configured)'} @@ -180,13 +197,34 @@ def suggested_tool(tool) end end - def ask(instructions, prompt) - @generate.call(instructions: instructions, prompt: prompt).to_s + def ask(instructions, prompt, kind) + @generate.call(**ask_arguments(instructions, prompt, kind)).to_s rescue StandardError => e warn_failure(e) nil end + # The block signature is public API, and every judge written before `kind:` + # existed takes exactly `instructions:` and `prompt:` — passing a third + # keyword to one of those raises ArgumentError, which `ask` would swallow + # as a judge failure, degrading the run to rule scoring. So the kind goes + # only to a block that asked for it. + def ask_arguments(instructions, prompt, kind) + arguments = { instructions: instructions, prompt: prompt } + arguments[:kind] = kind if generate_accepts_kind? + arguments + end + + # True for a block naming `kind:` or collecting `**`. Memoized because the + # answer cannot change for a given judge and `ask` runs per scored result. + def generate_accepts_kind? + return @generate_accepts_kind if defined?(@generate_accepts_kind) + + @generate_accepts_kind = @generate.parameters.any? do |type, name| + type == :keyrest || (name == :kind && (type == :key || type == :keyreq)) + end + end + def warn_failure(error) message = "[ActiveAgent::Evals] judge #{label} failed: #{error.class}: #{error.message}" if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger diff --git a/test/evals/judge_call_kind_test.rb b/test/evals/judge_call_kind_test.rb new file mode 100644 index 00000000..d2d6a438 --- /dev/null +++ b/test/evals/judge_call_kind_test.rb @@ -0,0 +1,84 @@ +# frozen_string_literal: true + +require "test_helper" +require_relative "evals_test_support" + +# A judge serves three calls — scoring an answer, recommending a fix, and +# writing the run's verdict — and a host routinely needs to tell them apart, to +# trace them separately or to grade with a cheaper model than it writes the +# verdict with. Before `kind:` the only signal was the `instructions` string, so +# hosts matched on the gem's own constants and a reworded constant mislabelled +# them silently rather than failing. +class EvalsJudgeCallKindTest < ActiveSupport::TestCase + include EvalsTestSupport + + # A judge whose block accepts the kind, recording what it was told. + def judging_judge(reply, kinds) + ActiveAgent::Evals::Judge.new(label: "judge") do |instructions:, prompt:, kind:| + _ = instructions, prompt + kinds << kind + reply + end + end + + def diagnosis + ActiveAgent::Evals::Diagnosis::Result.new(fault: "low_quality", summary: "s", recommendation: "r", evidence: {}) + end + + def test_scoring_calls_are_named_score + kinds = [] + judge = judging_judge('{"score": 0.5}', kinds) + + judge.score_task(scenario: scenario, answer: "Alice did.") + judge.score_criterion(criterion: { "key" => "tone" }, prompt: "p", answer: "a") + + assert_equal [ :score, :score ], kinds + end + + def test_a_recommendation_is_named_recommend + kinds = [] + judge = judging_judge('{"recommendation": "say more"}', kinds) + + judge.recommend(scenario: scenario, replay: replay, diagnosis: diagnosis) + + assert_equal [ :recommend ], kinds + end + + def test_the_verdict_is_named_verdict + kinds = [] + judge = judging_judge('{"winner": "gpt-5.5", "rationale": "cheapest"}', kinds) + + judge.verdict({ "gpt-5.5" => { "pass_rate" => 90 } }) + + assert_equal [ :verdict ], kinds + end + + # The block signature is public API. A judge written before `kind:` existed + # takes exactly two keywords, and passing a third raises ArgumentError — which + # `ask` catches and reports as a judge failure, silently degrading the run to + # rule scoring. So the keyword must not reach such a block at all. + def test_a_block_that_does_not_accept_the_kind_never_receives_it + seen = [] + judge = ActiveAgent::Evals::Judge.new(label: "judge") do |instructions:, prompt:| + _ = instructions, prompt + seen << :called + '{"score": 0.5}' + end + + assert_equal 0.5, judge.score_task(scenario: scenario, answer: "Alice did.") + assert_equal [ :called ], seen, "the legacy two-keyword block was not called" + end + + def test_a_block_collecting_keywords_receives_the_kind + seen = {} + judge = ActiveAgent::Evals::Judge.new(label: "judge") do |**options| + seen = options + '{"score": 0.5}' + end + + judge.score_task(scenario: scenario, answer: "Alice did.") + + assert_equal :score, seen[:kind] + assert seen.key?(:instructions) && seen.key?(:prompt) + end +end