feat(evals): tell a judge block which call it is serving

CHANGELOG.md

@@ -40,6 +40,15 @@ Releases `activeagent` and `actionagent` 1.6.3 from one tag.
   takes on no telemetry dependency and `require "active_agent/evals"` still
   loads on its own. Hand the object to `Runner.new(around_evaluation:)`
   directly; a plain lambda there keeps working unchanged.
+- A `Judge` block that accepts `kind:` is told which of the judge's three calls
+  it is serving — `:score`, `:recommend` or `:verdict` — so a host can trace,
+  budget or model them separately. Previously the only signal was the
+  `instructions` string, so hosts matched against the gem's own
+  `RECOMMEND_INSTRUCTIONS` / `VERDICT_INSTRUCTIONS` constants; rewording one
+  then sent every such host quietly down its `else` branch, mislabelling traces
+  rather than failing. The keyword reaches only a block that names it or
+  collects `**`, so judges taking `instructions:` and `prompt:` are unaffected.
+  (#462)
 
 ### Fixed
 

lib/active_agent/evals/judge.rb

@@ -10,6 +10,21 @@ module Evals
     #     RubyLLM.chat(model: "claude-opus-5").with_instructions(instructions).ask(prompt).content
     #   end
     #
+    # A judge serves three different calls, and a block that accepts `kind:` is
+    # told which one it is serving — `:score`, `:recommend` or `:verdict` — so a
+    # host can trace them apart, budget them apart, or score with a cheaper model
+    # than it writes the verdict with:
+    #
+    #   Judge.new(label: "claude-opus-5") do |instructions:, prompt:, kind:|
+    #     model = kind == :score ? "claude-haiku-4-5" : "claude-opus-5"
+    #     RubyLLM.chat(model: model).with_instructions(instructions).ask(prompt).content
+    #   end
+    #
+    # The keyword is passed only to a block that names it (or collects `**`), so
+    # a two-keyword block written before this is unaffected. Without it the only
+    # signal is the `instructions` string, which means matching on the gem's own
+    # prose — and a reworded constant then mislabels silently instead of failing.
+    #
     # Every method returns nil when the judge fails or answers unusably, so an
     # evaluation degrades to rule scoring rather than aborting.
     class Judge
@@ -23,6 +38,8 @@ class Judge
       # @param label [String] how reports name the judge (usually its model)
       # @yieldparam instructions [String] the system prompt
       # @yieldparam prompt [String] the user prompt
+      # @yieldparam kind [Symbol] which call this is — `:score`, `:recommend` or
+      #   `:verdict`. Passed only to a block that accepts it.
       # @yieldreturn [String] the completion text
       # How much of a scenario's notes the judge reads. Where a suite's notes
       # are its grading rubric, a "Must not…" clause tends to come last, and a
@@ -41,7 +58,7 @@ def score_criterion(criterion:, prompt:, answer:)
         return nil if answer.blank?
 
         guidance = criterion.dig("config", "prompt").presence || criterion["key"].to_s.humanize
-        parse_score(ask(SCORE_INSTRUCTIONS, <<~PROMPT))
+        parse_score(ask(SCORE_INSTRUCTIONS, <<~PROMPT, :score))
           Criterion: #{guidance}
 
           The user asked:
@@ -64,7 +81,7 @@ def score_criterion(criterion:, prompt:, answer:)
       def score_task(scenario:, answer:)
         return nil if answer.blank?
 
-        parse_score(ask(SCORE_INSTRUCTIONS, <<~PROMPT))
+        parse_score(ask(SCORE_INSTRUCTIONS, <<~PROMPT, :score))
           A user asked an assistant:
           ---
           #{scenario.prompt}
@@ -90,7 +107,7 @@ def recommend(scenario:, replay:, diagnosis:, available_tools: {}, instructions:
           "- #{call['name']}#{' (errored)' if call['error']}: #{call['arguments'].to_json.truncate(200)}"
         end.join("\n")
 
-        parsed = parse_object(ask(RECOMMEND_INSTRUCTIONS, <<~PROMPT))
+        parsed = parse_object(ask(RECOMMEND_INSTRUCTIONS, <<~PROMPT, :recommend))
           An AI agent failed one evaluation scenario. Recommend the fix.
 
           Agent instructions:
@@ -146,7 +163,7 @@ def verdict(summaries, instructions: nil)
             "#{", faults: #{faults}" if faults.present?}"
         end
 
-        parsed = parse_object(ask(VERDICT_INSTRUCTIONS, <<~PROMPT))
+        parsed = parse_object(ask(VERDICT_INSTRUCTIONS, <<~PROMPT, :verdict))
           An AI agent ran the same scenarios under several models. Its goals:
           ---
           #{instructions.to_s.truncate(1_000).presence || '(no instructions configured)'}
@@ -180,13 +197,34 @@ def suggested_tool(tool)
         end
       end
 
-      def ask(instructions, prompt)
-        @generate.call(instructions: instructions, prompt: prompt).to_s
+      def ask(instructions, prompt, kind)
+        @generate.call(**ask_arguments(instructions, prompt, kind)).to_s
       rescue StandardError => e
         warn_failure(e)
         nil
       end
 
+      # The block signature is public API, and every judge written before `kind:`
+      # existed takes exactly `instructions:` and `prompt:` — passing a third
+      # keyword to one of those raises ArgumentError, which `ask` would swallow
+      # as a judge failure, degrading the run to rule scoring. So the kind goes
+      # only to a block that asked for it.
+      def ask_arguments(instructions, prompt, kind)
+        arguments = { instructions: instructions, prompt: prompt }
+        arguments[:kind] = kind if generate_accepts_kind?
+        arguments
+      end
+
+      # True for a block naming `kind:` or collecting `**`. Memoized because the
+      # answer cannot change for a given judge and `ask` runs per scored result.
+      def generate_accepts_kind?
+        return @generate_accepts_kind if defined?(@generate_accepts_kind)
+
+        @generate_accepts_kind = @generate.parameters.any? do |type, name|
+          type == :keyrest || (name == :kind && (type == :key || type == :keyreq))
+        end
+      end
+
       def warn_failure(error)
         message = "[ActiveAgent::Evals] judge #{label} failed: #{error.class}: #{error.message}"
         if defined?(Rails) && Rails.respond_to?(:logger) && Rails.logger

test/evals/judge_call_kind_test.rb

@@ -0,0 +1,84 @@
+# frozen_string_literal: true
+
+require "test_helper"
+require_relative "evals_test_support"
+
+# A judge serves three calls — scoring an answer, recommending a fix, and
+# writing the run's verdict — and a host routinely needs to tell them apart, to
+# trace them separately or to grade with a cheaper model than it writes the
+# verdict with. Before `kind:` the only signal was the `instructions` string, so
+# hosts matched on the gem's own constants and a reworded constant mislabelled
+# them silently rather than failing.
+class EvalsJudgeCallKindTest < ActiveSupport::TestCase
+  include EvalsTestSupport
+
+  # A judge whose block accepts the kind, recording what it was told.
+  def judging_judge(reply, kinds)
+    ActiveAgent::Evals::Judge.new(label: "judge") do |instructions:, prompt:, kind:|
+      _ = instructions, prompt
+      kinds << kind
+      reply
+    end
+  end
+
+  def diagnosis
+    ActiveAgent::Evals::Diagnosis::Result.new(fault: "low_quality", summary: "s", recommendation: "r", evidence: {})
+  end
+
+  def test_scoring_calls_are_named_score
+    kinds = []
+    judge = judging_judge('{"score": 0.5}', kinds)
+
+    judge.score_task(scenario: scenario, answer: "Alice did.")
+    judge.score_criterion(criterion: { "key" => "tone" }, prompt: "p", answer: "a")
+
+    assert_equal [ :score, :score ], kinds
+  end
+
+  def test_a_recommendation_is_named_recommend
+    kinds = []
+    judge = judging_judge('{"recommendation": "say more"}', kinds)
+
+    judge.recommend(scenario: scenario, replay: replay, diagnosis: diagnosis)
+
+    assert_equal [ :recommend ], kinds
+  end
+
+  def test_the_verdict_is_named_verdict
+    kinds = []
+    judge = judging_judge('{"winner": "gpt-5.5", "rationale": "cheapest"}', kinds)
+
+    judge.verdict({ "gpt-5.5" => { "pass_rate" => 90 } })
+
+    assert_equal [ :verdict ], kinds
+  end
+
+  # The block signature is public API. A judge written before `kind:` existed
+  # takes exactly two keywords, and passing a third raises ArgumentError — which
+  # `ask` catches and reports as a judge failure, silently degrading the run to
+  # rule scoring. So the keyword must not reach such a block at all.
+  def test_a_block_that_does_not_accept_the_kind_never_receives_it
+    seen = []
+    judge = ActiveAgent::Evals::Judge.new(label: "judge") do |instructions:, prompt:|
+      _ = instructions, prompt
+      seen << :called
+      '{"score": 0.5}'
+    end
+
+    assert_equal 0.5, judge.score_task(scenario: scenario, answer: "Alice did.")
+    assert_equal [ :called ], seen, "the legacy two-keyword block was not called"
+  end
+
+  def test_a_block_collecting_keywords_receives_the_kind
+    seen = {}
+    judge = ActiveAgent::Evals::Judge.new(label: "judge") do |**options|
+      seen = options
+      '{"score": 0.5}'
+    end
+
+    judge.score_task(scenario: scenario, answer: "Alice did.")
+
+    assert_equal :score, seen[:kind]
+    assert seen.key?(:instructions) && seen.key?(:prompt)
+  end
+end