From 4f0de87aea8aff7bae10a2bcd1b9575839312910 Mon Sep 17 00:00:00 2001 From: "mars.yu" Date: Wed, 26 Aug 2026 18:03:32 +0800 Subject: [PATCH] Agent test: count utilities as mock targets GET /agent-test/mock-targets enumerated Functions, SecondaryFunctions and McpTools only, so an agent whose whole toolset comes from utilities -- Lessen Work Order Summary, Property Summary -- reported no callable functions at all. Utilities are expanded into SecondaryFunctions by BasicAgentHook.OnAgentUtilityLoaded at conversation time, and IAgentService.GetAgent, which is what every caller here holds, is a plain repository read that never runs that hook. Cost of the gap: a case authored against such an agent blocks its own tools on the first run, and the case editor's tool picker has nothing to offer. The filter mirrors the hook -- a disabled utility is off, only a `util-` prefixed name is ever loaded as a function, and the real FunctionDef is read off UtilityAssistant rather than the agent's own declaration. VisibilityExpression is deliberately not mirrored: it needs the conversation's render data and cannot be evaluated against a stored agent, so a conditionally-visible utility is offered and may turn out not to load. Co-Authored-By: Claude Opus 5 --- .../Controllers/AgentTestController.cs | 4 +- .../Services/LlmCaseAuthor.cs | 3 +- .../Services/MockTargetCatalogue.cs | 48 +++++++++++++++++-- 3 files changed, 50 insertions(+), 5 deletions(-) diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Controllers/AgentTestController.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Controllers/AgentTestController.cs index f507f545d..770c24c0c 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Controllers/AgentTestController.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Controllers/AgentTestController.cs @@ -2,6 +2,7 @@ using MongoDB.Bson; using MongoDB.Bson.Serialization; using BotSharp.Abstraction.Agents; +using BotSharp.Abstraction.Agents.Enums; using BotSharp.Abstraction.MLTasks; using BotSharp.Abstraction.Infrastructures.Attributes; using Microsoft.AspNetCore.Authorization; @@ -672,7 +673,8 @@ public async Task>> GetMockTargets([FromQuery] string? // Same derivation the authoring prompt uses, so the names the editor offers and the names // a model is allowed to mock can never diverge. Still names only on the wire: the case // editor's pickers consume a plain string list. - return MockTargetCatalogue.Names(agent); + var utilityAssistant = await _agents.GetAgent(BuiltInAgentId.UtilityAssistant); + return MockTargetCatalogue.Names(agent, utilityAssistant); } /// diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/LlmCaseAuthor.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/LlmCaseAuthor.cs index e023ef878..06b25a464 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/LlmCaseAuthor.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/LlmCaseAuthor.cs @@ -104,7 +104,8 @@ public async Task AuthorAsync( ?? throw new CaseAuthorUnavailableException( $"agent {agentId} not found, so there is nothing to author a case against"); - var targets = MockTargetCatalogue.Describe(agent); + var utilityAssistant = await _agents.GetAgent(BuiltInAgentId.UtilityAssistant); + var targets = MockTargetCatalogue.Describe(agent, utilityAssistant); var existingCases = await _repo.ListCasesAsync(suite.Id); var grounding = await LoadGroundingAsync(suite.Id, request.CaseId, ct); var agentNames = await AgentNamesAsync(); diff --git a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/MockTargetCatalogue.cs b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/MockTargetCatalogue.cs index 4a180e9d5..be564a877 100644 --- a/src/Plugins/BotSharp.Plugin.AgentTesting/Services/MockTargetCatalogue.cs +++ b/src/Plugins/BotSharp.Plugin.AgentTesting/Services/MockTargetCatalogue.cs @@ -17,9 +17,13 @@ namespace BotSharp.Plugin.AgentTesting.Services; /// public static class MockTargetCatalogue { + /// The prefix a utility item's function name must carry to be loaded as a function. + /// Same constant, same ordinal comparison, as BasicAgentHook.UTIL_PREFIX. + private const string UtilityPrefix = "util-"; + /// Function names only, sorted and de-duplicated. The wire shape the UI consumes. - public static List Names(Agent agent) - => Describe(agent).Select(t => t.Name).ToList(); + public static List Names(Agent agent, Agent? utilityAssistant = null) + => Describe(agent, utilityAssistant).Select(t => t.Name).ToList(); /// /// Every callable function with whatever description and parameter shape the agent definition @@ -30,7 +34,12 @@ public static List Names(Agent agent) /// here -- it is why an MCP mock's argsMatchJson is more likely to need a human fix than a /// plugin function's. /// - public static List Describe(Agent agent) + /// The agent whose callable surface is wanted. + /// + /// The UtilityAssistant agent, which is where a utility item's real FunctionDef lives -- an agent + /// only names the utilities it turns on. Pass null and utilities still appear, name-only. + /// + public static List Describe(Agent agent, Agent? utilityAssistant = null) { var targets = new List(); @@ -46,6 +55,39 @@ public static List Describe(Agent agent) targets.Add(new MockTargetInfo(fn!.Name, null, null)); } + // Utilities are the third way an agent gets a function, and the one that does not show up + // anywhere in Functions/SecondaryFunctions on a stored agent: BasicAgentHook.OnAgentUtilityLoaded + // expands them into SecondaryFunctions at conversation time, and IAgentService.GetAgent -- what + // every caller here holds -- is a plain repository read that never runs that hook. Left out, an + // agent whose whole toolset is utilities (Lessen Work Order Summary, Property Summary) reports + // no callable functions at all, and a case authored against it blocks its own tools on the + // first run. + // + // The filter mirrors that hook: a disabled utility is off, and only a `util-` prefixed name is + // ever loaded as a function. What is deliberately NOT mirrored is VisibilityExpression, which + // needs the conversation's render data and cannot be evaluated against a stored agent -- a + // conditionally-visible utility is offered here and may turn out not to load at run time. + foreach (var utility in agent.Utilities ?? []) + { + if (utility == null || utility.Disabled) continue; + + foreach (var item in utility.Items ?? []) + { + var name = item?.FunctionName; + if (string.IsNullOrWhiteSpace(name) || !name.StartsWith(UtilityPrefix, StringComparison.Ordinal)) continue; + + // The definition the agent will actually be given, not the agent's own declaration -- + // the description and parameter list live on UtilityAssistant. + var definition = utilityAssistant?.Functions? + .FirstOrDefault(f => string.Equals(f?.Name, name, StringComparison.OrdinalIgnoreCase)); + + targets.Add(new MockTargetInfo( + name!, + definition?.Description ?? item!.Description, + ParameterSummary(definition?.Parameters))); + } + } + // First entry wins on a duplicate name: a function declared both primary and secondary is // one function, and the primary declaration is the one with the fuller definition. return targets