feat(llmobs): add annotateAgentManifest manual API to LLMObs SDK

dd-java-agent/agent-llmobs/src/main/java/datadog/trace/llmobs/domain/DDLLMObsSpan.java

@@ -41,6 +41,8 @@ public class DDLLMObsSpan implements LLMObsSpan {
   private static final String SPAN_KIND = LLMOBS_TAG_PREFIX + Tags.SPAN_KIND;
   private static final String METADATA = LLMOBS_TAG_PREFIX + LLMObsTags.METADATA;
   private static final String TOOL_DEFINITIONS = LLMOBS_TAG_PREFIX + LLMObsTags.TOOL_DEFINITIONS;
+  private static final String AGENT_MANIFEST = LLMOBS_TAG_PREFIX + LLMObsTags.AGENT_MANIFEST;
+  private static final String MANUAL_FRAMEWORK = "AgentObs SDK";
   private static final String PROMPT_TRACKING_INSTRUMENTATION_METHOD =
       LLMOBS_TAG_PREFIX + "prompt_tracking_instrumentation_method";
   private static final String INSTRUMENTATION_METHOD_ANNOTATED = "annotated";
@@ -291,6 +293,64 @@ public void annotatePrompt(LLMObs.Prompt prompt) {
     span.setTag(PROMPT_TRACKING_INSTRUMENTATION_METHOD, INSTRUMENTATION_METHOD_ANNOTATED);
   }
 
+  @Override
+  public void annotateAgentManifest(LLMObs.AgentManifest manifest) {
+    if (finished || manifest == null) {
+      return;
+    }
+    if (!Tags.LLMOBS_AGENT_SPAN_KIND.equals(spanKind)) {
+      LOGGER.warn(
+          "dropping agent manifest on non-agent span kind; annotateAgentManifest is only supported for agent spans");
+      return;
+    }
+    Map<String, Object> manifestMap = buildManifestMap(manifest);
+    if (!manifestMap.isEmpty()) {
+      manifestMap.put("framework", MANUAL_FRAMEWORK);
+      span.setTag(AGENT_MANIFEST, manifestMap);
+    }
+  }
+
+  private Map<String, Object> buildManifestMap(LLMObs.AgentManifest manifest) {
+    Map<String, Object> map = new LinkedHashMap<>();
+    CharSequence sn = span.getSpanName();
+    String name =
+        manifest.getName() != null ? manifest.getName() : (sn != null ? sn.toString() : null);
+    if (name != null && !name.isEmpty()) {
+      map.put("name", name);
+    }
+    if (manifest.getInstructions() != null && !manifest.getInstructions().isEmpty()) {
+      map.put("instructions", manifest.getInstructions());
+    }
+    if (manifest.getModel() != null && !manifest.getModel().isEmpty()) {
+      map.put("model", manifest.getModel());
+    }
+    if (manifest.getModelSettings() != null && !manifest.getModelSettings().isEmpty()) {
+      map.put("model_settings", new LinkedHashMap<>(manifest.getModelSettings()));
+    }
+    if (manifest.getTools() != null && !manifest.getTools().isEmpty()) {
+      List<Map<String, Object>> toolList = new ArrayList<>();
+      for (LLMObs.AgentTool tool : manifest.getTools()) {
+        if (tool == null || tool.getName() == null || tool.getName().isEmpty()) {
+          LOGGER.warn("agent manifest tool missing required name; skipping");
+          continue;
+        }
+        Map<String, Object> toolMap = new LinkedHashMap<>();
+        toolMap.put("name", tool.getName());
+        if (tool.getDescription() != null) {
+          toolMap.put("description", tool.getDescription());
+        }
+        if (tool.getParameters() != null && !tool.getParameters().isEmpty()) {
+          toolMap.put("parameters", new LinkedHashMap<>(tool.getParameters()));
+        }
+        toolList.add(toolMap);
+      }
+      if (!toolList.isEmpty()) {
+        map.put("tools", toolList);
+      }
+    }
+    return map;
+  }
+
   private static Map<String, Object> copyStringKeyedMap(Map<?, ?> source) {
     Map<String, Object> copy = new LinkedHashMap<>();
     for (Map.Entry<?, ?> entry : source.entrySet()) {

dd-java-agent/agent-llmobs/src/test/groovy/datadog/trace/llmobs/domain/DDLLMObsSpanTest.groovy

@@ -63,6 +63,7 @@ class DDLLMObsSpanTest  extends DDSpecification{
   private static final String OUTPUT = LLMOBS_TAG_PREFIX + "output"
   private static final String METADATA = LLMOBS_TAG_PREFIX + LLMObsTags.METADATA
   private static final String TOOL_DEFINITIONS = LLMOBS_TAG_PREFIX + LLMObsTags.TOOL_DEFINITIONS
+  private static final String AGENT_MANIFEST = LLMOBS_TAG_PREFIX + "agent_manifest"
   private static final String PROMPT_TRACKING_INSTRUMENTATION_METHOD =
   LLMOBS_TAG_PREFIX + "prompt_tracking_instrumentation_method"
 
@@ -791,6 +792,159 @@ class DDLLMObsSpanTest  extends DDSpecification{
     innerSpan.getTag(LLMOBS_TAG_PREFIX + "owner") == "ml-platform"
   }
 
+  def "agent manifest full annotation sets correct tag"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent")
+    def settings = [temperature: 0.7, max_tokens: 1024]
+    def params = [city: [type: "string"]]
+    def tools = [LLMObs.AgentTool.from("get_weather", "Look up weather", params)]
+    def manifest = LLMObs.AgentManifest.builder()
+      .name("travel_desk")
+      .instructions("Book travel.")
+      .model("gpt-4o")
+      .modelSettings(settings)
+      .tools(tools)
+      .build()
+
+    when:
+    test.annotateAgentManifest(manifest)
+
+    then:
+    def innerSpan = (AgentSpan) test.span
+    def stored = (Map) innerSpan.getTag(AGENT_MANIFEST)
+    stored["name"] == "travel_desk"
+    stored["instructions"] == "Book travel."
+    stored["model"] == "gpt-4o"
+    stored["framework"] == "AgentObs SDK"
+    def ms = (Map) stored["model_settings"]
+    ms["temperature"] == 0.7
+    ms["max_tokens"] == 1024
+    def toolList = (List) stored["tools"]
+    toolList.size() == 1
+    toolList[0]["name"] == "get_weather"
+    toolList[0]["description"] == "Look up weather"
+    toolList[0]["parameters"] == [city: [type: "string"]]
+
+    cleanup:
+    test.finish()
+  }
+
+  def "agent manifest name defaults to span name when not provided"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent")
+    def manifest = LLMObs.AgentManifest.builder().instructions("Do something.").build()
+
+    when:
+    test.annotateAgentManifest(manifest)
+
+    then:
+    def innerSpan = (AgentSpan) test.span
+    def stored = (Map) innerSpan.getTag(AGENT_MANIFEST)
+    stored["name"] == "my-agent"
+    stored["instructions"] == "Do something."
+    stored["framework"] == "AgentObs SDK"
+
+    cleanup:
+    test.finish()
+  }
+
+  def "agent manifest drops tool with null name"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent")
+    def validTool = LLMObs.AgentTool.from("valid-tool")
+    def badTool = LLMObs.AgentTool.from(null)
+    def manifest = LLMObs.AgentManifest.builder()
+      .tools([badTool, validTool])
+      .build()
+
+    when:
+    test.annotateAgentManifest(manifest)
+
+    then:
+    def innerSpan = (AgentSpan) test.span
+    def stored = (Map) innerSpan.getTag(AGENT_MANIFEST)
+    def toolList = (List) stored["tools"]
+    toolList.size() == 1
+    toolList[0]["name"] == "valid-tool"
+
+    cleanup:
+    test.finish()
+  }
+
+  def "agent manifest on non-agent span is silently dropped"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_LLM_SPAN_KIND, "llm-span")
+    def manifest = LLMObs.AgentManifest.builder().name("agent").build()
+
+    when:
+    test.annotateAgentManifest(manifest)
+
+    then:
+    def innerSpan = (AgentSpan) test.span
+    innerSpan.getTag(AGENT_MANIFEST) == null
+
+    cleanup:
+    test.finish()
+  }
+
+  def "second annotateAgentManifest call overwrites the first"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent")
+    def first = LLMObs.AgentManifest.builder().name("first").instructions("v1").build()
+    def second = LLMObs.AgentManifest.builder().name("second").model("gpt-4o").build()
+
+    when:
+    test.annotateAgentManifest(first)
+    test.annotateAgentManifest(second)
+
+    then:
+    def innerSpan = (AgentSpan) test.span
+    def stored = (Map) innerSpan.getTag(AGENT_MANIFEST)
+    stored["name"] == "second"
+    stored["model"] == "gpt-4o"
+    !stored.containsKey("instructions")
+
+    cleanup:
+    test.finish()
+  }
+
+  def "agent manifest model_settings forwarded as-is"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent")
+    def manifest = LLMObs.AgentManifest.builder()
+      .name("agent")
+      .modelSettings([temperature: 0.5, custom_key: "custom_val"])
+      .build()
+
+    when:
+    test.annotateAgentManifest(manifest)
+
+    then:
+    def innerSpan = (AgentSpan) test.span
+    def stored = (Map) innerSpan.getTag(AGENT_MANIFEST)
+    def ms = (Map) stored["model_settings"]
+    ms["temperature"] == 0.5
+    ms["custom_key"] == "custom_val"
+
+    cleanup:
+    test.finish()
+  }
+
+  def "annotateAgentManifest null manifest is ignored"() {
+    setup:
+    def test = llmObsSpan(Tags.LLMOBS_AGENT_SPAN_KIND, "my-agent")
+
+    when:
+    test.annotateAgentManifest(null)
+
+    then:
+    def innerSpan = (AgentSpan) test.span
+    innerSpan.getTag(AGENT_MANIFEST) == null
+
+    cleanup:
+    test.finish()
+  }
+
   private LLMObsSpan llmObsSpan(String kind, name) {
     llmObsSpan(kind, name, null)
   }

dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObs.java

@@ -1109,4 +1109,139 @@ public Double getScore() {
       return score;
     }
   }
+
+  /** A tool declared in an agent manifest. */
+  public static final class AgentTool {
+    private final String name;
+    private final String description;
+    private final Map<String, Object> parameters;
+
+    public static AgentTool from(String name) {
+      return new AgentTool(name, null, null);
+    }
+
+    public static AgentTool from(
+        String name, @Nullable String description, @Nullable Map<String, Object> parameters) {
+      return new AgentTool(name, description, parameters);
+    }
+
+    private AgentTool(String name, String description, Map<String, Object> parameters) {
+      this.name = name;
+      this.description = description;
+      this.parameters =
+          parameters == null ? null : Collections.unmodifiableMap(new LinkedHashMap<>(parameters));
+    }
+
+    public String getName() {
+      return name;
+    }
+
+    @Nullable
+    public String getDescription() {
+      return description;
+    }
+
+    @Nullable
+    public Map<String, Object> getParameters() {
+      return parameters;
+    }
+  }
+
+  /**
+   * Declares the configuration of an agent span: what model it calls, what instructions it runs
+   * with, and which tools it has available.
+   *
+   * <p>Build via {@link AgentManifest#builder()} and pass to {@link
+   * LLMObsSpan#annotateAgentManifest(AgentManifest)}. Only applied on agent spans; ignored on other
+   * span kinds. A subsequent call on the same span overwrites the previous manifest.
+   */
+  public static final class AgentManifest {
+    private final String name;
+    private final String instructions;
+    private final String model;
+    private final Map<String, Object> modelSettings;
+    private final List<AgentTool> tools;
+
+    public static Builder builder() {
+      return new Builder();
+    }
+
+    private AgentManifest(Builder builder) {
+      this.name = builder.name;
+      this.instructions = builder.instructions;
+      this.model = builder.model;
+      this.modelSettings =
+          builder.modelSettings == null
+              ? null
+              : Collections.unmodifiableMap(new LinkedHashMap<>(builder.modelSettings));
+      this.tools =
+          builder.tools == null
+              ? null
+              : Collections.unmodifiableList(new ArrayList<>(builder.tools));
+    }
+
+    @Nullable
+    public String getName() {
+      return name;
+    }
+
+    @Nullable
+    public String getInstructions() {
+      return instructions;
+    }
+
+    @Nullable
+    public String getModel() {
+      return model;
+    }
+
+    @Nullable
+    public Map<String, Object> getModelSettings() {
+      return modelSettings;
+    }
+
+    @Nullable
+    public List<AgentTool> getTools() {
+      return tools;
+    }
+
+    public static final class Builder {
+      private String name;
+      private String instructions;
+      private String model;
+      private Map<String, Object> modelSettings;
+      private List<AgentTool> tools;
+
+      private Builder() {}
+
+      public Builder name(String name) {
+        this.name = name;
+        return this;
+      }
+
+      public Builder instructions(String instructions) {
+        this.instructions = instructions;
+        return this;
+      }
+
+      public Builder model(String model) {
+        this.model = model;
+        return this;
+      }
+
+      public Builder modelSettings(Map<String, Object> modelSettings) {
+        this.modelSettings = modelSettings;
+        return this;
+      }
+
+      public Builder tools(List<AgentTool> tools) {
+        this.tools = tools;
+        return this;
+      }
+
+      public AgentManifest build() {
+        return new AgentManifest(this);
+      }
+    }
+  }
 }

dd-trace-api/src/main/java/datadog/trace/api/llmobs/LLMObsSpan.java

@@ -59,6 +59,15 @@ default void annotatePrompt(LLMObs.Prompt prompt) {}
    */
   default void setToolDefinitions(List<LLMObs.ToolDefinition> toolDefinitions) {}
 
+  /**
+   * Annotate an agent span with its manifest configuration.
+   *
+   * <p>This annotation is ignored for non-agent spans.
+   *
+   * @param agentManifest The agent manifest configuration
+   */
+  default void annotateAgentManifest(LLMObs.AgentManifest agentManifest) {}
+
   /**
    * Annotate the span with metadata
    *