fix(annotated)!: integrate lossless request codecs

crates/adaptive/src/acg/ir_builder.rs

@@ -37,6 +37,16 @@ pub fn build_prompt_ir(request: &AnnotatedLlmRequest) -> Result<PromptIR> {
     let mut sequence_index: u32 = 0;
     let mut inserted_tool_blocks = false;
 
+    if let Some(instructions) = &request.instructions {
+        blocks.push(build_text_block(
+            &mut sequence_index,
+            instructions,
+            PromptRole::System,
+            ProvenanceLabel::System,
+            Some("instructions"),
+        ));
+    }
+
     for message in &request.messages {
         if should_insert_tool_blocks_before_message(inserted_tool_blocks, request, message) {
             append_tool_schema_blocks(&mut blocks, &mut sequence_index, request.tools.as_deref())?;
@@ -71,7 +81,9 @@ fn should_insert_tool_blocks_before_message(
     request: &AnnotatedLlmRequest,
     message: &Message,
 ) -> bool {
-    !inserted_tool_blocks && !matches!(message, Message::System { .. }) && request.tools.is_some()
+    !inserted_tool_blocks
+        && !matches!(message, Message::System { .. } | Message::Developer { .. })
+        && request.tools.is_some()
 }
 
 fn append_message_blocks(
@@ -87,6 +99,13 @@ fn append_message_blocks(
             ProvenanceLabel::System,
             None,
         )),
+        Message::Developer { content, .. } => blocks.push(build_text_block(
+            sequence_index,
+            content,
+            PromptRole::System,
+            ProvenanceLabel::Developer,
+            Some("developer"),
+        )),
         Message::User { content, .. } => blocks.push(build_text_block(
             sequence_index,
             content,
@@ -112,6 +131,37 @@ fn append_message_blocks(
             content,
             tool_call_id,
         )),
+        Message::Function { content, name } => {
+            let content = MessageContent::Text(content.clone().unwrap_or_default());
+            blocks.push(build_text_block(
+                sequence_index,
+                &content,
+                PromptRole::Tool,
+                ProvenanceLabel::Tool,
+                Some(name),
+            ));
+        }
+        Message::ToolCallItem { name, .. } => blocks.push(build_serialized_message_block(
+            sequence_index,
+            message,
+            PromptRole::Assistant,
+            ProvenanceLabel::Developer,
+            Some(name),
+        )?),
+        Message::ToolResultItem { call_id, .. } => blocks.push(build_serialized_message_block(
+            sequence_index,
+            message,
+            PromptRole::Tool,
+            ProvenanceLabel::Tool,
+            Some(call_id),
+        )?),
+        Message::ProviderNative { kind, .. } => blocks.push(build_serialized_message_block(
+            sequence_index,
+            message,
+            PromptRole::User,
+            ProvenanceLabel::Developer,
+            Some(kind),
+        )?),
     }
 
     Ok(())
@@ -147,15 +197,36 @@ fn extract_text(content: &MessageContent) -> String {
         MessageContent::Text(text) => text.clone(),
         MessageContent::Parts(parts) => parts
             .iter()
-            .filter_map(|part| match part {
-                ContentPart::Text { text } => Some(text.as_str()),
-                ContentPart::ImageUrl { .. } => None,
+            .map(|part| match part {
+                ContentPart::Text { text, .. } => text.clone(),
+                ContentPart::Refusal { refusal, .. } => refusal.clone(),
+                ContentPart::ImageUrl { .. } | ContentPart::Image { .. } => String::new(),
+                ContentPart::Audio { .. }
+                | ContentPart::File { .. }
+                | ContentPart::ToolUse { .. }
+                | ContentPart::ToolResult { .. }
+                | ContentPart::ProviderNative { .. } => {
+                    serde_json::to_string(part).unwrap_or_default()
+                }
             })
+            .filter(|text| !text.is_empty())
             .collect::<Vec<_>>()
             .join("\n"),
     }
 }
 
+fn build_serialized_message_block(
+    seq: &mut u32,
+    message: &Message,
+    role: PromptRole,
+    provenance: ProvenanceLabel,
+    suffix: Option<&str>,
+) -> Result<PromptBlock> {
+    let value = serde_json::to_value(message)?;
+    let content = MessageContent::Text(canonicalize_value(&value)?);
+    Ok(build_text_block(seq, &content, role, provenance, suffix))
+}
+
 fn generate_span_id(role: PromptRole, index: u32, suffix: Option<&str>) -> SpanId {
     let role_str = match role {
         PromptRole::System => "system",
@@ -257,7 +328,7 @@ fn build_tool_schema_block(seq: &mut u32, tool: &ToolDefinition) -> Result<Promp
     let canonical = canonicalize_value(&tool_value)?;
     let content = normalize_whitespace(&canonical);
     let index = *seq;
-    let span_id = generate_span_id(PromptRole::System, index, Some(&tool.function.name));
+    let span_id = generate_span_id(PromptRole::System, index, Some(tool_definition_name(tool)));
     *seq += 1;
 
     Ok(PromptBlock {
@@ -279,13 +350,20 @@ fn build_tool_schema_hashes(tools: &[ToolDefinition]) -> Result<Vec<ToolSchemaHa
             let value = serde_json::to_value(tool_definition)?;
             let canonical = canonicalize_value(&value)?;
             Ok(ToolSchemaHash {
-                tool_name: tool_definition.function.name.clone(),
+                tool_name: tool_definition_name(tool_definition).to_string(),
                 schema_hash: sha256_hex(&canonical),
             })
         })
         .collect()
 }
 
+fn tool_definition_name(tool: &ToolDefinition) -> &str {
+    match tool {
+        ToolDefinition::Function { function, .. } => &function.name,
+        ToolDefinition::ProviderNative { kind, .. } => kind,
+    }
+}
+
 fn compute_request_hash(request: &AnnotatedLlmRequest) -> Result<String> {
     let value = serde_json::to_value(request)?;
     let canonical = canonicalize_value(&value)?;

crates/adaptive/src/acg_profile.rs

@@ -76,26 +76,47 @@ fn derive_key_parts(annotated_request: &AnnotatedLlmRequest) -> AcgKeyParts<'_>
 fn message_role_tag(message: &Message) -> &'static str {
     match message {
         Message::System { .. } => "system",
+        Message::Developer { .. } => "developer",
         Message::User { .. } => "user",
         Message::Assistant { .. } => "assistant",
         Message::Tool { .. } => "tool",
+        Message::Function { .. } => "function",
+        Message::ToolCallItem { .. } => "tool_call",
+        Message::ToolResultItem { .. } => "tool_result",
+        Message::ProviderNative { .. } => "provider_native",
     }
 }
 
 fn system_prompt_fingerprint(annotated_request: &AnnotatedLlmRequest) -> String {
-    let system_content = annotated_request
-        .messages
-        .iter()
-        .filter_map(|message| match message {
-            Message::System { content, .. } => Some(extract_text(content)),
-            _ => None,
-        })
-        .collect::<Vec<_>>()
-        .join("\n");
+    let mut system_content = Vec::new();
+    if let Some(instructions) = &annotated_request.instructions {
+        system_content.push(serde_json::json!({
+            "source": "instructions",
+            "content": instructions,
+        }));
+    }
+    system_content.extend(
+        annotated_request
+            .messages
+            .iter()
+            .filter_map(|message| match message {
+                Message::System { content, .. } => Some(serde_json::json!({
+                    "source": "message",
+                    "role": "system",
+                    "content": content,
+                })),
+                Message::Developer { content, .. } => Some(serde_json::json!({
+                    "source": "message",
+                    "role": "developer",
+                    "content": content,
+                })),
+                _ => None,
+            }),
+    );
     if system_content.is_empty() {
         "no-system".to_string()
     } else {
-        sha256_hex(&system_content)
+        hash_canonical_json(&serde_json::Value::Array(system_content))
     }
 }
 
@@ -148,7 +169,7 @@ fn learning_seed_fingerprint(annotated_request: &AnnotatedLlmRequest) -> String
         .messages
         .iter()
         .find_map(|message| match message {
-            Message::System { .. } => None,
+            Message::System { .. } | Message::Developer { .. } => None,
             Message::User { content, .. } => {
                 Some(format!("user:{}", sha256_hex(&extract_text(content))))
             }
@@ -160,10 +181,46 @@ fn learning_seed_fingerprint(annotated_request: &AnnotatedLlmRequest) -> String
             Message::Tool { content, .. } => {
                 Some(format!("tool:{}", sha256_hex(&extract_text(content))))
             }
+            Message::Function { content, name } => Some(format!(
+                "function:{}",
+                hash_canonical_json(&serde_json::json!({
+                    "name": name,
+                    "content": content,
+                }))
+            )),
+            Message::ToolCallItem {
+                name, arguments, ..
+            } => Some(format!(
+                "tool-call:{}",
+                hash_canonical_json(&serde_json::json!({
+                    "name": name,
+                    "arguments": arguments,
+                }))
+            )),
+            Message::ToolResultItem { output, .. } => {
+                Some(format!("tool-result:{}", sha256_hex(&output.to_string())))
+            }
+            Message::ProviderNative {
+                provider,
+                kind,
+                value,
+            } => Some(format!(
+                "native:{}",
+                hash_canonical_json(&serde_json::json!({
+                    "provider": provider,
+                    "kind": kind,
+                    "value": value,
+                }))
+            )),
         })
         .unwrap_or_else(|| "no-seed".to_string())
 }
 
+fn hash_canonical_json(value: &serde_json::Value) -> String {
+    let canonical = canonicalize_value(value).unwrap_or_else(|_| value.to_string());
+    sha256_hex(&canonical)
+}
+
 fn tool_schema_fingerprint(tools: Option<&[ToolDefinition]>) -> String {
     let Some(tools) = tools else {
         return "no-tools".to_string();
@@ -189,12 +246,21 @@ fn extract_text(content: &MessageContent) -> String {
         MessageContent::Parts(parts) => parts
             .iter()
             .map(|part| match part {
-                ContentPart::Text { text } => text.clone(),
-                ContentPart::ImageUrl { image_url } => format!(
+                ContentPart::Text { text, .. } => text.clone(),
+                ContentPart::Refusal { refusal, .. } => refusal.clone(),
+                ContentPart::ImageUrl { image_url, .. } => format!(
                     "[image:{}:{}]",
                     image_url.detail.as_deref().unwrap_or("none"),
                     sha256_hex(&image_url.url)
                 ),
+                ContentPart::Image { .. }
+                | ContentPart::Audio { .. }
+                | ContentPart::File { .. }
+                | ContentPart::ToolUse { .. }
+                | ContentPart::ToolResult { .. }
+                | ContentPart::ProviderNative { .. } => {
+                    serde_json::to_string(part).unwrap_or_default()
+                }
             })
             .collect::<Vec<_>>()
             .join("\n"),

crates/adaptive/tests/integration/acg_module_surface_tests.rs

@@ -129,6 +129,8 @@ fn acg_module_surface_policy_and_ir_builder_symbols_compile_from_canonical_names
     assert!(envelope.policy.warm_first_enabled);
 
     let request = AnnotatedLlmRequest {
+        instructions: None,
+        api_specific: None,
         messages: vec![
             Message::System {
                 content: MessageContent::Text("You are helpful.".to_string()),
@@ -175,6 +177,8 @@ fn acg_module_surface_build_prompt_ir_inserts_tool_schema_before_first_non_syste
     use nemo_relay_adaptive::acg::prompt_ir::{BlockContentType, PromptRole};
 
     let request = AnnotatedLlmRequest {
+        instructions: None,
+        api_specific: None,
         messages: vec![
             Message::System {
                 content: MessageContent::Text("You are helpful.".to_string()),
@@ -187,8 +191,7 @@ fn acg_module_surface_build_prompt_ir_inserts_tool_schema_before_first_non_syste
         ],
         model: Some("claude-sonnet".to_string()),
         params: None,
-        tools: Some(vec![ToolDefinition {
-            tool_type: "function".to_string(),
+        tools: Some(vec![ToolDefinition::Function {
             function: FunctionDefinition {
                 name: "get_weather".to_string(),
                 description: Some("Look up the weather".to_string()),
@@ -198,7 +201,10 @@ fn acg_module_surface_build_prompt_ir_inserts_tool_schema_before_first_non_syste
                         "location": {"type": "string"}
                     }
                 })),
+                strict: None,
+                extra: serde_json::Map::new(),
             },
+            extra: serde_json::Map::new(),
         }]),
         tool_choice: None,
         store: None,

crates/adaptive/tests/integration/redis_tests.rs

@@ -118,6 +118,8 @@ fn make_test_run(agent_id: &str) -> RunRecord {
 
 fn sample_annotated_request(model: &str) -> AnnotatedLlmRequest {
     AnnotatedLlmRequest {
+        instructions: None,
+        api_specific: None,
         messages: vec![
             Message::System {
                 content: MessageContent::Text("You are a careful planner".to_string()),

crates/adaptive/tests/integration/runtime_integration_tests.rs

@@ -78,6 +78,8 @@ fn reset_global() {
 
 fn sample_annotated_request(model: &str) -> AnnotatedLlmRequest {
     AnnotatedLlmRequest {
+        instructions: None,
+        api_specific: None,
         messages: vec![
             Message::System {
                 content: MessageContent::Text("You are a careful planner".to_string()),
@@ -112,6 +114,8 @@ fn sample_annotated_request(model: &str) -> AnnotatedLlmRequest {
 fn sample_growing_chat_requests(model: &str) -> Vec<AnnotatedLlmRequest> {
     vec![
         AnnotatedLlmRequest {
+            instructions: None,
+            api_specific: None,
             messages: vec![
                 Message::System {
                     content: MessageContent::Text("You are a careful planner".to_string()),
@@ -142,6 +146,8 @@ fn sample_growing_chat_requests(model: &str) -> Vec<AnnotatedLlmRequest> {
             extra: Map::new(),
         },
         AnnotatedLlmRequest {
+            instructions: None,
+            api_specific: None,
             messages: vec![
                 Message::System {
                     content: MessageContent::Text("You are a careful planner".to_string()),
@@ -183,6 +189,8 @@ fn sample_growing_chat_requests(model: &str) -> Vec<AnnotatedLlmRequest> {
             extra: Map::new(),
         },
         AnnotatedLlmRequest {
+            instructions: None,
+            api_specific: None,
             messages: vec![
                 Message::System {
                     content: MessageContent::Text("You are a careful planner".to_string()),
@@ -517,14 +525,11 @@ async fn runtime_integration_acg_learner_reuses_learning_buckets_across_growing_
     let requests = sample_growing_chat_requests("claude-3-5-sonnet");
     let learner = AcgLearner::new(agent_id, 8, StabilityThresholds::default());
     let learning_key = format!(
-        "{agent_id}::model=claude-3-5-sonnet::seed={}::system={}::tools=no-tools",
+        "{agent_id}::model=claude-3-5-sonnet::seed={}::system=sha256:3087d8fd4::tools=no-tools",
         short_hash(&format!(
             "user:{}",
             nemo_relay_adaptive::acg::sha256_hex("Summarize the latest findings")
         )),
-        short_hash(&nemo_relay_adaptive::acg::sha256_hex(
-            "You are a careful planner"
-        )),
     );
 
     learner