diff --git a/control_plane/examples/calibration_candidates.rs b/control_plane/examples/calibration_candidates.rs new file mode 100644 index 00000000..d4612c92 --- /dev/null +++ b/control_plane/examples/calibration_candidates.rs @@ -0,0 +1,57 @@ +//! Export every bindable candidate for isolated measurement, without selecting a winner. +use control_plane::physical::{ + compiler::{BackendLocalPlanningSnapshot, PhysicalCompiler}, + workload_cost, +}; +use serde_json::json; + +fn main() -> Result<(), Box> { + let path = std::env::args() + .nth(1) + .ok_or("usage: calibration_candidates SNAPSHOT.json")?; + let snapshot: BackendLocalPlanningSnapshot = serde_json::from_slice(&std::fs::read(path)?)?; + let (request, environment) = snapshot.planning_request()?; + let mut results = Vec::new(); + for (index, candidate) in workload_cost::with_exact_alternative(request)? + .into_iter() + .enumerate() + { + let queries = candidate.queries.clone(); + let plan = match PhysicalCompiler.compile(candidate, environment.clone()) { + Ok(plan) => plan, + Err(error) => { + results.push( + json!({"candidate_index": index, "unavailable_reason": error.to_string()}), + ); + continue; + } + }; + let manifest = match workload_cost::manifest(&plan, &queries) { + Ok(manifest) => manifest, + Err(error) => { + results.push( + json!({"candidate_index": index, "unavailable_reason": error.to_string()}), + ); + continue; + } + }; + results.push(json!({ + "candidate_index": index, + "manifest": manifest, + "lifecycle_estimates": plan.lifecycle_estimates, + "install_request": { + "precompute_plan": plan.precompute_plan, + "transmission_plan": plan.transmission_plan, + "backend_plan": plan.backend_plan.encode_to_vec(), + "query_plan": plan.query_plan, + "storage_routing": null, + "adaptation_evidence": [] + } + })); + } + println!( + "{}", + serde_json::to_string_pretty(&json!({"purpose":"calibration_only", "candidates":results}))? + ); + Ok(()) +} diff --git a/control_plane/src/main.rs b/control_plane/src/main.rs index cf08aa43..cec31229 100644 --- a/control_plane/src/main.rs +++ b/control_plane/src/main.rs @@ -828,6 +828,7 @@ fn compile_physical_plan_request( let planning_request = physical::compiler::PlanningRequest { query_workload: None, queries, + local_raw_execution: false, evidence: request.evidence, planner_revision: request.planner_revision, }; diff --git a/control_plane/src/physical/compiler.rs b/control_plane/src/physical/compiler.rs index b5e9278c..eb79859b 100644 --- a/control_plane/src/physical/compiler.rs +++ b/control_plane/src/physical/compiler.rs @@ -126,6 +126,8 @@ pub struct LifecyclePlanningInput { #[derive(Debug, Clone, Default)] pub struct PlanningRequest { + /// Enable installed typed raw residuals for this backend-local candidate. + pub local_raw_execution: bool, /// Original dashboard demand, in the same order as queries. None is legacy input. pub query_workload: Option, pub queries: Vec, @@ -1555,7 +1557,14 @@ impl BackendLocalPlanningSnapshot { environment, &evidence, ), - None => PhysicalCompiler.compile(request, environment), + None => { + // Unquoted v1 startup snapshots keep the established summary/native + // compatibility policy. Local residual candidates are enumerated by + // planning_request and admitted through measured workload selection. + let mut request = request; + request.local_raw_execution = false; + PhysicalCompiler.compile(request, environment) + } } } @@ -1642,16 +1651,10 @@ impl BackendLocalPlanningSnapshot { crate::query_parser::parse_query_expr_canonical(&query_string, accuracy.clone()) .map_err(|error| CompileError::Snapshot(format!("query {index}: {error}")))?; let metadata = crate::query_parser::qe_to_parsed_query(&parsed); - if metadata.metric_name.is_empty() { - return Err(CompileError::Snapshot(format!( - "query {index} has no unique time-series source" - ))); - } - if !metadata.label_filters.is_empty() { - return Err(CompileError::Snapshot(format!( - "query {index} uses label filters not yet represented by the physical materialization contract" - ))); - } + let source_metrics = super::workload_cost::exact_source_metrics(&parsed)?; + let source_hint = source_metrics.iter().next().cloned().ok_or_else(|| { + CompileError::Snapshot(format!("query {index} has no named time-series source")) + })?; let lifecycle = LifecyclePlanningInput { evaluation_interval_ms, ingestion_rate_per_second: ingestion_rate.0, @@ -1676,7 +1679,7 @@ impl BackendLocalPlanningSnapshot { query_string: query_string.clone(), post_asap, source: Source::TimeSeries { - metric: metadata.metric_name, + metric: source_hint, }, window_secs: lookback_ms / 1_000, group_by: metadata.group_by_labels, @@ -1701,9 +1704,10 @@ impl BackendLocalPlanningSnapshot { }); } select_workload_roots(&mut queries, canonical_roots, &topk_evidence_by_id)?; - preserve_native_unsafe_raw_roots(&mut queries)?; + // Composable lowering residualizes unsafe leaves individually; retain Planner siblings. Ok(( PlanningRequest { + local_raw_execution: true, query_workload: Some(workload), queries, evidence: topk_evidence_by_id, @@ -1764,12 +1768,13 @@ fn has_unsafe_raw_entity_leaf( /// Preserve native execution for raw states that cannot preserve source semantics. fn preserve_native_unsafe_raw_roots(queries: &mut [PlanningQuery]) -> Result<(), CompileError> { for query in queries { - let selected = collect_selected_materializations(&query.post_asap).map_err(|reason| { - CompileError::Query { - query_id: query.query_id.clone(), - reason, - } - })?; + let selected = + collect_selected_materializations(&query.post_asap, false).map_err(|reason| { + CompileError::Query { + query_id: query.query_id.clone(), + reason, + } + })?; let unsafe_entities = has_unsafe_raw_entity_leaf( &query.post_asap, &selected.iter().map(|state| state.node_identity).collect(), @@ -1813,6 +1818,13 @@ impl PhysicalCompiler { mut request: PlanningRequest, environment: DeploymentEnvironment, ) -> Result { + if request.local_raw_execution + && environment.target != PhysicalDeploymentTarget::BackendLocalRemoteWrite + { + return Err(CompileError::Snapshot( + "typed raw residuals require backend-local execution".into(), + )); + } if request.planner_revision != PLANNER_REVISION { return Err(CompileError::PlannerRevision { request: request.planner_revision, @@ -1840,7 +1852,9 @@ impl PhysicalCompiler { } } - if environment.target == PhysicalDeploymentTarget::BackendLocalRemoteWrite { + if environment.target == PhysicalDeploymentTarget::BackendLocalRemoteWrite + && !request.local_raw_execution + { preserve_native_unsafe_raw_roots(&mut request.queries)?; } @@ -1864,7 +1878,11 @@ impl PhysicalCompiler { // the planner DAG node identity across the workload; serving never scans // BackendPlan candidates to rediscover this decision. let mut node_bindings = HashMap::::new(); - let consumers = materialization_consumers(&request.queries, environment.target)?; + let consumers = materialization_consumers( + &request.queries, + environment.target, + request.local_raw_execution, + )?; let mut lifecycle_estimates = BTreeMap::::new(); @@ -1874,39 +1892,46 @@ impl PhysicalCompiler { validate_evidence(&query.query_id, e, &environment)?; } let node = query.post_asap.clone(); - let selected = - collect_selected_materializations(&node).map_err(|reason| CompileError::Query { + let selected = collect_selected_materializations(&node, request.local_raw_execution) + .map_err(|reason| CompileError::Query { query_id: query.query_id.clone(), reason, })?; + let selected = selected + .into_iter() + .filter(|state| { + !request.local_raw_execution + || state.window_secs.is_none_or(|window| { + query + .window_implementations + .iter() + .any(|candidate| candidate.window_secs == window) + }) + }) + .collect::>(); // An exact native fallback has no maintained state and must not // depend on evidence for unused window/state implementations. if selected.is_empty() { continue; } validate_lifecycle_input(&query.query_id, &query.lifecycle)?; - let lifecycle_costs = SummaryMaintenanceLifecycleCostInputs { - build_cost: Some(Cost(query.lifecycle.costs.build)), - maintenance_cost_per_update: Some(Cost( - query.lifecycle.costs.maintenance_per_update, - )), - summary_read_cost: Some(Cost(query.lifecycle.costs.read)), - retention_cost_rate: Some(CostRate(query.lifecycle.costs.retention_per_second)), - retirement_cost: Some(Cost(query.lifecycle.costs.retirement)), - }; - let window_costs = validate_window_implementations(query, &environment)?; - let model = ControlPlaneCostModel::new(query.accuracy.clone()) - .with_summary_maintenance( - lifecycle_costs, - SummaryMaintenanceCapabilities { - incremental_update: true, - merge: true, - // Expired whole panes are excluded when reading a moving scope. - delete: environment.target - == PhysicalDeploymentTarget::BackendLocalRemoteWrite, - }, - ) - .with_window_implementation_costs(window_costs); + if environment.target == PhysicalDeploymentTarget::BackendLocalRemoteWrite + && selected.iter().any(|state| { + matches!( + state.family, + SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Increase + | planner_types::post_asap::ExactKind::Rate, + _ + ) + ) + }) + { + return Err(CompileError::Query { + query_id: query.query_id.clone(), + reason: "raw counter materialization requires independent per-series reset and timestamp state".into(), + }); + } if environment.target == PhysicalDeploymentTarget::DistributedCollectors && selected.iter().any(|state| { matches!( @@ -1934,6 +1959,37 @@ impl PhysicalCompiler { } } for (ordinal, selected) in selected.into_iter().enumerate() { + let mut branch_query = query.clone(); + branch_query.window_secs = selected.window_secs.unwrap_or(query.window_secs); + branch_query.group_by = selected + .group_by + .clone() + .unwrap_or_else(|| query.group_by.clone()); + branch_query + .window_implementations + .retain(|candidate| candidate.window_secs == branch_query.window_secs); + let query = &branch_query; + let lifecycle_costs = SummaryMaintenanceLifecycleCostInputs { + build_cost: Some(Cost(query.lifecycle.costs.build)), + maintenance_cost_per_update: Some(Cost( + query.lifecycle.costs.maintenance_per_update, + )), + summary_read_cost: Some(Cost(query.lifecycle.costs.read)), + retention_cost_rate: Some(CostRate(query.lifecycle.costs.retention_per_second)), + retirement_cost: Some(Cost(query.lifecycle.costs.retirement)), + }; + let window_costs = validate_window_implementations(query, &environment)?; + let model = ControlPlaneCostModel::new(query.accuracy.clone()) + .with_summary_maintenance( + lifecycle_costs, + SummaryMaintenanceCapabilities { + incremental_update: true, + merge: true, + delete: environment.target + == PhysicalDeploymentTarget::BackendLocalRemoteWrite, + }, + ) + .with_window_implementation_costs(window_costs); let metric = selected.metric.clone(); let aggregation_id = format!("{}:{ordinal}:{}", query.query_id, metric); // Rate is a readout over the same reset-aware counter state @@ -2079,7 +2135,18 @@ impl PhysicalCompiler { } } - let plan_id = stable_workload_plan_id(&plan_materializations, &request.queries); + let plan_id = if request.local_raw_execution { + use std::hash::{Hash, Hasher}; + let mut hash = std::collections::hash_map::DefaultHasher::new(); + stable_workload_plan_id(&plan_materializations, &request.queries).hash(&mut hash); + "typed-local-residual-v1".hash(&mut hash); + for query in &request.queries { + format!("{:?}", query.post_asap).hash(&mut hash); + } + hash.finish() + } else { + stable_workload_plan_id(&plan_materializations, &request.queries) + }; let envelope = PlanEnvelope { plan_id, plan_version: environment.plan_version, @@ -2177,17 +2244,7 @@ impl PhysicalCompiler { let mut query_entries = BTreeMap::new(); for query in &request.queries { let canonical = canonical_promql(&query.query_string)?; - let entry = QueryPlanEntry::compile_bound( - query.query_id.clone(), - canonical.clone(), - &query.post_asap, - InstantExecution { - lookback_ms: query.window_secs.saturating_mul(1_000), - full_history: false, - cumulative_readout: true, - }, - FallbackPolicy::ExactBackend, - |node, node_family| { + let binding = |node: &SummaryNode, node_family: &SummaryFamilyType| -> Result { let planned_metric = summary_agg_metric(node).ok_or_else(|| { crate::query_plan::QueryPlanError::Invalid( "materialized node has no unique time-series source".into(), @@ -2210,11 +2267,12 @@ impl PhysicalCompiler { fingerprint.0 )) })?; + let (_, source_window) = materialization_leaf_contract(node) + .map_err(crate::query_plan::QueryPlanError::Invalid)?; if materialization.family != physical_materialization_family(node_family) || materialization.window.size_ms == 0 - || query.window_secs.saturating_mul(1_000) % materialization.window.size_ms - != 0 - || materialization.group_by != query.group_by + || source_window.unwrap_or(query.window_secs).saturating_mul(1_000) + % materialization.window.size_ms != 0 { return Err(crate::query_plan::QueryPlanError::Invalid(format!( "compiled binding {} disagrees with post-ASAP/deployment semantics", @@ -2222,14 +2280,38 @@ impl PhysicalCompiler { ))); } Ok(MaterializationBinding { + readout_lookback_ms: source_window.map(|seconds| seconds.saturating_mul(1_000)), materialization: fingerprint, metric: planned_metric, - sid_grouping: query.group_by.clone(), - output_grouping: PhysicalGrouping::Reduce(query.group_by.clone()), + sid_grouping: materialization.group_by.clone(), + output_grouping: PhysicalGrouping::Reduce(materialization.group_by.clone()), window_ms: materialization.window.size_ms, }) - }, - )?; + }; + let instant = InstantExecution { + lookback_ms: query.window_secs.saturating_mul(1_000), + full_history: false, + cumulative_readout: true, + }; + let entry = if request.local_raw_execution { + QueryPlanEntry::compile_bound_composable( + query.query_id.clone(), + canonical.clone(), + &query.post_asap, + instant, + FallbackPolicy::ExactBackend, + binding, + ) + } else { + QueryPlanEntry::compile_bound( + query.query_id.clone(), + canonical.clone(), + &query.post_asap, + instant, + FallbackPolicy::ExactBackend, + binding, + ) + }?; if query_entries.insert(canonical.clone(), entry).is_some() { return Err(CompileError::Query { query_id: query.query_id.clone(), @@ -2567,7 +2649,13 @@ fn select_lifecycle( // Collector windows are retired as whole states. They do not // claim deletion support for moving-window retractions. scope: QueryTimeScope::Unknown, - lookback: Some(DurationMs(query.window_secs.saturating_mul(1_000))), + lookback: Some(DurationMs( + materialization_leaf_contract(node) + .ok() + .and_then(|(_, window)| window) + .unwrap_or(query.window_secs) + .saturating_mul(1_000), + )), as_of: None, }, }) @@ -2682,10 +2770,56 @@ fn select_lifecycle( }) } +/// A warm producer may consume only a source whose semantics its raw updater +/// implements. Predicates and shifted ranges remain executable residual nodes. +pub(crate) fn materialization_leaf_contract( + node: &SummaryNode, +) -> Result<(String, Option), String> { + use planner_types::pre_asap::QueryExpr; + let SummaryExpr::SummaryAgg { child, .. } = &node.expr else { + return Err("materialization requires a SummaryAgg leaf".into()); + }; + let SummaryExpr::KeepPreAsap(expr) = &child.expr else { + return Err("materialization input is not a raw source".into()); + }; + let (source, window_secs) = match expr.as_ref() { + QueryExpr::TimeRange { child, range } => { + if range.as_millis() == 0 || range.as_millis() % 1000 != 0 { + return Err("warm producer requires a positive whole-second range".into()); + } + (child.as_ref(), Some(range.as_secs())) + } + QueryExpr::Scan { .. } + if matches!( + &node.expr, + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::ExactAggregate(..), + .. + } + ) => + { + return Err( + "instantaneous sample selection is not a temporal accumulator readout".into(), + ); + } + source => (source, None), + }; + match source { + QueryExpr::Scan { + source: Source::TimeSeries { metric }, + predicates, + .. + } if !metric.is_empty() && predicates.is_empty() => Ok((metric.clone(), window_secs)), + _ => Err("source predicates or temporal modifiers require a local raw residual".into()), + } +} + struct SelectedMaterialization { node: Rc, node_identity: usize, metric: String, + window_secs: Option, + group_by: Option>, family: SummaryFamilyType, readout: Option, algorithm: String, @@ -2702,9 +2836,12 @@ fn physical_aggregation( aggregation_id, metric_name: selected.metric.clone(), family: physical_materialization_family(&selected.family), - window_secs: query.window_secs, + window_secs: selected.window_secs.unwrap_or(query.window_secs), spatial_filter: String::new(), - grouping: query.group_by.clone(), + grouping: selected + .group_by + .clone() + .unwrap_or_else(|| query.group_by.clone()), item_label: None, heap_update_mode: selected.parameters.get("weight_mode").and_then(|mode| { match mode.as_str() { @@ -2723,16 +2860,28 @@ fn physical_aggregation( fn materialization_consumers( queries: &[PlanningQuery], target: PhysicalDeploymentTarget, + composable: bool, ) -> Result>, CompileError> { let mut consumers = BTreeMap::<_, BTreeSet<_>>::new(); for (index, query) in queries.iter().enumerate() { - let states = collect_selected_materializations(&query.post_asap).map_err(|reason| { - CompileError::Query { - query_id: query.query_id.clone(), - reason, - } - })?; + let states = + collect_selected_materializations(&query.post_asap, composable).map_err(|reason| { + CompileError::Query { + query_id: query.query_id.clone(), + reason, + } + })?; for state in states { + if composable + && state.window_secs.is_some_and(|window| { + !query + .window_implementations + .iter() + .any(|candidate| candidate.window_secs == window) + }) + { + continue; + } let config = backend_plan::aggregation_config_for_materialization( &physical_aggregation(query, &state, query.query_id.clone(), target), )?; @@ -2751,20 +2900,70 @@ fn materialization_consumers( /// serialized QueryPlan retains the merge edges. Unsupported operators are /// intentionally not traversed: QueryPlan lowers them to an explicit exact /// fallback node and no unused warm state is provisioned. +/// Raw accumulators do not retain arbitrary source labels. Preserve native semantics +/// unless the selected DAG explicitly authorizes pooling the source entities. fn collect_selected_materializations( node: &Rc, + composable: bool, ) -> Result, String> { fn walk( node: &Rc, readout: Option<&SketchQuery>, + composable: bool, + inherited_grouping: Option>, selected: &mut Vec, ) -> Result<(), String> { + if composable + && matches!( + &node.expr, + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::ExactAggregate( + planner_types::post_asap::ExactKind::Increase + | planner_types::post_asap::ExactKind::Rate, + _ + ), + .. + } + ) + { + return Ok(()); + } + let grouping = if composable { + if let SummaryExpr::SummaryAgg { + reduction, child, .. + } = &node.expr + { + if let Some(keys) = reduction.group_keys() { + Some( + keys.keys() + .iter() + .map(|id| { + child + .schema + .fields + .get(*id) + .map(|field| field.name.clone()) + .ok_or_else(|| { + format!("unresolved producer grouping column {id}") + }) + }) + .collect::, _>>()?, + ) + } else { + inherited_grouping + } + } else { + inherited_grouping + } + } else { + None + }; match &node.expr { SummaryExpr::BinaryOp { lhs, rhs, .. } - if crate::query_plan::exact_value_executable(node) => + if composable || crate::query_plan::exact_value_executable(node) => { - walk(lhs, readout, selected)?; - walk(rhs, readout, selected)?; + walk(lhs, readout, composable, grouping.clone(), selected)?; + walk(rhs, readout, composable, grouping.clone(), selected)?; } SummaryExpr::SummaryAgg { child, @@ -2772,9 +2971,10 @@ fn collect_selected_materializations( SummaryFamilyType::ExactAggregate(planner_types::post_asap::ExactKind::Sum, _), .. } if !matches!(child.expr, SummaryExpr::KeepPreAsap(_)) - && crate::query_plan::exact_value_executable(node) => + && ((composable && matches!(child.expr, SummaryExpr::SummaryAgg { .. })) + || crate::query_plan::exact_value_executable(node)) => { - walk(child, readout, selected)?; + walk(child, readout, composable, grouping.clone(), selected)?; } SummaryExpr::SummaryAgg { child, .. } if !matches!(child.expr, SummaryExpr::KeepPreAsap(_)) => {} @@ -2786,10 +2986,16 @@ fn collect_selected_materializations( SummaryExpr::SummaryEstimate { summary_input, query, - } => walk(summary_input, Some(query), selected)?, + } => walk( + summary_input, + Some(query), + composable, + grouping.clone(), + selected, + )?, SummaryExpr::SummaryMerge { children } => { for child in children { - walk(child, readout, selected)?; + walk(child, readout, composable, grouping.clone(), selected)?; } } SummaryExpr::SummaryAgg { @@ -2810,13 +3016,17 @@ fn collect_selected_materializations( }; parameters["weight_mode"] = mode.into(); } - let metric = summary_agg_metric(node).ok_or_else(|| { - "SummaryAgg has no unique time-series source in post-ASAP IR".to_string() - })?; + let (metric, window_secs) = match materialization_leaf_contract(node) { + Ok(contract) => contract, + Err(_) if composable => return Ok(()), + Err(error) => return Err(error), + }; selected.push(SelectedMaterialization { node: Rc::clone(node), node_identity: Rc::as_ptr(node) as usize, metric, + window_secs, + group_by: grouping.clone(), family: SummaryFamilyType::Sketch( kind.clone(), planner_types::post_asap::GroupingStrategy::PerSubpopulationInstance, @@ -2831,13 +3041,17 @@ fn collect_selected_materializations( family: SummaryFamilyType::ExactAggregate(kind, params), .. } => { - let metric = summary_agg_metric(node).ok_or_else(|| { - "SummaryAgg has no unique time-series source in post-ASAP IR".to_string() - })?; + let (metric, window_secs) = match materialization_leaf_contract(node) { + Ok(contract) => contract, + Err(_) if composable => return Ok(()), + Err(error) => return Err(error), + }; selected.push(SelectedMaterialization { node: Rc::clone(node), node_identity: Rc::as_ptr(node) as usize, metric, + window_secs, + group_by: grouping.clone(), family: SummaryFamilyType::ExactAggregate(kind.clone(), params.clone()), readout: None, algorithm: format!("{kind:?}").to_ascii_lowercase(), @@ -2855,7 +3069,12 @@ fn collect_selected_materializations( } let mut selected = Vec::new(); - walk(node, None, &mut selected)?; + walk(node, None, composable, None, &mut selected)?; + if composable { + selected.retain(|state| { + !has_unsafe_raw_entity_leaf(node, &BTreeSet::from([state.node_identity]), false) + }); + } Ok(selected) } @@ -3031,7 +3250,7 @@ mod tests { // Capability normalization precedes candidate enumeration, avoiding duplicate exact quotes. #[test] - fn counter_only_snapshot_has_one_exact_cost_alternative() { + fn counter_only_snapshot_has_distinct_local_and_native_cost_alternatives() { let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( "../../../docs/examples/asapquery-planning-snapshot.json" )) @@ -3044,7 +3263,7 @@ mod tests { super::super::workload_cost::with_exact_alternative(request) .unwrap() .len(), - 1 + 2 ); } @@ -3119,6 +3338,7 @@ mod tests { evidence_by_query.insert(query_id.to_string(), evidence); } Ok(PlanningRequest { + local_raw_execution: false, query_workload: None, queries: vec![PlanningQuery { query_id: query_id.into(), @@ -3386,7 +3606,8 @@ mod tests { .into(), ); entries.push(mean); - let bundle = snapshot.compile().unwrap(); + let (request, env) = snapshot.planning_request().unwrap(); + let bundle = PhysicalCompiler.compile(request, env).unwrap(); assert_eq!(bundle.precompute_plan.materializations.len(), 1); assert_eq!(bundle.query_plan.entries.len(), 2); for entry in bundle.query_plan.entries.values() { @@ -3399,10 +3620,17 @@ mod tests { ); assert_eq!(entry.materialization_bindings().len(), 1); } - assert!(bundle.query_plan.entries.values().any(|entry| entry - .nodes + assert!(bundle + .query_plan + .entries .values() - .any(|node| matches!(node, crate::query_plan::QueryPlanNode::Binary { .. })))); + .any(|entry| entry.nodes.values().any(|node| matches!( + node, + crate::query_plan::QueryPlanNode::Logical { + operator: crate::query_plan::logical::LogicalOperator::Binary { .. }, + .. + } + )))); } // Grouping must not move through non-additive arithmetic during physical @@ -3423,7 +3651,171 @@ mod tests { let entries = snapshot.query_workload.repeating_queries.as_mut().unwrap(); entries[0].query = Query(query.into()); entries[0].requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); - let bundle = snapshot.compile().unwrap(); + let (mut request, environment) = snapshot.planning_request().unwrap(); + request.local_raw_execution = false; + let bundle = PhysicalCompiler.compile(request, environment).unwrap(); + assert!(bundle.precompute_plan.materializations.is_empty()); + assert!(bundle.query_plan.entries.values().all(|entry| matches!( + entry.nodes[&entry.root], + crate::query_plan::QueryPlanNode::ExactFallback { .. } + ))); + } + } + + // Local composition must evaluate per-series division before the outer sum. + #[test] + fn composable_non_additive_rollup_does_not_pool_raw_producers() { + let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; + entry.query = + Query("sum by (service) (sum_over_time(m[1m]) / count_over_time(m[1m]))".into()); + entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); + let (request, environment) = snapshot.planning_request().unwrap(); + let candidates = super::super::workload_cost::with_exact_alternative(request).unwrap(); + match PhysicalCompiler.compile(candidates[0].clone(), environment.clone()) { + Ok(plan) => assert!(plan.precompute_plan.materializations.is_empty()), + Err(error) => assert!(error + .to_string() + .contains("semantically identical original subtree witness")), + } + let native = PhysicalCompiler + .compile(candidates.last().unwrap().clone(), environment) + .unwrap(); + assert!(native.precompute_plan.materializations.is_empty()); + } + + #[test] + fn composable_per_entity_window_uses_raw_rows_instead_of_pooled_state() { + use crate::query_plan::{logical::LogicalOperator, QueryPlanNode}; + let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; + entry.query = Query("sum_over_time(m[1m])".into()); + entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); + let (request, env) = snapshot.planning_request().unwrap(); + let plan = PhysicalCompiler.compile(request, env).unwrap(); + assert!(plan.precompute_plan.materializations.is_empty()); + let entry = plan.query_plan.lookup("sum_over_time(m[1m])").unwrap(); + assert!(entry.nodes.values().any(|node| matches!( + node, + QueryPlanNode::Logical { + operator: LogicalOperator::Scan { .. }, + .. + } + ))); + assert!(entry.nodes.values().all(|node| !matches!( + node, + QueryPlanNode::ReadMaterialization { .. } | QueryPlanNode::ExactFallback { .. } + ))); + } + + // Each operand retains its source and range rather than borrowing the root window. + #[test] + fn composable_binary_binds_independent_source_windows() { + let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; + entry.query = Query("sum(sum_over_time(a[1m])) / sum(sum_over_time(b[5m]))".into()); + entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); + let (mut request, env) = snapshot.planning_request().unwrap(); + let query = &mut request.queries[0]; + let mut five_minutes = query.window_implementations[0].clone(); + five_minutes.implementation_id = "five-minute-evidence".into(); + five_minutes.window_secs = 300; + five_minutes.pane_secs = 300; + query.window_implementations.push(five_minutes); + let plan = PhysicalCompiler.compile(request, env).unwrap(); + let bindings = plan + .query_plan + .entries + .values() + .next() + .unwrap() + .materialization_bindings(); + let actual = bindings + .iter() + .map(|binding| (binding.metric.as_str(), binding.window_ms)) + .collect::>(); + assert_eq!(actual, BTreeSet::from([("a", 60_000), ("b", 300_000)])); + assert_eq!(plan.precompute_plan.materializations.len(), 2); + } + + // A filtered denominator is a typed residual while its summary sibling remains installed. + #[test] + fn composable_binary_retains_summary_sibling_of_filtered_raw_branch() { + use crate::query_plan::{ + logical::{LabelMatch, LogicalOperator}, + QueryPlanNode, + }; + let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; + entry.query = + Query("sum(sum_over_time(a[1m])) / sum(sum_over_time(b{job!=\"x\"}[5m]))".into()); + entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); + let (request, env) = snapshot.planning_request().unwrap(); + let plan = PhysicalCompiler.compile(request, env).unwrap(); + let query = plan.query_plan.entries.values().next().unwrap(); + let bindings = query.materialization_bindings(); + assert_eq!(bindings.len(), 1); + assert_eq!((&*bindings[0].metric, bindings[0].window_ms), ("a", 60_000)); + assert!(!query + .nodes + .values() + .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. }))); + assert!(query.nodes.values().any(|node| matches!(node, + QueryPlanNode::Logical { operator: LogicalOperator::Scan { metric: Some(metric), range_ms: Some(300_000), matchers, .. }, .. } + if metric == "b" && matchers.iter().any(|matcher| matcher.name == "job" && matcher.value == "x" && matcher.operation == LabelMatch::NotEqual)))); + assert_eq!(plan.precompute_plan.materializations.len(), 1); + } + + // No equality, inequality or regex predicate may disappear at the producer boundary. + #[test] + fn warm_leaf_contract_rejects_unimplemented_predicate_semantics() { + for query in [ + "sum_over_time(m{job=\"a\"}[1m])", + "sum_over_time(m{job!=\"a\"}[1m])", + "sum_over_time(m{job=~\"a.*\"}[1m])", + "sum_over_time(m{job!~\"a.*\"}[1m])", + "sum_over_time(m[1m] offset 1h)", + ] { + let request = request("scope", query); + let selected = collect_selected_materializations(&request.queries[0].post_asap, false); + assert!(selected.is_err() || selected.unwrap().is_empty(), "{query}"); + } + } + + // Unsupported producer filters/multiple sources retain executable native fallback. + #[test] + fn snapshot_accepts_filtered_and_multisource_exact_fallback() { + for query in [ + "sum(rate(http_requests_total{job=\"order-service\"}[5m]))", + "sum(rate(a[5m])) / sum(rate(b[5m]))", + "sum(avg_over_time(m{job=~\".+\"}[6h]))", + "sum(rate(m[5m] offset 1h))", + ] { + let mut snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; + entry.query = Query(query.into()); + entry.requirements.accuracy = AccuracyRequirement::Explicit(AccuracyTarget::Exact); + let (mut request, environment) = snapshot.planning_request().unwrap(); + request = super::super::workload_cost::with_exact_alternative(request) + .unwrap() + .pop() + .unwrap(); + let bundle = PhysicalCompiler.compile(request, environment).unwrap(); assert!(bundle.precompute_plan.materializations.is_empty()); assert!(bundle.query_plan.entries.values().all(|entry| matches!( entry.nodes[&entry.root], @@ -3442,7 +3834,7 @@ mod tests { let entries = snapshot.query_workload.repeating_queries.as_mut().unwrap(); entries[0].query = Query("sum(sum_over_time(m[1m]))".into()); let mut second = entries[0].clone(); - second.query = Query("sum(count_over_time(m[1m]))".into()); + second.query = Query("sum(sum_over_time(m[1m])) * 2".into()); entries.push(second); let bundle = snapshot.compile().unwrap(); assert_eq!(bundle.query_plan.entries.len(), 2); @@ -3629,7 +4021,7 @@ mod tests { language: QueryLanguage::PromQL, query_batch: None, repeating_queries: Some(vec![RepeatingEntry { - query: Query("quantile_over_time(0.99, m[1m])".into()), + query: Query("sum(sum_over_time(m[1m]))".into()), demand: RepeatedDemand::FixedInterval(RepetitionInterval(10_000)), requirements: QueryRequirements { accuracy: AccuracyRequirement::Explicit(AccuracyTarget::EpsilonDelta { @@ -3650,7 +4042,7 @@ mod tests { let mut environment = environment(10_000); environment.target = PhysicalDeploymentTarget::BackendLocalRemoteWrite; environment.collector_ids.clear(); - let template = request("template", "quantile_over_time(0.99, m[1m])") + let template = request("template", "sum(sum_over_time(m[1m]))") .queries .remove(0); let snapshot = BackendLocalPlanningSnapshot { @@ -3773,7 +4165,7 @@ mod tests { } #[test] - fn checked_in_backend_local_snapshot_is_canonical_and_compilable() { + fn checked_in_per_entity_snapshot_preserves_native_alternative() { let source = include_str!("../../../docs/examples/asapquery-planning-snapshot.json"); let snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(source).expect("strict canonical workload fixture"); @@ -3781,7 +4173,25 @@ mod tests { let fixture: serde_json::Value = serde_json::from_str(source).expect("fixture JSON"); assert_eq!(encoded, fixture); - let plan = snapshot.compile().expect("fixture compiles"); + snapshot + .clone() + .compile() + .expect("unquoted v1 compatibility startup remains available"); + let (local, env) = snapshot.clone().planning_request().unwrap(); + assert!(PhysicalCompiler + .compile(local, env) + .unwrap_err() + .to_string() + .contains("native residual substitution requires an exact selected value")); + let (request, environment) = snapshot.planning_request().unwrap(); + let native = crate::physical::workload_cost::with_exact_alternative(request) + .unwrap() + .pop() + .unwrap(); + let plan = PhysicalCompiler + .compile(native, environment) + .expect("native fixture compiles"); + assert!(plan.precompute_plan.materializations.is_empty()); assert!(plan.collector_plans.is_empty()); assert!(plan.transmission_plan.rules.is_empty()); assert_eq!( @@ -3792,17 +4202,34 @@ mod tests { } #[test] - fn compatibility_demo_snapshot_compiles_the_complete_query_matrix() { + fn compatibility_demo_preserves_complete_native_query_matrix() { let source = include_str!("../../../docs/examples/asapquery-compatibility-demo-snapshot.json"); let snapshot: BackendLocalPlanningSnapshot = serde_json::from_str(source).expect("strict compatibility demo fixture"); - let plan = snapshot.compile().expect("compatibility demo compiles"); + snapshot + .clone() + .compile() + .expect("unquoted v1 compatibility startup remains available"); + let (local, env) = snapshot.clone().planning_request().unwrap(); + assert!(PhysicalCompiler + .compile(local, env) + .unwrap_err() + .to_string() + .contains("native residual substitution requires an exact selected value")); + let (request, environment) = snapshot.planning_request().unwrap(); + let native = crate::physical::workload_cost::with_exact_alternative(request) + .unwrap() + .pop() + .unwrap(); + let plan = PhysicalCompiler + .compile(native, environment) + .expect("native demo compiles"); assert!(plan.collector_plans.is_empty()); assert!(plan.transmission_plan.rules.is_empty()); assert_eq!(plan.query_plan.entries.len(), 6); - assert_eq!(plan.precompute_plan.materializations.len(), 3); + assert!(plan.precompute_plan.materializations.is_empty()); for query in [ "rate(asap_demo_counter_total[5s])", "increase(asap_demo_counter_total[5s])", diff --git a/control_plane/src/physical/workload_cost.rs b/control_plane/src/physical/workload_cost.rs index 98fb0d2c..5938196d 100644 --- a/control_plane/src/physical/workload_cost.rs +++ b/control_plane/src/physical/workload_cost.rs @@ -203,6 +203,29 @@ pub fn manifest( add(format!("source:{}", source), source, "horizon", 1.0); } } + // Typed local scans require retained input and ingest/update work even + // when no precomputed summary is installed. Deduplicate by source. + for node in entry.nodes.values() { + if let crate::query_plan::QueryPlanNode::Logical { + operator: crate::query_plan::logical::LogicalOperator::Scan { metric, .. }, + .. + } = node + { + let metric = metric + .as_ref() + .ok_or_else(|| invalid("local raw scan requires named-source pricing"))?; + let source = json!({"source": planner_types::pre_asap::Source::TimeSeries { metric: metric.clone() }, "location": "backend", "ingest": plan.precompute_plan.ingest}); + add(format!("source:{}", source), source.clone(), "horizon", 1.0); + for operation in ["build", "update", "residency", "retire"] { + add( + format!("raw-state:{metric}:{operation}"), + json!({"operation": operation, "source": source}), + "horizon", + 1.0, + ); + } + } + } // Reachability comes from QueryPlan, including materialization reads, // arithmetic, reduction and a complete engine-native exact fallback. for node_id in entry.topological_order()? { @@ -234,7 +257,7 @@ pub fn manifest( /// Walk the canonical relational tree, preserving every input to binary and /// fan-in operators. Unsupported source discovery must not produce a partial quote. -fn exact_source_metrics( +pub(crate) fn exact_source_metrics( expr: &planner_types::pre_asap::QueryExpr, ) -> Result, CompileError> { use planner_types::pre_asap::{QueryExpr, Source}; @@ -434,6 +457,7 @@ pub fn with_exact_alternative( request: PlanningRequest, ) -> Result, CompileError> { let mut exact = request.clone(); + exact.local_raw_execution = false; for query in &mut exact.queries { let parsed = crate::query_parser::parse_query_expr_canonical( &query.query_string, @@ -443,11 +467,12 @@ pub fn with_exact_alternative( query.post_asap = crate::planner_selection::keep_pre_asap(&parsed) .map_err(|error| invalid(error.to_string()))?; } - if request - .queries - .iter() - .zip(&exact.queries) - .all(|(a, b)| a.post_asap == b.post_asap) + if !request.local_raw_execution + && request + .queries + .iter() + .zip(&exact.queries) + .all(|(a, b)| a.post_asap == b.post_asap) { Ok(vec![request]) } else { @@ -506,6 +531,50 @@ mod tests { (candidates, env, evidence) } + // Retained local input is priced once per metric, separate from the native service. + #[test] + fn local_raw_manifest_prices_shared_storage_and_distinct_native_alternative() { + use planner_types::workload::{AccuracyRequirement, Query}; + let mut snapshot = fixture(); + let entry = &mut snapshot.query_workload.repeating_queries.as_mut().unwrap()[0]; + entry.query = Query("sum(rate(a{job=\"x\"}[1m])) / sum(rate(a{job!=\"x\"}[5m]))".into()); + entry.requirements.accuracy = + AccuracyRequirement::Explicit(crate::types_v2::AccuracyTarget::Exact); + let (request, environment) = snapshot.planning_request().unwrap(); + let candidates = with_exact_alternative(request).unwrap(); + assert_eq!(candidates.len(), 2); + let local = PhysicalCompiler + .compile(candidates[0].clone(), environment.clone()) + .unwrap(); + let native = PhysicalCompiler + .compile(candidates[1].clone(), environment) + .unwrap(); + assert_ne!(local.envelope.plan_id, native.envelope.plan_id); + let manifest = manifest(&local, &candidates[0].queries).unwrap(); + assert_eq!( + manifest + .components + .keys() + .filter(|key| key.starts_with("raw-state:a:")) + .count(), + 4 + ); + assert_eq!( + manifest + .components + .values() + .filter(|demand| demand.implementation.get("location") + == Some(&serde_json::json!("backend"))) + .count(), + 1 + ); + assert!(!manifest + .components + .values() + .any(|demand| demand.implementation.get("location") + == Some(&serde_json::json!("exact_backend")))); + } + // All input metrics need upkeep quotes; repeated reads share that upkeep. #[test] fn exact_manifest_covers_and_deduplicates_query_sources() { @@ -666,7 +735,7 @@ mod tests { let mut shared = request.clone(); let mut second = shared.queries[0].clone(); second.query_id = "second-consumer".into(); - second.query_string = "sum(count_over_time(m[1m]))".into(); + second.query_string = "sum(sum_over_time(m[1m])) * 2".into(); let entries = shared .query_workload .as_mut() diff --git a/control_plane/src/query_plan.rs b/control_plane/src/query_plan.rs index 4d46306f..1739e2c2 100644 --- a/control_plane/src/query_plan.rs +++ b/control_plane/src/query_plan.rs @@ -5,6 +5,8 @@ //! node IDs. Serving executes this graph without reconstructing Planner IR or //! searching for compatible materializations. +pub mod logical; + use std::collections::{BTreeMap, BTreeSet}; use std::rc::Rc; @@ -116,6 +118,41 @@ impl QueryPlanEntry { nodes: BTreeMap::new(), seen: BTreeMap::new(), bind: &mut bind, + logical_source: None, + }; + let root = compiler.lower(root)?; + Ok(Self { + query_id, + canonical_promql, + root, + nodes: compiler.nodes, + instant, + fallback, + }) + } + + /// Compile selected summary nodes and verified native residuals into one DAG. + /// This is a distinct physical alternative; native execution remains available. + pub fn compile_bound_composable( + query_id: String, + canonical_promql: String, + root: &Rc, + instant: InstantExecution, + fallback: FallbackPolicy, + mut bind: F, + ) -> Result + where + F: FnMut( + &SummaryNode, + &SummaryFamilyType, + ) -> Result, + { + let mut compiler = DagCompiler { + next_id: 0, + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + bind: &mut bind, + logical_source: Some(canonical_promql.clone()), }; let root = compiler.lower(root)?; Ok(Self { @@ -137,6 +174,9 @@ impl QueryPlanEntry { ))); } for (id, node) in &self.nodes { + if let QueryPlanNode::Logical { operator, inputs } = node { + operator.validate(inputs.len())?; + } if matches!(node, QueryPlanNode::Scalar { value } if !value.is_finite()) { return Err(QueryPlanError::Invalid("non-finite scalar constant".into())); } @@ -149,6 +189,11 @@ impl QueryPlanEntry { } } if let QueryPlanNode::ReadMaterialization { binding } = node { + if binding.readout_lookback_ms == Some(0) { + return Err(QueryPlanError::Invalid( + "zero semantic readout lookback".into(), + )); + } if !available.contains(&binding.materialization) { return Err(QueryPlanError::Invalid(format!( "query `{}` node {} references absent materialization {}", @@ -225,6 +270,9 @@ pub struct MaterializationBinding { /// Query operator grouping applied while folding those SIDs. pub output_grouping: PhysicalGrouping, pub window_ms: u64, + /// Semantic query lookback, independent of the physical pane duration. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub readout_lookback_ms: Option, } #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] @@ -237,6 +285,10 @@ pub enum PhysicalGrouping { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] #[serde(tag = "op", rename_all = "snake_case", deny_unknown_fields)] pub enum QueryPlanNode { + Logical { + operator: logical::LogicalOperator, + inputs: Vec, + }, Scalar { value: f64, }, @@ -277,7 +329,7 @@ impl QueryPlanNode { Self::ReduceSum { input, .. } | Self::SummaryEstimate { input, .. } | Self::ExactReadout { input, .. } => std::slice::from_ref(input), - Self::SummaryMerge { inputs } => inputs, + Self::SummaryMerge { inputs } | Self::Logical { inputs, .. } => inputs, } } } @@ -334,12 +386,41 @@ struct DagCompiler<'a, F> { nodes: BTreeMap, seen: BTreeMap, bind: &'a mut F, + logical_source: Option, } impl DagCompiler<'_, F> where F: FnMut(&SummaryNode, &SummaryFamilyType) -> Result, { + fn graft( + &mut self, + id: QueryNodeId, + root: QueryNodeId, + nodes: BTreeMap, + ) -> Result { + let mut remap = BTreeMap::new(); + for local in nodes.keys() { + let global = if *local == root { + id + } else { + let next = QueryNodeId(self.next_id); + self.next_id += 1; + next + }; + remap.insert(*local, global); + } + for (local, mut physical) in nodes { + if let QueryPlanNode::Logical { inputs, .. } = &mut physical { + for input in inputs { + *input = remap[input]; + } + } + self.nodes.insert(remap[&local], physical); + } + return Ok(id); + } + fn lower(&mut self, node: &Rc) -> Result { let identity = Rc::as_ptr(node) as usize; if let Some(id) = self.seen.get(&identity) { @@ -348,7 +429,100 @@ where let id = QueryNodeId(self.next_id); self.next_id += 1; self.seen.insert(identity, id); + let residual = match (&self.logical_source, &node.expr) { + (Some(original), SummaryExpr::KeepPreAsap(expr)) => { + Some(logical::residual_nodes(original, expr)?) + } + (Some(original), SummaryExpr::SummaryAgg { child, .. }) + if matches!(child.expr, SummaryExpr::KeepPreAsap(_)) + && !matches!( + crate::physical::compiler::materialization_leaf_contract(node), + Ok((_, Some(_))) + ) => + { + Some(logical::selected_residual_nodes(original, node)?) + } + _ => None, + }; + if let Some((root, nodes)) = residual { + return self.graft(id, root, nodes); + } + let physical = match &node.expr { + SummaryExpr::BinaryOp { lhs, rhs, operator } if self.logical_source.is_some() => { + let operator = logical::binary_operator(operator)?; + QueryPlanNode::Logical { + operator, + inputs: vec![self.lower(lhs)?, self.lower(rhs)?], + } + } + + SummaryExpr::SummaryAgg { + family: SummaryFamilyType::ExactAggregate(kind, _), + child, + reduction, + .. + } if self.logical_source.is_some() + && !matches!(child.expr, SummaryExpr::KeepPreAsap(_)) => + { + if !matches!( + kind, + planner_types::post_asap::ExactKind::Sum + | planner_types::post_asap::ExactKind::Count + ) { + let operator = logical::selected_aggregate_operator( + self.logical_source.as_deref().unwrap(), + node, + )?; + let input = self.lower(child)?; + self.nodes.insert( + id, + QueryPlanNode::Logical { + operator, + inputs: vec![input], + }, + ); + return Ok(id); + } + let operation = match kind { + planner_types::post_asap::ExactKind::Sum => logical::Aggregation::Sum, + planner_types::post_asap::ExactKind::Count => logical::Aggregation::Count, + _ => { + return Err(QueryPlanError::Invalid( + "unsupported aggregation over selected summary values".into(), + )) + } + }; + let keys = reduction.group_keys().ok_or_else(|| { + QueryPlanError::Invalid( + "per-entity summary reduction requires a temporal operator".into(), + ) + })?; + let labels = keys + .keys() + .iter() + .map(|&column| { + child + .schema + .fields + .get(column) + .map(|field| field.name.clone()) + .ok_or_else(|| { + QueryPlanError::Invalid("unresolved logical grouping column".into()) + }) + }) + .collect::, _>>()?; + QueryPlanNode::Logical { + operator: logical::LogicalOperator::Aggregate { + operation, + grouping: logical::Grouping { + labels, + without: keys.is_without(), + }, + }, + inputs: vec![self.lower(child)?], + } + } SummaryExpr::BinaryOp { lhs, rhs, operator } if exact_value_executable(node) => { let planner_types::pre_asap::BinaryOpKind::Arithmetic(operator) = &operator.kind else { @@ -392,11 +566,12 @@ where } if !exact_value_executable(node) => QueryPlanNode::ExactFallback { reason: "only temporal observation counts are supported".into(), }, - SummaryExpr::BinaryOp { .. } | SummaryExpr::KeepPreAsap(_) => { - QueryPlanNode::ExactFallback { - reason: "post-ASAP node requires exact execution".into(), - } - } + SummaryExpr::BinaryOp { .. } => QueryPlanNode::ExactFallback { + reason: "summary binary operation is not executable by the warm tier".into(), + }, + SummaryExpr::KeepPreAsap(_) => QueryPlanNode::ExactFallback { + reason: "post-ASAP node requires exact execution".into(), + }, SummaryExpr::SummaryAgg { family, reduction, @@ -404,7 +579,17 @@ where .. } => match family { SummaryFamilyType::ExactAggregate(..) | SummaryFamilyType::Sketch(..) => { - let mut binding = (self.bind)(node, family)?; + let mut binding = match (self.bind)(node, family) { + Ok(binding) => binding, + Err(error) => { + if let Some(original) = &self.logical_source { + let (root, nodes) = + logical::selected_residual_nodes(original, node)?; + return self.graft(id, root, nodes); + } + return Err(error); + } + }; binding.output_grouping = physical_grouping(reduction, child)?; if let Some(readout) = exact_readout(family) { let existing = self.nodes.iter().find_map(|(id, node)| { diff --git a/control_plane/src/query_plan/logical.rs b/control_plane/src/query_plan/logical.rs new file mode 100644 index 00000000..c22e0829 --- /dev/null +++ b/control_plane/src/query_plan/logical.rs @@ -0,0 +1,793 @@ +//! Typed residual operations compiled once by the control plane, never parsed at serving time. +use super::{ + FallbackPolicy, InstantExecution, QueryNodeId, QueryPlanEntry, QueryPlanError, QueryPlanNode, +}; +use promql_parser::{ + label::MatchOp, + parser::{self, Expr, LabelModifier, Offset, VectorSelector}, +}; +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +#[serde(tag = "kind", rename_all = "snake_case", deny_unknown_fields)] +pub enum LogicalOperator { + Scan { + metric: Option, + matchers: Vec, + range_ms: Option, + offset_ms: i64, + }, + UnaryNegate, + Aggregate { + operation: Aggregation, + grouping: Grouping, + }, + Binary { + operation: BinaryOperation, + return_bool: bool, + }, + Temporal { + operation: TemporalOperation, + }, + Sort { + descending: bool, + }, + HistogramQuantile, + Subquery { + range_ms: u64, + step_ms: u64, + offset_ms: i64, + }, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct Grouping { + pub labels: Vec, + pub without: bool, +} +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(deny_unknown_fields)] +pub struct LabelMatcher { + pub name: String, + pub value: String, + pub operation: LabelMatch, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum LabelMatch { + Equal, + NotEqual, + Regex, + NotRegex, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum Aggregation { + Sum, + Max, + Min, + Avg, + Count, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum BinaryOperation { + Add, + Sub, + Mul, + Div, + Mod, + Pow, + Equal, + NotEqual, + Less, + LessEqual, + Greater, + GreaterEqual, +} +#[derive(Debug, Clone, Copy, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum TemporalOperation { + Rate, + Increase, + Avg, + Max, + Min, + Sum, + Count, +} + +fn invalid(message: impl Into) -> QueryPlanError { + QueryPlanError::Invalid(message.into()) +} +fn millis(duration: std::time::Duration) -> Result { + u64::try_from(duration.as_millis()).map_err(|_| invalid("logical duration overflow")) +} +fn offset(value: &Option) -> Result { + match value { + None => Ok(0), + Some(Offset::Pos(d)) => i64::try_from(millis(*d)?).map_err(|_| invalid("offset overflow")), + Some(Offset::Neg(d)) => i64::try_from(millis(*d)?) + .map(|v| -v) + .map_err(|_| invalid("offset overflow")), + } +} + +impl LogicalOperator { + pub fn validate(&self, inputs: usize) -> Result<(), QueryPlanError> { + let expected = match self { + Self::Scan { .. } => 0, + Self::Binary { .. } | Self::HistogramQuantile => 2, + _ => 1, + }; + if inputs != expected { + return Err(invalid("logical operator input arity mismatch")); + } + if matches!( + self, + Self::Scan { + range_ms: Some(0), + .. + } + ) { + return Err(invalid("zero range")); + } + if let Self::Subquery { + range_ms, step_ms, .. + } = self + { + if *range_ms == 0 || *step_ms == 0 || range_ms / step_ms > 100_000 { + return Err(invalid("invalid or excessive subquery grid")); + } + } + Ok(()) + } +} + +struct Lower { + nodes: BTreeMap, + seen: BTreeMap, +} +impl Lower { + fn add(&mut self, node: QueryPlanNode) -> Result { + let key = serde_json::to_string(&node).map_err(|e| invalid(e.to_string()))?; + if let Some(id) = self.seen.get(&key) { + return Ok(*id); + } + let id = QueryNodeId(self.nodes.len() as u64); + self.nodes.insert(id, node); + self.seen.insert(key, id); + Ok(id) + } + fn operation( + &mut self, + operator: LogicalOperator, + inputs: Vec, + ) -> Result { + operator.validate(inputs.len())?; + self.add(QueryPlanNode::Logical { operator, inputs }) + } + fn scan( + &mut self, + s: &VectorSelector, + range_ms: Option, + ) -> Result { + if s.at.is_some() || !s.matchers.or_matchers.is_empty() { + return Err(invalid("logical @/OR selector is not supported")); + } + let matchers = s + .matchers + .matchers + .iter() + .map(|m| LabelMatcher { + name: m.name.clone(), + value: m.value.clone(), + operation: match m.op { + MatchOp::Equal => LabelMatch::Equal, + MatchOp::NotEqual => LabelMatch::NotEqual, + MatchOp::Re(_) => LabelMatch::Regex, + MatchOp::NotRe(_) => LabelMatch::NotRegex, + }, + }) + .collect(); + self.operation( + LogicalOperator::Scan { + metric: s.name.clone(), + matchers, + range_ms, + offset_ms: offset(&s.offset)?, + }, + vec![], + ) + } + fn lower(&mut self, expr: &Expr) -> Result { + match expr { + Expr::NumberLiteral(n) if n.val.is_finite() => { + self.add(QueryPlanNode::Scalar { value: n.val }) + } + Expr::Paren(p) => self.lower(&p.expr), + Expr::Unary(u) => { + let input = self.lower(&u.expr)?; + self.operation(LogicalOperator::UnaryNegate, vec![input]) + } + Expr::VectorSelector(s) => self.scan(s, None), + Expr::MatrixSelector(s) => self.scan(&s.vs, Some(millis(s.range)?)), + Expr::Subquery(s) => { + if s.at.is_some() { + return Err(invalid("logical subquery @ is unsupported")); + } + let input = self.lower(&s.expr)?; + self.operation( + LogicalOperator::Subquery { + range_ms: millis(s.range)?, + step_ms: millis( + s.step + .ok_or_else(|| invalid("explicit subquery step required"))?, + )?, + offset_ms: offset(&s.offset)?, + }, + vec![input], + ) + } + Expr::Aggregate(a) => { + if a.param.is_some() { + return Err(invalid("parameterized aggregate unsupported")); + } + let operation = match a.op.to_string().as_str() { + "sum" => Aggregation::Sum, + "max" => Aggregation::Max, + "min" => Aggregation::Min, + "avg" => Aggregation::Avg, + "count" => Aggregation::Count, + other => return Err(invalid(format!("unsupported logical aggregate {other}"))), + }; + let grouping = match &a.modifier { + None => Grouping { + labels: vec![], + without: false, + }, + Some(LabelModifier::Include(labels)) => Grouping { + labels: labels.labels.clone(), + without: false, + }, + Some(LabelModifier::Exclude(labels)) => Grouping { + labels: labels.labels.clone(), + without: true, + }, + }; + let input = self.lower(&a.expr)?; + self.operation( + LogicalOperator::Aggregate { + operation, + grouping, + }, + vec![input], + ) + } + Expr::Call(c) => { + let operator = match c.func.name { + "histogram_quantile" => LogicalOperator::HistogramQuantile, + "sort" => LogicalOperator::Sort { descending: false }, + "sort_desc" => LogicalOperator::Sort { descending: true }, + name => LogicalOperator::Temporal { + operation: match name { + "rate" => TemporalOperation::Rate, + "increase" => TemporalOperation::Increase, + "avg_over_time" => TemporalOperation::Avg, + "max_over_time" => TemporalOperation::Max, + "min_over_time" => TemporalOperation::Min, + "sum_over_time" => TemporalOperation::Sum, + "count_over_time" => TemporalOperation::Count, + _ => { + return Err(invalid(format!("unsupported logical function {name}"))) + } + }, + }, + }; + let inputs = c + .args + .args + .iter() + .map(|e| self.lower(e)) + .collect::, _>>()?; + self.operation(operator, inputs) + } + Expr::Binary(b) => { + if b.modifier.as_ref().is_some_and(|m| { + m.matching.is_some() + || !matches!(m.card, parser::VectorMatchCardinality::OneToOne) + }) { + return Err(invalid("logical explicit vector matching unsupported")); + } + let operation = match b.op.to_string().as_str() { + "+" => BinaryOperation::Add, + "-" => BinaryOperation::Sub, + "*" => BinaryOperation::Mul, + "/" => BinaryOperation::Div, + "%" => BinaryOperation::Mod, + "^" => BinaryOperation::Pow, + "==" => BinaryOperation::Equal, + "!=" => BinaryOperation::NotEqual, + "<" => BinaryOperation::Less, + "<=" => BinaryOperation::LessEqual, + ">" => BinaryOperation::Greater, + ">=" => BinaryOperation::GreaterEqual, + other => return Err(invalid(format!("unsupported logical binary {other}"))), + }; + let inputs = vec![self.lower(&b.lhs)?, self.lower(&b.rhs)?]; + self.operation( + LogicalOperator::Binary { + operation, + return_bool: b.return_bool(), + }, + inputs, + ) + } + _ => Err(invalid("unsupported logical expression")), + } + } +} + +impl QueryPlanEntry { + /// Lower a Planner-authorized native residual into typed backend operations. + /// Callers retain a separate external-native alternative for cost comparison. + pub fn compile_logical( + query_id: String, + canonical_promql: String, + instant: InstantExecution, + fallback: FallbackPolicy, + ) -> Result { + let expr = parser::parse(&canonical_promql).map_err(|e| invalid(e.to_string()))?; + let mut lower = Lower { + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + }; + let root = lower.lower(&expr)?; + let entry = Self { + query_id, + canonical_promql, + root, + nodes: lower.nodes, + instant, + fallback, + }; + entry.validate(&Default::default())?; + Ok(entry) + } + /// Promote only a wholly native entry; never discard selected summary bindings. + pub fn lower_native_residual(&self) -> Result { + if self.nodes.len() != 1 + || !matches!( + self.nodes.get(&self.root), + Some(QueryPlanNode::ExactFallback { .. }) + ) + { + return Err(invalid( + "logical residual promotion requires a whole native root", + )); + } + Self::compile_logical( + self.query_id.clone(), + self.canonical_promql.clone(), + self.instant, + self.fallback, + ) + } +} + +/// Match residuals by semantic IR equality, not display text or source names. +/// This ensures a subtree parsed for physical lowering is the subtree Planner kept. +pub(super) fn residual_nodes( + original: &str, + residual: &planner_types::pre_asap::QueryExpr, +) -> Result<(QueryNodeId, BTreeMap), QueryPlanError> { + fn visit<'a>(expr: &'a Expr, out: &mut Vec<&'a Expr>) { + out.push(expr); + match expr { + Expr::Paren(e) => visit(&e.expr, out), + Expr::Unary(e) => visit(&e.expr, out), + Expr::Subquery(e) => visit(&e.expr, out), + Expr::Aggregate(e) => visit(&e.expr, out), + Expr::Binary(e) => { + visit(&e.lhs, out); + visit(&e.rhs, out); + } + Expr::Call(e) => { + for input in &e.args.args { + visit(input, out); + } + } + _ => {} + } + } + let original = parser::parse(original).map_err(|e| invalid(e.to_string()))?; + let mut expressions = Vec::new(); + visit(&original, &mut expressions); + for expression in expressions { + if let Ok(candidate) = crate::query_parser::parse_query_expr_canonical( + &expression.to_string(), + planner_types::types::AccuracyTarget::Exact, + ) { + if &candidate == residual { + let mut lower = Lower { + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + }; + let root = lower.lower(expression)?; + return Ok((root, lower.nodes)); + } + } + } + Err(invalid( + "Planner residual does not match any original query subtree", + )) +} + +pub(super) fn binary_operator( + operator: &planner_types::post_asap::BinaryOperator, +) -> Result { + if operator.vector_match.is_some() { + return Err(invalid("explicit residual vector matching unsupported")); + } + let operation = match operator.kind.to_string().as_str() { + "+" => BinaryOperation::Add, + "-" => BinaryOperation::Sub, + "*" => BinaryOperation::Mul, + "/" => BinaryOperation::Div, + "%" => BinaryOperation::Mod, + "^" => BinaryOperation::Pow, + "=" | "==" => BinaryOperation::Equal, + "<>" | "!=" => BinaryOperation::NotEqual, + "<" => BinaryOperation::Less, + "<=" => BinaryOperation::LessEqual, + ">" => BinaryOperation::Greater, + ">=" => BinaryOperation::GreaterEqual, + other => { + return Err(invalid(format!( + "unsupported Planner binary operator {other}" + ))) + } + }; + Ok(LogicalOperator::Binary { + operation, + return_bool: false, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + fn instant() -> InstantExecution { + InstantExecution { + lookback_ms: 300_000, + full_history: false, + cumulative_readout: false, + } + } + + #[test] + fn complete_o11y_corpus_lowers_to_serialized_operations() { + // Every original workload occurrence must compile to an executable typed graph. + let corpus: serde_json::Value = + serde_json::from_str(include_str!("../../tests/fixtures/o11y_queries.json")).unwrap(); + for row in corpus["queries"].as_array().unwrap() { + let query = row["query"].as_str().unwrap(); + let entry = QueryPlanEntry::compile_logical( + row["id"].as_str().unwrap().into(), + query.into(), + instant(), + FallbackPolicy::Reject, + ) + .unwrap_or_else(|error| panic!("{query}: {error}")); + let encoded = serde_json::to_string(&entry).unwrap(); + let restored: QueryPlanEntry = serde_json::from_str(&encoded).unwrap(); + restored.validate(&Default::default()).unwrap(); + assert!(!restored + .nodes + .values() + .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. }))); + } + } + #[test] + fn residual_mapping_preserves_filters_and_rejects_different_sources() { + // Physical lowering must prove correspondence with the Planner-kept semantic subtree. + let query = "sum(rate(requests_total{job=\"api\"}[5m]))"; + let residual = crate::query_parser::parse_query_expr_canonical( + query, + planner_types::types::AccuracyTarget::Exact, + ) + .unwrap(); + let (_, nodes) = residual_nodes(query, &residual).unwrap(); + assert!(nodes.values().any(|node| matches!(node, QueryPlanNode::Logical { operator: LogicalOperator::Scan { matchers, .. }, .. } if matchers.iter().any(|m| m.name == "job" && m.value == "api")))); + assert!(residual_nodes("sum(rate(other_total[5m]))", &residual).is_err()); + } + #[test] + fn repeated_subexpressions_share_node_identity() { + // Serialized edges must retain CSE rather than duplicating raw work. + let entry = QueryPlanEntry::compile_logical( + "q".into(), + "sum(up) / sum(up)".into(), + instant(), + FallbackPolicy::Reject, + ) + .unwrap(); + let QueryPlanNode::Logical { inputs, .. } = &entry.nodes[&entry.root] else { + panic!("binary expected") + }; + assert_eq!(inputs[0], inputs[1]); + } + #[test] + fn malformed_operator_arity_is_rejected_at_installation() { + // A serialized graph cannot bypass the operation's input contract. + assert!(LogicalOperator::HistogramQuantile.validate(1).is_err()); + assert!(LogicalOperator::Subquery { + range_ms: 60_000, + step_ms: 0, + offset_ms: 0 + } + .validate(1) + .is_err()); + } +} + +/// Prove a physical-native substitute represents exactly the selected summary leaf. +/// A second Planner invocation is an equality witness, not a replacement selection. +pub(crate) fn selected_residual_nodes( + original: &str, + selected: &planner_types::post_asap::SummaryNode, +) -> Result<(QueryNodeId, BTreeMap), QueryPlanError> { + if !selected.guarantee.as_ref().is_some_and(|g| g.is_exact()) { + return Err(invalid( + "native residual substitution requires an exact selected value", + )); + } + fn visit<'a>(expr: &'a Expr, output: &mut Vec<&'a Expr>) { + output.push(expr); + match expr { + Expr::Paren(e) => visit(&e.expr, output), + Expr::Unary(e) => visit(&e.expr, output), + Expr::Subquery(e) => visit(&e.expr, output), + Expr::Aggregate(e) => visit(&e.expr, output), + Expr::Binary(e) => { + visit(&e.lhs, output); + visit(&e.rhs, output); + } + Expr::Call(e) => { + for input in &e.args.args { + visit(input, output); + } + } + _ => {} + } + } + let parsed = parser::parse(original).map_err(|e| invalid(e.to_string()))?; + let mut expressions = Vec::new(); + visit(&parsed, &mut expressions); + let mut matched = None; + for expression in expressions { + let Ok(canonical) = crate::query_parser::parse_query_expr_canonical( + &expression.to_string(), + planner_types::types::AccuracyTarget::Exact, + ) else { + continue; + }; + let Ok(witness) = crate::planner_selection::select_summary_default(&canonical) else { + continue; + }; + if witness.as_ref() == selected { + let mut lower = Lower { + nodes: BTreeMap::new(), + seen: BTreeMap::new(), + }; + let root = lower.lower(expression)?; + let candidate = (root, lower.nodes); + if matched + .as_ref() + .is_some_and(|previous| previous != &candidate) + { + return Err(invalid( + "ambiguous original subtrees share a Planner summary representation", + )); + } + matched = Some(candidate); + } + } + matched.ok_or_else(|| { + invalid("selected summary leaf has no semantically identical original subtree witness") + }) +} + +/// Read the original aggregate operation only after proving its selected-node identity. +/// Min and max share a Planner accumulator family, so the family name alone is insufficient. +pub(super) fn selected_aggregate_operator( + original: &str, + selected: &planner_types::post_asap::SummaryNode, +) -> Result { + let (root, nodes) = selected_residual_nodes(original, selected)?; + match nodes.get(&root) { + Some(QueryPlanNode::Logical { + operator: operator @ LogicalOperator::Aggregate { .. }, + .. + }) => Ok(operator.clone()), + _ => Err(invalid( + "selected value aggregation has no verified original aggregate operator", + )), + } +} + +#[cfg(test)] +mod hybrid_tests { + use super::*; + use crate::query_plan::{MaterializationBinding, PhysicalGrouping}; + #[test] + fn selected_summary_and_filtered_residual_share_installed_binary() { + // An unsupported filtered leaf must not discard its supported sibling's selected materialization. + let query = "sum_over_time(m[5m]) + sum_over_time(m{job=\"api\"}[5m])"; + let canonical = crate::query_parser::parse_query_expr_canonical( + query, + planner_types::types::AccuracyTarget::Exact, + ) + .unwrap(); + let selected = crate::planner_selection::select_summary_default(&canonical).unwrap(); + let entry = QueryPlanEntry::compile_bound_composable( + "hybrid".into(), + query.into(), + &selected, + InstantExecution { + lookback_ms: 300_000, + full_history: false, + cumulative_readout: false, + }, + FallbackPolicy::Reject, + |node, _| { + crate::physical::compiler::materialization_leaf_contract(node) + .map_err(QueryPlanError::Invalid)?; + Ok(MaterializationBinding { + materialization: asap_types::PolicyFingerprint(7), + metric: "m".into(), + sid_grouping: vec![], + output_grouping: PhysicalGrouping::PerEntity, + window_ms: 300_000, + readout_lookback_ms: Some(300_000), + }) + }, + ) + .unwrap(); + assert_eq!(entry.materialization_bindings().len(), 1); + assert!(entry.nodes.values().any(|node| matches!(node, QueryPlanNode::Logical { operator: LogicalOperator::Scan { matchers, .. }, .. } if matchers.iter().any(|m| m.name == "job" && m.value == "api")))); + assert!(matches!( + entry.nodes[&entry.root], + QueryPlanNode::Logical { + operator: LogicalOperator::Binary { .. }, + .. + } + )); + entry + .validate(&[asap_types::PolicyFingerprint(7)].into_iter().collect()) + .unwrap(); + } + + #[test] + fn different_filter_cannot_witness_selected_residual() { + // Equality includes filter predicates, not just family, source, or window. + let canonical = crate::query_parser::parse_query_expr_canonical( + "sum_over_time(m{job=\"api\"}[5m])", + planner_types::types::AccuracyTarget::Exact, + ) + .unwrap(); + let selected = crate::planner_selection::select_summary_default(&canonical).unwrap(); + assert!( + selected_residual_nodes("sum_over_time(m{job=\"worker\"}[5m])", &selected).is_err() + ); + } +} + +#[cfg(test)] +mod planner_workload_tests { + use super::*; + use crate::physical::compiler::{BackendLocalPlanningSnapshot, PhysicalCompiler}; + + fn lookback(expr: &Expr) -> u64 { + match expr { + Expr::MatrixSelector(e) => e.range.as_millis() as u64, + Expr::Subquery(e) => (e.range.as_millis() as u64).max(lookback(&e.expr)), + Expr::Aggregate(e) => lookback(&e.expr), + Expr::Paren(e) => lookback(&e.expr), + Expr::Unary(e) => lookback(&e.expr), + Expr::Binary(e) => lookback(&e.lhs).max(lookback(&e.rhs)), + Expr::Call(e) => e.args.args.iter().map(|e| lookback(e)).max().unwrap_or(0), + _ => 0, + } + } + + #[test] + fn whole_o11y_planner_candidate_lowers_every_original_query() { + // AST support alone is insufficient: the actual selected Planner forest must bind too. + let corpus: serde_json::Value = + serde_json::from_str(include_str!("../../tests/fixtures/o11y_queries.json")).unwrap(); + let mut fixture: serde_json::Value = serde_json::from_str(include_str!( + "../../../docs/examples/asapquery-planning-snapshot.json" + )) + .unwrap(); + let template = fixture["query_workload"]["repeating_queries"][0].clone(); + let mut seen = std::collections::BTreeSet::new(); + let mut entries = Vec::new(); + for row in corpus["queries"].as_array().unwrap() { + let query = row["query"].as_str().unwrap(); + if !seen.insert(query.to_string()) { + continue; + } + let mut entry = template.clone(); + entry["query"] = query.into(); + entry["requirements"]["accuracy"] = serde_json::json!({"explicit":"Exact"}); + let window = lookback(&parser::parse(query).unwrap()); + entry["time_selection"]["lookback"] = + (if window == 0 { 300_000 } else { window }).into(); + entries.push(entry); + } + fixture["query_workload"]["repeating_queries"] = entries.into(); + let snapshot: BackendLocalPlanningSnapshot = serde_json::from_value(fixture).unwrap(); + let (request, environment) = snapshot.planning_request().unwrap(); + assert!(request.local_raw_execution); + let plan = PhysicalCompiler.compile(request, environment).unwrap(); + assert_eq!(plan.query_plan.entries.len(), 24); + assert!(plan.query_plan.entries.values().all(|entry| !entry + .nodes + .values() + .any(|node| matches!(node, QueryPlanNode::ExactFallback { .. })))); + } + + #[test] + fn max_of_selected_values_uses_original_max_operator() { + // MinMax storage type does not authorize choosing min or replacing the selected operand graph. + let query = "max(sum_over_time(m[5m]) / 2)"; + let canonical = crate::query_parser::parse_query_expr_canonical( + query, + planner_types::types::AccuracyTarget::Exact, + ) + .unwrap(); + let selected = crate::planner_selection::select_summary_default(&canonical).unwrap(); + let operator = selected_aggregate_operator(query, &selected).unwrap(); + assert!(matches!( + operator, + LogicalOperator::Aggregate { + operation: Aggregation::Max, + .. + } + )); + } + + #[test] + fn ambiguous_extremum_witness_is_rejected() { + // Different readouts over the same MinMax state cannot be resolved by taking the first AST match. + let canonical = crate::query_parser::parse_query_expr_canonical( + "min(m)", + planner_types::types::AccuracyTarget::Exact, + ) + .unwrap(); + let selected = crate::planner_selection::select_summary_default(&canonical).unwrap(); + let maximum = crate::query_parser::parse_query_expr_canonical( + "max(m)", + planner_types::types::AccuracyTarget::Exact, + ) + .unwrap(); + let maximum = crate::planner_selection::select_summary_default(&maximum).unwrap(); + let result = selected_residual_nodes("min(m) + max(m)", &selected); + if selected == maximum { + assert!(result.is_err()); + } else { + let (root, nodes) = result.unwrap(); + assert!(matches!( + nodes[&root], + QueryPlanNode::Logical { + operator: LogicalOperator::Aggregate { + operation: Aggregation::Min, + .. + }, + .. + } + )); + } + } +} diff --git a/control_plane/tests/fixtures/o11y_queries.json b/control_plane/tests/fixtures/o11y_queries.json new file mode 100644 index 00000000..9ea29a17 --- /dev/null +++ b/control_plane/tests/fixtures/o11y_queries.json @@ -0,0 +1,296 @@ +{ + "upstream_revision": "3b9567319296366c6a73640d21826fc2fb26bc7e", + "eval_timestamp_ms": 1788891296000, + "source_files": [ + { + "path": "tasks-spec/prometheus_query/check-service-health.yaml", + "sha256": "387dff16abf294107f92cb81384c4ca3964206f1ba47d773671fd6ac7aed785c" + }, + { + "path": "tasks-spec/prometheus_query/promql-burn-rate-assessment.yaml", + "sha256": "6b3aee1e2adfc3c5b6004f29abfe374edc7294b86a55792d669f03fd122e878b" + }, + { + "path": "tasks-spec/prometheus_query/promql-cache-lag-vs-user-latency.yaml", + "sha256": "606e0dd923a013bc8475a0efbaafc72afa1ee6035e2f48d1c4bd5e23f164cec3" + }, + { + "path": "tasks-spec/prometheus_query/promql-cache-refresh-lag-peak.yaml", + "sha256": "6a88045045cab7ac6aba1c84968ecf8e7474cdd77315f2b4db1f2c89869243b3" + }, + { + "path": "tasks-spec/prometheus_query/promql-capacity-analysis.yaml", + "sha256": "0760992a9f1ad15bfcf91fe14ccab3d130232773cd4a9feaadb96ffeec2982fd" + }, + { + "path": "tasks-spec/prometheus_query/promql-discover-http-metric.yaml", + "sha256": "5a70c6fb1d575ec794252085afcef88e70b66214cf89aae3bfed8cc6600c78f9" + }, + { + "path": "tasks-spec/prometheus_query/promql-error-rate.yaml", + "sha256": "7756d896deddf03a8c3ebafc8b677d4d0caf953a55837d63b93c914f0dbb9d0a" + }, + { + "path": "tasks-spec/prometheus_query/promql-highest-backend-error-ratio.yaml", + "sha256": "39748917439d0201c5eca505f37b48f37d0fa39200c12d3ccd1fc2ce5f20183f" + }, + { + "path": "tasks-spec/prometheus_query/promql-label-matchers-service-errors.yaml", + "sha256": "db72f53039fba07ebc0b77eb16cdfa10a7cd57b06384c6a8540d95d7d6a94a2b" + }, + { + "path": "tasks-spec/prometheus_query/promql-offset-traffic-compare.yaml", + "sha256": "dca456d4948de758427725139c5123194d6e6de151815365eb12b5b801194f62" + }, + { + "path": "tasks-spec/prometheus_query/promql-order-latency-vs-traffic.yaml", + "sha256": "ee1df5d3d7727358c60236091e4f6c24bba77499f3928ae2a337983ad94e4018" + }, + { + "path": "tasks-spec/prometheus_query/promql-retry-backlog-triage.yaml", + "sha256": "3b11a1d3afb871558908b53a19d79bd38a659f890f244320b34eea8450cfef25" + }, + { + "path": "tasks-spec/prometheus_query/promql-subquery-peak-error-rate.yaml", + "sha256": "fbc6d1ea4386784ce9c6f1830f33640aa9f517c808d0909209d26d1d829bbee7" + }, + { + "path": "tasks-spec/prometheus_query/promql-topk-5xx-share.yaml", + "sha256": "d19c40a904b4d09560ad6782adf61dccf8ce0aa060a18da6c835fc81cdf3779e" + }, + { + "path": "tasks-spec/prometheus_query/query-cpu-metrics.yaml", + "sha256": "51058f02c2bd8616cdde085519e9139668b2eb5b5915afbabe40952062f8459c" + }, + { + "path": "tasks-spec/prometheus_query/query-memory-usage.yaml", + "sha256": "7ebf61f36fa97d5398cb64ba6543737349d087b91a8719ee62e3568ca23436b9" + } + ], + "queries": [ + { + "id": "check-service-health:rubric:0", + "task_id": "check-service-health", + "source_path": "tasks-spec/prometheus_query/check-service-health.yaml", + "rubric_index": 0, + "query": "sum(up)", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "check-service-health:rubric:1", + "task_id": "check-service-health", + "source_path": "tasks-spec/prometheus_query/check-service-health.yaml", + "rubric_index": 1, + "query": "max((sum by (job) (rate(http_requests_total{status=~\"5..\",job=~\"user-service|order-service|payment-service\"}[6h]))) / (sum by (job) (rate(http_requests_total{job=~\"user-service|order-service|payment-service\"}[6h]))))", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-burn-rate-assessment:rubric:0", + "task_id": "promql-burn-rate-assessment", + "source_path": "tasks-spec/prometheus_query/promql-burn-rate-assessment.yaml", + "rubric_index": 0, + "query": "sum(increase(http_requests_total{job=\"payment-service\",status=~\"5..\"}[1h])) / sum(increase(http_requests_total{job=\"payment-service\"}[1h]))", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-burn-rate-assessment:rubric:1", + "task_id": "promql-burn-rate-assessment", + "source_path": "tasks-spec/prometheus_query/promql-burn-rate-assessment.yaml", + "rubric_index": 1, + "query": "sum(increase(http_requests_total{job=\"payment-service\",status=~\"5..\"}[1h] offset 6h)) / sum(increase(http_requests_total{job=\"payment-service\"}[1h] offset 6h))", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-cache-lag-vs-user-latency:rubric:0", + "task_id": "promql-cache-lag-vs-user-latency", + "source_path": "tasks-spec/prometheus_query/promql-cache-lag-vs-user-latency.yaml", + "rubric_index": 0, + "query": "max_over_time(service_cache_refresh_lag_seconds{job=\"user-service\"}[12h])", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-cache-lag-vs-user-latency:rubric:1", + "task_id": "promql-cache-lag-vs-user-latency", + "source_path": "tasks-spec/prometheus_query/promql-cache-lag-vs-user-latency.yaml", + "rubric_index": 1, + "query": "max_over_time(histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"user-service\"}[5m])) by (le))[12h:1m])", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-cache-lag-vs-user-latency:rubric:2", + "task_id": "promql-cache-lag-vs-user-latency", + "source_path": "tasks-spec/prometheus_query/promql-cache-lag-vs-user-latency.yaml", + "rubric_index": 2, + "query": "service_cache_refresh_lag_seconds{job=\"user-service\"}", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-cache-refresh-lag-peak:rubric:0", + "task_id": "promql-cache-refresh-lag-peak", + "source_path": "tasks-spec/prometheus_query/promql-cache-refresh-lag-peak.yaml", + "rubric_index": 0, + "query": "max_over_time(service_cache_refresh_lag_seconds{job=\"user-service\"}[12h])", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-cache-refresh-lag-peak:rubric:1", + "task_id": "promql-cache-refresh-lag-peak", + "source_path": "tasks-spec/prometheus_query/promql-cache-refresh-lag-peak.yaml", + "rubric_index": 1, + "query": "service_cache_refresh_lag_seconds{job=\"user-service\"}", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-capacity-analysis:rubric:0", + "task_id": "promql-capacity-analysis", + "source_path": "tasks-spec/prometheus_query/promql-capacity-analysis.yaml", + "rubric_index": 0, + "query": "sum(rate(process_cpu_seconds_total[1h]))", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-capacity-analysis:rubric:1", + "task_id": "promql-capacity-analysis", + "source_path": "tasks-spec/prometheus_query/promql-capacity-analysis.yaml", + "rubric_index": 1, + "query": "sum(process_resident_memory_bytes)", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-capacity-analysis:rubric:2", + "task_id": "promql-capacity-analysis", + "source_path": "tasks-spec/prometheus_query/promql-capacity-analysis.yaml", + "rubric_index": 2, + "query": "max_over_time(sum(process_resident_memory_bytes)[6h:1m])", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-discover-http-metric:rubric:0", + "task_id": "promql-discover-http-metric", + "source_path": "tasks-spec/prometheus_query/promql-discover-http-metric.yaml", + "rubric_index": 0, + "query": "sum(rate(http_requests_total{job=~\"user-service|order-service|payment-service\"}[5m]))", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-error-rate:rubric:0", + "task_id": "promql-error-rate", + "source_path": "tasks-spec/prometheus_query/promql-error-rate.yaml", + "rubric_index": 0, + "query": "sum(rate(http_requests_total{status=~\"5..\",job=~\"user-service|order-service|payment-service\"}[1h])) / sum(rate(http_requests_total{job=~\"user-service|order-service|payment-service\"}[1h]))", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-highest-backend-error-ratio:rubric:2", + "task_id": "promql-highest-backend-error-ratio", + "source_path": "tasks-spec/prometheus_query/promql-highest-backend-error-ratio.yaml", + "rubric_index": 2, + "query": "(sum by (job) (increase(http_requests_total{status=~\"5..\",job=~\"user-service|order-service|payment-service\"}[6h]))) / (sum by (job) (increase(http_requests_total{job=~\"user-service|order-service|payment-service\"}[6h])))", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-highest-backend-error-ratio:rubric:3", + "task_id": "promql-highest-backend-error-ratio", + "source_path": "tasks-spec/prometheus_query/promql-highest-backend-error-ratio.yaml", + "rubric_index": 3, + "query": "(sum by (job) (increase(http_requests_total{status=~\"5..\",job=~\"user-service|order-service|payment-service\"}[6h]))) / (sum by (job) (increase(http_requests_total{job=~\"user-service|order-service|payment-service\"}[6h])))", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-label-matchers-service-errors:rubric:0", + "task_id": "promql-label-matchers-service-errors", + "source_path": "tasks-spec/prometheus_query/promql-label-matchers-service-errors.yaml", + "rubric_index": 0, + "query": "sum by (job) (increase(http_requests_total{status=~\"5..\",job=~\".+-service\"}[24h])) > 0", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-offset-traffic-compare:rubric:0", + "task_id": "promql-offset-traffic-compare", + "source_path": "tasks-spec/prometheus_query/promql-offset-traffic-compare.yaml", + "rubric_index": 0, + "query": "sum(rate(http_requests_total{job=\"order-service\"}[5m]))", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-offset-traffic-compare:rubric:1", + "task_id": "promql-offset-traffic-compare", + "source_path": "tasks-spec/prometheus_query/promql-offset-traffic-compare.yaml", + "rubric_index": 1, + "query": "sum(rate(http_requests_total{job=\"order-service\"}[5m] offset 1h))", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-order-latency-vs-traffic:rubric:0", + "task_id": "promql-order-latency-vs-traffic", + "source_path": "tasks-spec/prometheus_query/promql-order-latency-vs-traffic.yaml", + "rubric_index": 0, + "query": "sum(rate(http_requests_total{job=\"order-service\"}[5m]))", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-order-latency-vs-traffic:rubric:1", + "task_id": "promql-order-latency-vs-traffic", + "source_path": "tasks-spec/prometheus_query/promql-order-latency-vs-traffic.yaml", + "rubric_index": 1, + "query": "histogram_quantile(0.95, sum(rate(http_request_duration_seconds_bucket{job=\"order-service\"}[5m])) by (le))", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-retry-backlog-triage:rubric:0", + "task_id": "promql-retry-backlog-triage", + "source_path": "tasks-spec/prometheus_query/promql-retry-backlog-triage.yaml", + "rubric_index": 0, + "query": "max_over_time(service_retry_queue_depth{job=~\".+\"}[6h])", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-retry-backlog-triage:rubric:1", + "task_id": "promql-retry-backlog-triage", + "source_path": "tasks-spec/prometheus_query/promql-retry-backlog-triage.yaml", + "rubric_index": 1, + "query": "max_over_time(service_retry_queue_depth{job=\"order-service\"}[6h])", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-subquery-peak-error-rate:rubric:0", + "task_id": "promql-subquery-peak-error-rate", + "source_path": "tasks-spec/prometheus_query/promql-subquery-peak-error-rate.yaml", + "rubric_index": 0, + "query": "max_over_time((sum(rate(http_requests_total{job=\"order-service\",status=~\"5..\"}[5m])) / sum(rate(http_requests_total{job=\"order-service\"}[5m])))[6h:1m])", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "promql-topk-5xx-share:rubric:0", + "task_id": "promql-topk-5xx-share", + "source_path": "tasks-spec/prometheus_query/promql-topk-5xx-share.yaml", + "rubric_index": 0, + "query": "sum(increase(http_requests_total{job=\"order-service\",status=~\"5..\"}[24h])) / sum(increase(http_requests_total{status=~\"5..\"}[24h]))", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "query-cpu-metrics:rubric:0", + "task_id": "query-cpu-metrics", + "source_path": "tasks-spec/prometheus_query/query-cpu-metrics.yaml", + "rubric_index": 0, + "query": "sum by (job) (rate(process_cpu_seconds_total{job=~\".+\"}[6h]))", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "query-cpu-metrics:rubric:2", + "task_id": "query-cpu-metrics", + "source_path": "tasks-spec/prometheus_query/query-cpu-metrics.yaml", + "rubric_index": 2, + "query": "sort_desc(sum by (job) (rate(process_cpu_seconds_total{job=~\".+\"}[6h])))", + "eval_timestamp_ms": 1788891296000 + }, + { + "id": "query-memory-usage:rubric:0", + "task_id": "query-memory-usage", + "source_path": "tasks-spec/prometheus_query/query-memory-usage.yaml", + "rubric_index": 0, + "query": "sum(avg_over_time(process_resident_memory_bytes{job=~\".+\"}[6h]))", + "eval_timestamp_ms": 1788891296000 + } + ] +} diff --git a/data_plane/examples/logical_dag_replay.rs b/data_plane/examples/logical_dag_replay.rs new file mode 100644 index 00000000..58f770f7 --- /dev/null +++ b/data_plane/examples/logical_dag_replay.rs @@ -0,0 +1,103 @@ +//! Verify typed residual execution against an external Prometheus reference. +//! This is operator conformance only, not Planner selection or acceleration evidence. +use control_plane::query_plan::{FallbackPolicy, InstantExecution, QueryPlanEntry}; +use data_plane::{ + drivers::ingest::prometheus_remote_write::CanonicalSample, + query_engines::{ + asap_query_engine::logical_dag::{execute_prepared, PreparedSamples}, + EngineError, QueryResult, + }, +}; +use serde::Deserialize; +use serde_json::{json, Value}; +use std::{ + collections::BTreeMap, + io::{BufRead, BufReader}, +}; +#[derive(Deserialize)] +struct Row { + metric: String, + labels: BTreeMap, + timestamp_ms: i64, + value: f64, +} +fn response(result: QueryResult) -> Value { + let metric = |keys: Option>, values: Vec| { + keys.unwrap_or_default() + .into_iter() + .zip(values) + .collect::>() + }; + let data = match result { + QueryResult::Vector(v) => { + json!({"resultType":"vector","result": v.values.into_iter().map(|x| json!({"metric":metric(x.label_keys_override,x.labels.labels),"value":[v.timestamp as f64/1000.,x.value.to_string()]})).collect::>()}) + } + QueryResult::Matrix(v) => { + json!({"resultType":"matrix","result":v.values.into_iter().map(|x| json!({"metric":metric(x.label_keys_override,x.labels.labels),"values":x.samples.into_iter().map(|s| json!([s.timestamp as f64/1000.,s.value.to_string()])).collect::>()})).collect::>()}) + } + }; + json!({"status":"success","data":data}) +} +fn main() -> Result<(), Box> { + let args: Vec<_> = std::env::args().collect(); + if args.len() < 3 { + return Err("usage: logical_dag_replay METRICS.jsonl QUERIES.json [EVAL_MS ...]".into()); + } + let rows = BufReader::new(std::fs::File::open(&args[1])?) + .lines() + .map(|line| -> Result<_, Box> { + let row: Row = serde_json::from_str(&line?)?; + Ok(CanonicalSample { + series_key: serde_json::to_string(&(&row.metric, &row.labels))?, + metric: row.metric, + labels: row.labels.into_iter().collect(), + timestamp_ms: row.timestamp_ms, + value: Some(row.value), + }) + }) + .collect::, _>>()?; + let prepared = PreparedSamples::new(&rows)?; + let corpus: Value = serde_json::from_reader(std::fs::File::open(&args[2])?)?; + let times: Vec = if args.len() > 3 { + args[3..] + .iter() + .map(|s| s.parse()) + .collect::>()? + } else { + vec![corpus["eval_timestamp_ms"] + .as_u64() + .ok_or("missing eval timestamp")?] + }; + let mut results = Vec::new(); + for at in times { + for entry in corpus["queries"].as_array().ok_or("missing queries")? { + let q = entry["query"].as_str().ok_or("missing query")?; + let start = std::time::Instant::now(); + let plan = QueryPlanEntry::compile_logical( + entry["id"].to_string(), + q.to_owned(), + InstantExecution { + lookback_ms: 300_000, + full_history: false, + cumulative_readout: true, + }, + FallbackPolicy::ExactBackend, + )?; + let result = execute_prepared(&plan, &prepared, at, |_, _| { + Err(EngineError::capability_miss( + "conformance", + "unexpected summary callback", + )) + }); + results.push(match result { + Ok(result) => json!({"id":entry["id"],"query":q,"evaluation_ms":at,"route":"typed_residual_conformance","status":"success","response":response(result),"elapsed_ns":start.elapsed().as_nanos()}), + Err(err) => json!({"id":entry["id"],"query":q,"evaluation_ms":at,"route":"failed","status":"error","error":err.to_string()}) + }); + } + } + println!( + "{}", + serde_json::to_string_pretty(&json!({"sample_count":rows.len(),"results":results}))? + ); + Ok(()) +} diff --git a/data_plane/src/drivers/ingest/prometheus_remote_write.rs b/data_plane/src/drivers/ingest/prometheus_remote_write.rs index 1b8832fc..21c5b7b0 100644 --- a/data_plane/src/drivers/ingest/prometheus_remote_write.rs +++ b/data_plane/src/drivers/ingest/prometheus_remote_write.rs @@ -96,6 +96,7 @@ struct ReceiverInner { ingest: Arc, dedup: Mutex, stats: Arc, + raw_store: Arc, } #[derive(Default)] @@ -156,16 +157,29 @@ pub struct CanonicalSample { impl PrometheusRemoteWriteReceiver { pub fn new(config: PrometheusRemoteWriteConfig, ingest: Arc) -> Self { + Self::new_with_raw_store(config, ingest, Arc::new(Default::default())) + } + + pub fn new_with_raw_store( + config: PrometheusRemoteWriteConfig, + ingest: Arc, + raw_store: Arc, + ) -> Self { Self { inner: Arc::new(ReceiverInner { config, ingest, dedup: Mutex::new(DedupState::default()), stats: Arc::new(RemoteWriteStats::default()), + raw_store, }), } } + pub fn raw_store(&self) -> Arc { + self.inner.raw_store.clone() + } + pub fn config(&self) -> &PrometheusRemoteWriteConfig { &self.inner.config } @@ -180,7 +194,24 @@ impl PrometheusRemoteWriteReceiver { let mut state = self.inner.dedup.lock().map_err(|e| e.to_string())?; state.input_closed = true; } - self.inner.ingest.router.drain().await + self.inner.ingest.router.drain().await?; + // Finite-input preparation includes the residual index, so its build + // cost is not silently amortized into the first query's read cost. + if self.inner.raw_store.sample_count() > 0 { + let plan = self + .inner + .ingest + .physical_plan_snapshot() + .ok_or_else(|| "active plan disappeared during input drain".to_string())?; + self.inner + .raw_store + .snapshot( + plan.precompute_plan.envelope.plan_id, + plan.precompute_plan.envelope.plan_version, + ) + .map_err(|error| error.to_string())?; + } + Ok(()) } /// Decode, validate, deduplicate, and enqueue one whole v1 request. @@ -295,6 +326,32 @@ impl PrometheusRemoteWriteReceiver { .router .try_route_group_batch_atomic(messages)?; + let mut raw_metrics = std::collections::BTreeSet::new(); + let mut all_metrics = false; + for entry in physical_plan.query_plan.entries.values() { + for node in entry.nodes.values() { + if let control_plane::query_plan::QueryPlanNode::Logical { + operator: + control_plane::query_plan::logical::LogicalOperator::Scan { metric, .. }, + .. + } = node + { + if let Some(metric) = metric { + raw_metrics.insert(metric.clone()); + } else { + all_metrics = true; + } + } + } + } + self.inner.raw_store.append_admitted( + plan_identity.0, + plan_identity.1, + &new_samples, + &raw_metrics, + all_metrics, + ); + for ((plan_id, plan_version, series, timestamp), value) in batch_values { dedup .values @@ -552,6 +609,12 @@ mod tests { } fn physical_config(streaming: StreamingConfig) -> HotReloadStreamingConfig { + physical_config_with_raw(streaming, false) + } + fn physical_config_with_raw( + streaming: StreamingConfig, + retain_raw: bool, + ) -> HotReloadStreamingConfig { use control_plane::physical::compiler::{ FrameIdentityContract, IngestContract, IngestProtocol, PlanEnvelope, PrecomputePlan, SequenceScope, TimestampUnit, TransmissionPlan, PLANNER_REVISION, @@ -566,7 +629,7 @@ mod tests { planner_revision: PLANNER_REVISION.into(), capability_snapshot_id: "test".into(), }; - let active = ActivePhysicalPlan { + let mut active = ActivePhysicalPlan { precompute_plan: PrecomputePlan { envelope: envelope.clone(), ingest: IngestContract { @@ -596,6 +659,38 @@ mod tests { query_plan: Arc::new(control_plane::query_plan::QueryPlan::empty()), storage_routing: Arc::new(BackendStorageRouting::empty()), }; + if retain_raw { + use control_plane::query_plan::{ + FallbackPolicy, InstantExecution, QueryNodeId, QueryPlanEntry, QueryPlanNode, + }; + let id = QueryNodeId(0); + let query = QueryPlanEntry { + query_id: "raw".into(), + canonical_promql: "requests_total".into(), + root: id, + nodes: std::collections::BTreeMap::from([( + id, + QueryPlanNode::Logical { + operator: control_plane::query_plan::logical::LogicalOperator::Scan { + metric: Some("requests_total".into()), + matchers: vec![], + range_ms: None, + offset_ms: 0, + }, + inputs: vec![], + }, + )]), + instant: InstantExecution { + lookback_ms: 300_000, + full_history: false, + cumulative_readout: true, + }, + fallback: FallbackPolicy::ExactBackend, + }; + let mut plan = control_plane::query_plan::QueryPlan::empty(); + plan.entries.insert("requests_total".into(), query); + active.query_plan = Arc::new(plan); + } HotReloadStreamingConfig::from_active(HotReloadActivePhysicalPlan::new(active)) } @@ -616,6 +711,11 @@ mod tests { } fn configured_receiver() -> (PrometheusRemoteWriteReceiver, mpsc::Receiver) { + configured_receiver_with_raw(false) + } + fn configured_receiver_with_raw( + retain_raw: bool, + ) -> (PrometheusRemoteWriteReceiver, mpsc::Receiver) { use asap_types::enums::WindowKind; use asap_types::{AggregationConfig, AggregationType, KeyByLabelNames}; let aggregation = AggregationConfig { @@ -643,7 +743,7 @@ mod tests { router: SeriesRouter::new(vec![sender]), samples_ingested: AtomicU64::new(0), samples_blocked_by_schema_barrier: AtomicU64::new(0), - hot_reload_config: physical_config(streaming), + hot_reload_config: physical_config_with_raw(streaming, retain_raw), pass_raw_samples: false, sketch_snapshots: dashmap::DashMap::new(), series_resolver: Arc::new(super::super::SeriesIdResolver::new()), @@ -679,6 +779,39 @@ mod tests { }) } + // A rejected or duplicate Remote Write request never changes retained raw input. + #[test] + fn retained_raw_samples_follow_atomic_admission_and_deduplication() { + let (receiver, _worker) = configured_receiver_with_raw(true); + receiver.accept(&one_sample(1.0)).unwrap(); + receiver.accept(&one_sample(1.0)).unwrap(); + assert_eq!(receiver.raw_store().sample_count(), 1); + assert!(receiver.accept(&one_sample(2.0)).is_err()); + assert_eq!(receiver.raw_store().sample_count(), 1); + for timestamp in 101..108 { + let bytes = snap::raw::Decoder::new() + .decompress_vec(&one_sample(1.0)) + .unwrap(); + let mut write = WriteRequest::decode(bytes.as_slice()).unwrap(); + write.timeseries[0].samples[0].timestamp = timestamp; + receiver.accept(&compressed(write)).unwrap(); + } + assert_eq!(receiver.raw_store().sample_count(), 8); + let bytes = snap::raw::Decoder::new() + .decompress_vec(&one_sample(1.0)) + .unwrap(); + let mut write = WriteRequest::decode(bytes.as_slice()).unwrap(); + write.timeseries[0].samples[0].timestamp = 108; + assert!(matches!( + receiver.accept(&compressed(write)), + Err(RemoteWriteError::Backpressure(_)) + )); + assert_eq!(receiver.raw_store().sample_count(), 8); + let (native, _worker) = configured_receiver(); + native.accept(&one_sample(1.0)).unwrap(); + assert_eq!(native.raw_store().sample_count(), 0); + } + // Closing finite input prevents writes racing behind the completion barrier. #[tokio::test] async fn finite_input_drain_seals_receiver_and_propagates_worker_failure() { diff --git a/data_plane/src/drivers/query/servers/http.rs b/data_plane/src/drivers/query/servers/http.rs index 088cc0dd..76cddd3b 100644 --- a/data_plane/src/drivers/query/servers/http.rs +++ b/data_plane/src/drivers/query/servers/http.rs @@ -569,6 +569,7 @@ impl HttpServer { let range_query_endpoint = adapter.get_range_query_endpoint(); let app = Router::new() + .route("/api/v1/precompute/drain", post(handle_precompute_drain)) .route(query_endpoint, get(handle_instant_query)) .route(query_endpoint, post(handle_instant_query_post)) .route(range_query_endpoint, get(handle_range_query)) @@ -1534,6 +1535,74 @@ async fn process_via_router( } } +fn extract_logical_provenance( + value: &mut serde_json::Value, +) -> Option> { + let warnings = value.get_mut("warnings")?.as_array_mut()?; + let mut route = None; + let mut stats = None; + let mut found = false; + let mut malformed = false; + warnings.retain(|warning| { + let Some(text) = warning.as_str() else { + return true; + }; + if let Some(mode) = text.strip_prefix("asap_execution:") { + found = true; + if route.replace(mode.to_owned()).is_some() { + malformed = true; + } + return false; + } + if let Some(text) = text.strip_prefix("asap_logical_stats:") { + found = true; + let values = text.split(',').collect::>(); + let parsed = if values.len() == 3 { + values[0] + .strip_prefix("raw=") + .and_then(|v| v.parse::().ok()) + .zip( + values[1] + .strip_prefix("summary=") + .and_then(|v| v.parse::().ok()), + ) + .zip( + values[2] + .strip_prefix("memo_hits=") + .and_then(|v| v.parse::().ok()), + ) + .map(|((raw, summary), memo)| (raw, summary, memo)) + } else { + None + }; + if stats.is_some() || parsed.is_none() { + malformed = true; + } + stats = parsed; + return false; + } + true + }); + if !found { + return None; + } + let Some((raw, summary, memo)) = stats else { + return Some(Err(())); + }; + let expected = if raw > 0 && summary > 0 { + "hybrid" + } else if summary == 0 { + "raw_dag" + } else { + "asap" + }; + if malformed || route.as_deref().is_some_and(|mode| mode != expected) { + Some(Err(())) + } else { + Some(Ok((raw, summary, memo))) + } +} + /// Append a `data_source: ` info-line to the response JSON's /// `infos` array (Prometheus 3.0-style, mirrors the wire-format /// extension the `GorillaQueryEngine` documents in §6 of @@ -1544,7 +1613,7 @@ async fn process_via_router( async fn annotate_data_source(response: Response, data_source_id: &'static str) -> Response { use axum::body::to_bytes; - let (parts, body) = response.into_parts(); + let (mut parts, body) = response.into_parts(); // Adapter responses are bounded JSON objects; cap at 16 MiB to // bracket pathological cases without blowing memory. let bytes = match to_bytes(body, 16 * 1024 * 1024).await { @@ -1563,6 +1632,52 @@ async fn annotate_data_source(response: Response, data_source_id: &'static str) return Response::from_parts(parts, axum::body::Body::from(bytes)); } }; + if data_source_id == "asap_query" { + if let Some(provenance) = extract_logical_provenance(&mut value) { + let (route, detail) = match provenance { + Ok((raw, summary, memo)) => { + for (name, count) in [ + ("x-asap-raw-scan-evaluations", raw), + ("x-asap-summary-readout-evaluations", summary), + ("x-asap-memo-hits", memo), + ] { + parts.headers.insert( + name, + axum::http::HeaderValue::from_str(&count.to_string()).unwrap(), + ); + } + if raw > 0 || summary == 0 { + ( + "exact_fallback", + if summary > 0 { "hybrid" } else { "local_raw" }, + ) + } else { + ("warm", "asap") + } + } + Err(()) => ("failed", "invalid_provenance"), + }; + parts.headers.insert( + "x-asap-execution", + axum::http::HeaderValue::from_static(route), + ); + parts.headers.insert( + "x-asap-execution-detail", + axum::http::HeaderValue::from_static(detail), + ); + if let Some(map) = value.as_object_mut() { + if let Some(infos) = map + .entry("infos") + .or_insert_with(|| serde_json::json!([])) + .as_array_mut() + { + infos.push(serde_json::json!(format!("execution: {detail}"))); + } + } + } + } + // The body changed; an adapter's original Content-Length is no longer valid. + parts.headers.remove(axum::http::header::CONTENT_LENGTH); if let serde_json::Value::Object(map) = &mut value { let infos_entry = map .entry("infos".to_string()) @@ -5673,6 +5788,9 @@ async fn handle_store_metrics(State(state): State) -> axum::response:: let body = serde_json::json!({ "status": "success", "sid_count": timestamps.len(), + "approx_resident_bytes": state.sketch_index.approx_resident_bytes(), + "raw_store_estimated_bytes": state.remote_write.as_ref().map(|receiver| receiver.raw_store().estimated_bytes()).unwrap_or(0), + "raw_store_samples": state.remote_write.as_ref().map(|receiver| receiver.raw_store().sample_count()).unwrap_or(0), "earliest_timestamps_per_sid": timestamps}); (StatusCode::OK, axum::Json(body)).into_response() } @@ -6788,3 +6906,44 @@ async fn handle_delete_backfill_job( "job_id": job_id}); (StatusCode::OK, axum::Json(body)).into_response() } + +#[cfg(test)] +mod logical_provenance_tests { + use super::*; + + #[tokio::test] + async fn hybrid_execution_is_fallback_with_measured_branch_counts() { + // A successful mixed graph must never inherit the pure-ASAP route from its engine name. + let response = Json(serde_json::json!({"status":"success", "warnings":[ + "asap_execution:hybrid", "asap_logical_stats:raw=2,summary=1,memo_hits=3" + ], "data":{"resultType":"vector", "result":[]}})) + .into_response(); + let response = annotate_data_source(response, "asap_query").await; + assert_eq!(response.headers()["x-asap-execution"], "exact_fallback"); + assert_eq!(response.headers()["x-asap-execution-detail"], "hybrid"); + assert_eq!( + response.headers()["x-asap-summary-readout-evaluations"], + "1" + ); + let bytes = axum::body::to_bytes(response.into_body(), 4096) + .await + .unwrap(); + let value: serde_json::Value = serde_json::from_slice(&bytes).unwrap(); + assert!(value["warnings"].as_array().unwrap().is_empty()); + } + + #[test] + fn partial_result_warning_survives_internal_metadata_extraction() { + // Only internal metadata is removed; incomplete-result warnings still invalidate comparison. + let mut value = serde_json::json!({"warnings":["partial data", "asap_execution:raw_dag", "asap_logical_stats:raw=1,summary=0,memo_hits=0"]}); + assert_eq!(extract_logical_provenance(&mut value), Some(Ok((1, 0, 0)))); + assert_eq!(value["warnings"], serde_json::json!(["partial data"])); + } + + #[test] + fn contradictory_provenance_is_not_warm() { + // Claimed route cannot override the observed raw branch count. + let mut value = serde_json::json!({"warnings":["asap_execution:asap", "asap_logical_stats:raw=1,summary=1,memo_hits=0"]}); + assert_eq!(extract_logical_provenance(&mut value), Some(Err(()))); + } +} diff --git a/data_plane/src/main.rs b/data_plane/src/main.rs index 0ca072a2..2e67cc15 100644 --- a/data_plane/src/main.rs +++ b/data_plane/src/main.rs @@ -740,6 +740,7 @@ async fn main() -> Result<()> { // (PR E phase 2). Without sharing the handle, ASAPQueryEngine // would take a one-time snapshot at construction and ignore // subsequent swaps. + let raw_store = Arc::new(data_plane::query_engines::raw_store::RawSampleStore::default()); let engine = { let mut engine = ASAPQueryEngine::new_with_hot_reload( hot_reload_config.clone(), @@ -751,7 +752,8 @@ async fn main() -> Result<()> { // EngineError::CapabilityMiss when the ASAP tier is empty // / ghost / unknown. .with_sketch_index(sketch_index.clone()) - .with_active_physical_plan(active_physical_plan.clone()); + .with_active_physical_plan(active_physical_plan.clone()) + .with_raw_store(raw_store.clone()); if let Some(control_plane_endpoint) = args.control_plane_endpoint.as_ref() { info!( "Capability-miss notifications enabled → {}", @@ -1045,7 +1047,7 @@ async fn main() -> Result<()> { } if args.enable_remote_write || args.profile == RuntimeProfile::Asapquery { - let receiver = PrometheusRemoteWriteReceiver::new( + let receiver = PrometheusRemoteWriteReceiver::new_with_raw_store( PrometheusRemoteWriteConfig { max_compressed_bytes: args.remote_write_max_compressed_bytes, max_decompressed_bytes: args.remote_write_max_decompressed_bytes, @@ -1057,6 +1059,7 @@ async fn main() -> Result<()> { precompute_ingest_state .clone() .expect("precompute ingest state is always constructed"), + raw_store.clone(), ); info!("Prometheus Remote Write v1 enabled at POST /api/v1/write"); server = server.with_remote_write(receiver); diff --git a/data_plane/src/query_engines/asap_query_engine/engine.rs b/data_plane/src/query_engines/asap_query_engine/engine.rs index 042cc285..6935dce1 100644 --- a/data_plane/src/query_engines/asap_query_engine/engine.rs +++ b/data_plane/src/query_engines/asap_query_engine/engine.rs @@ -119,6 +119,7 @@ pub struct ASAPQueryEngine { /// Generation-consistent physical snapshot used by the production query /// path. QueryPlan and BackendPlan must never be sampled separately. active_physical_plan: Option, + raw_store: Option>, } impl ASAPQueryEngine { @@ -150,9 +151,198 @@ impl ASAPQueryEngine { sketch_index: None, archive_engine: None, active_physical_plan: None, + raw_store: None, } } + pub fn with_raw_store( + mut self, + store: Arc, + ) -> Self { + self.raw_store = Some(store); + self + } + + fn execute_logical_entry( + &self, + physical: &crate::storage_engines::types::ActivePhysicalPlan, + entry: &control_plane::query_plan::QueryPlanEntry, + samples: &super::logical_dag::PreparedSamples, + at: u64, + ) -> Result< + ( + crate::query_engines::query_result::QueryResult, + super::logical_dag::ExecutionStats, + ), + crate::query_engines::EngineError, + > { + use crate::query_engines::EngineError; + super::logical_dag::execute_prepared_with_stats( + entry, + samples, + at, + |root, evaluation_ms| { + let mut subtree = entry.clone(); + subtree.root = root; + let reachable = subtree.topological_order().map_err(|e| { + EngineError::capability_miss("installed_logical_dag", e.to_string()) + })?; + subtree.nodes.retain(|id, _| reachable.contains(id)); + let bindings = subtree.materialization_bindings(); + let windows: std::collections::BTreeSet> = + bindings.iter().map(|b| b.readout_lookback_ms).collect(); + if windows.len() != 1 || windows.contains(&None) || windows.contains(&Some(0)) { + return Err(EngineError::capability_miss( + "installed_logical_dag", + "bound subtree requires one explicit positive window", + )); + } + subtree.instant.lookback_ms = windows + .first() + .copied() + .flatten() + .expect("explicit semantic lookback checked"); + subtree.instant.full_history = false; + subtree.instant.cumulative_readout = true; + let requirement = readiness_requirement(&subtree); + let index = self.sketch_index.as_ref().ok_or_else(|| { + EngineError::capability_miss( + "installed_logical_dag", + "summary store unavailable", + ) + })?; + let (result, t0) = super::live_serve::serve_instant_from_query_plan( + index, + &subtree, + evaluation_ms, + ) + .map_err(|e| { + EngineError::capability_miss( + "installed_logical_dag", + format!("bound readout failed: {e:?}"), + ) + })?; + let active = self.active_physical_plan.as_ref().ok_or_else(|| { + EngineError::capability_miss( + "installed_logical_dag", + "readiness registry unavailable", + ) + })?; + let plan_id = physical.backend_plan.plan_id; + let version = physical.backend_plan.plan_version; + if !complete_window_coverage( + result.coverage, + t0, + evaluation_ms, + requirement.max_window_ms, + ) { + active.mark_materializing( + plan_id, + version, + &requirement.materializations, + result.coverage, + ); + return Err(EngineError::capability_miss( + "installed_logical_dag", + format!("bound readout incomplete at {evaluation_ms}"), + )); + } + let coverage = result.coverage.expect("coverage checked"); + if !active.mark_ready(plan_id, version, &requirement.materializations, coverage) + || !active.mark_serving( + plan_id, + version, + &requirement.materializations, + coverage, + ) + { + return Err(EngineError::capability_miss( + "installed_logical_dag", + "physical generation changed during bound readout", + )); + } + Ok(asap_tier_result_to_query_result( + result, + evaluation_ms, + false, + )) + }, + ) + } + + fn execute_logical_range( + &self, + physical: &crate::storage_engines::types::ActivePhysicalPlan, + entry: &control_plane::query_plan::QueryPlanEntry, + start: u64, + end: u64, + step: u64, + ) -> Result + { + use crate::query_engines::{ + query_result::{QueryResult, RangeVectorElement}, + EngineError, + }; + if step == 0 || start > end || (end - start) / step >= 11_000 { + return Err(EngineError::capability_miss( + "installed_logical_dag", + "invalid range or more than 11000 evaluations", + )); + } + let store = self.raw_store.as_ref().ok_or_else(|| { + EngineError::capability_miss("installed_logical_dag", "raw store unavailable") + })?; + let samples = store.snapshot( + physical.backend_plan.plan_id, + physical.backend_plan.plan_version, + )?; + let mut series = + std::collections::BTreeMap::, RangeVectorElement>::new(); + let mut total = super::logical_dag::ExecutionStats::default(); + let mut at = start; + loop { + let (result, stats) = self.execute_logical_entry(physical, entry, &samples, at)?; + total.raw_scan_evaluations += stats.raw_scan_evaluations; + total.summary_readout_evaluations += stats.summary_readout_evaluations; + total.memo_hits += stats.memo_hits; + let QueryResult::Vector(result) = result else { + return Err(EngineError::capability_miss( + "installed_logical_dag", + "range step requires vector", + )); + }; + for point in result.values { + let keys = point.label_keys_override.ok_or_else(|| { + EngineError::capability_miss( + "installed_logical_dag", + "missing range label identity", + ) + })?; + let identity = keys + .iter() + .cloned() + .zip(point.labels.labels.iter().cloned()) + .collect(); + series + .entry(identity) + .or_insert_with(|| { + RangeVectorElement::new(point.labels).with_label_keys_override(keys) + }) + .add_sample(at, point.value); + } + let Some(next) = at.checked_add(step) else { + break; + }; + if next > end { + break; + } + at = next; + } + let mut result = QueryResult::matrix(series.into_values().collect()); + annotate_logical_execution(&mut result, &total); + Ok(result) + } + pub fn with_active_physical_plan( mut self, handle: crate::storage_engines::types::HotReloadActivePhysicalPlan, @@ -351,6 +541,18 @@ impl ASAPQueryEngine { step_ms: u64, ) -> Result { + if let Some(physical) = self.physical_plan_snapshot() { + if let Ok(entry) = physical.query_plan.lookup(query) { + if entry.nodes.values().any(|node| { + matches!( + node, + control_plane::query_plan::QueryPlanNode::Logical { .. } + ) + }) { + return self.execute_logical_range(&physical, entry, start_ms, end_ms, step_ms); + } + } + } let Some(idx) = self.sketch_index.as_ref() else { return Err(crate::query_engines::EngineError::capability_miss( crate::storage_engines::types::StorageBackend::SketchStore.data_source_id(), @@ -569,6 +771,31 @@ fn stitch_warm_and_archive( QueryResult::matrix(elements) } +fn annotate_logical_execution( + result: &mut crate::query_engines::query_result::QueryResult, + stats: &super::logical_dag::ExecutionStats, +) { + use crate::query_engines::query_result::QueryResult; + let warnings = match result { + QueryResult::Vector(v) => &mut v.warnings, + QueryResult::Matrix(m) => &mut m.warnings, + }; + if stats.raw_scan_evaluations > 0 || stats.summary_readout_evaluations == 0 { + warnings.push( + if stats.summary_readout_evaluations > 0 { + "asap_execution:hybrid" + } else { + "asap_execution:raw_dag" + } + .into(), + ); + } + warnings.push(format!( + "asap_logical_stats:raw={},summary={},memo_hits={}", + stats.raw_scan_evaluations, stats.summary_readout_evaluations, stats.memo_hits + )); +} + fn asap_tier_result_to_query_result( result: crate::storage_engines::sketch_db::query::ASAPTierResult, now_ms: u64, @@ -660,6 +887,31 @@ impl crate::query_engines::routing::query_engine_routing::QueryEngine for ASAPQu now_ms: u64, ) -> Result { + if let Some(physical) = self.physical_plan_snapshot() { + if let Ok(entry) = physical.query_plan.lookup(query) { + if entry.nodes.values().any(|node| { + matches!( + node, + control_plane::query_plan::QueryPlanNode::Logical { .. } + ) + }) { + let store = self.raw_store.as_ref().ok_or_else(|| { + crate::query_engines::EngineError::capability_miss( + "installed_logical_dag", + "raw store unavailable", + ) + })?; + let samples = store.snapshot( + physical.backend_plan.plan_id, + physical.backend_plan.plan_version, + )?; + let (mut result, stats) = + self.execute_logical_entry(&physical, entry, &samples, now_ms)?; + annotate_logical_execution(&mut result, &stats); + return Ok(result); + } + } + } // One authoritative warm path: ASAPPlanner post-ASAP DAG → // BackendPlan/materialization resolver → SID lookup → DAG executor. // A typed resolver/executor error becomes CapabilityMiss, which lets diff --git a/data_plane/src/query_engines/asap_query_engine/live_serve.rs b/data_plane/src/query_engines/asap_query_engine/live_serve.rs index 9efc044e..fef202bb 100644 --- a/data_plane/src/query_engines/asap_query_engine/live_serve.rs +++ b/data_plane/src/query_engines/asap_query_engine/live_serve.rs @@ -490,6 +490,7 @@ mod tests { sid_grouping: vec![], output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, window_ms: 1_000, + readout_lookback_ms: Some(1_000), }, }, ), diff --git a/data_plane/src/query_engines/asap_query_engine/logical_dag.rs b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs new file mode 100644 index 00000000..476956dd --- /dev/null +++ b/data_plane/src/query_engines/asap_query_engine/logical_dag.rs @@ -0,0 +1,871 @@ +//! Executes the installed typed logical DAG. No serving-time PromQL parsing. +use crate::drivers::ingest::prometheus_remote_write::CanonicalSample; +use crate::query_engines::{ + query_result::{InstantVectorElement, QueryResult}, + EngineError, +}; +use crate::storage_engines::types::KeyByLabelValues; +use control_plane::query_plan::logical::{ + Aggregation, BinaryOperation, Grouping, LabelMatch, LabelMatcher, LogicalOperator, + TemporalOperation, +}; +use control_plane::query_plan::{QueryNodeId, QueryPlanEntry, QueryPlanNode}; +use std::collections::{BTreeMap, BTreeSet}; + +type Labels = BTreeMap; +type Vector = Vec<(Labels, f64)>; +type Matrix = Vec<(Labels, Vec<(i64, f64)>)>; +#[derive(Clone)] +enum Value { + Scalar(f64), + Vector(Vector), + Matrix(Matrix, i64, i64), +} +struct Point { + timestamp_ms: i64, + value: Option, +} +struct Series { + labels: Labels, + points: Vec, +} +/// One immutable indexed raw-data snapshot; callers invalidate it after ingestion. +pub struct PreparedSamples { + series: Vec, +} +#[derive(Debug, Default, Clone)] +pub struct ExecutionStats { + pub raw_scan_evaluations: usize, + pub summary_readout_evaluations: usize, + pub memo_hits: usize, +} +impl PreparedSamples { + pub fn new(samples: &[CanonicalSample]) -> Result { + let mut grouped: BTreeMap> = BTreeMap::new(); + for sample in samples { + let mut labels: Labels = sample.labels.clone().into_iter().collect(); + labels.insert("__name__".into(), sample.metric.clone()); + grouped.entry(labels).or_default().push(Point { + timestamp_ms: sample.timestamp_ms, + value: sample.value, + }); + } + Self::from_grouped(grouped) + } + + /// Build directly from grouped storage without duplicating labels per sample. + /// Label maps include the `__name__` metric label. + pub fn from_series( + input: impl IntoIterator, Vec<(i64, Option)>)>, + ) -> Result { + let mut grouped: BTreeMap> = BTreeMap::new(); + for (labels, points) in input { + grouped + .entry(labels) + .or_default() + .extend(points.into_iter().map(|(timestamp_ms, value)| Point { + timestamp_ms, + value, + })); + } + Self::from_grouped(grouped) + } + + fn from_grouped(grouped: BTreeMap>) -> Result { + let mut series = Vec::with_capacity(grouped.len()); + for (labels, mut points) in grouped { + points.sort_by_key(|point| point.timestamp_ms); + for pair in points.windows(2) { + if pair[0].timestamp_ms == pair[1].timestamp_ms + && pair[0].value.map(f64::to_bits) != pair[1].value.map(f64::to_bits) + { + return Err(miss( + "conflicting raw samples for one label set and timestamp", + )); + } + } + points.dedup_by_key(|point| point.timestamp_ms); + series.push(Series { labels, points }); + } + Ok(Self { series }) + } + pub fn estimated_bytes(&self) -> usize { + self.series + .iter() + .map(|series| { + std::mem::size_of::() + + series.points.capacity() * std::mem::size_of::() + + series + .labels + .iter() + .map(|(key, value)| key.capacity() + value.capacity()) + .sum::() + }) + .sum() + } +} +fn miss(detail: impl Into) -> EngineError { + EngineError::capability_miss("installed_logical_dag", detail) +} +fn no_name(mut labels: Labels) -> Labels { + labels.remove("__name__"); + labels +} +fn vector(value: Value) -> Result { + let Value::Vector(values) = value else { + return Err(miss("instant vector required")); + }; + let mut seen = BTreeSet::new(); + if values.iter().any(|(labels, _)| !seen.insert(labels)) { + return Err(miss("duplicate vector label sets")); + } + Ok(values) +} +fn from_result(result: QueryResult) -> Result { + let QueryResult::Vector(value) = result else { + return Err(miss("bound readout must return instant vector")); + }; + if !value.warnings.is_empty() { + return Err(miss("partial bound readout cannot feed logical operator")); + } + let values = value + .values + .into_iter() + .map(|point| { + let keys = point + .label_keys_override + .ok_or_else(|| miss("bound readout must carry explicit label keys"))?; + if keys.len() != point.labels.labels.len() + || keys.iter().collect::>().len() != keys.len() + { + return Err(miss("bound readout label arity mismatch")); + } + Ok(( + keys.into_iter().zip(point.labels.labels).collect(), + point.value, + )) + }) + .collect::>()?; + Ok(Value::Vector(vector(Value::Vector(values))?)) +} + +pub fn execute( + entry: &QueryPlanEntry, + samples: &[CanonicalSample], + at: u64, +) -> Result { + execute_with_summary(entry, samples, at, |_, _| { + Err(miss("summary callback required")) + }) +} + +pub fn execute_with_summary( + entry: &QueryPlanEntry, + samples: &[CanonicalSample], + at: u64, + callback: F, +) -> Result +where + F: FnMut(QueryNodeId, u64) -> Result, +{ + execute_prepared(entry, &PreparedSamples::new(samples)?, at, callback) +} + +pub fn execute_prepared( + entry: &QueryPlanEntry, + samples: &PreparedSamples, + at: u64, + callback: F, +) -> Result +where + F: FnMut(QueryNodeId, u64) -> Result, +{ + execute_prepared_with_stats(entry, samples, at, callback).map(|(result, _)| result) +} + +pub fn execute_prepared_with_stats( + entry: &QueryPlanEntry, + samples: &PreparedSamples, + at: u64, + callback: F, +) -> Result<(QueryResult, ExecutionStats), EngineError> +where + F: FnMut(QueryNodeId, u64) -> Result, +{ + let mut evaluator = Evaluator { + entry, + series: &samples.series, + callback, + stats: ExecutionStats::default(), + memo: BTreeMap::new(), + active: BTreeSet::new(), + regexes: BTreeMap::new(), + }; + let at_signed = i64::try_from(at).map_err(|_| miss("evaluation timestamp overflow"))?; + let result = vector(evaluator.eval(entry.root, at_signed)?)?; + Ok(( + QueryResult::vector( + result + .into_iter() + .map(|(labels, value)| { + InstantVectorElement::new( + KeyByLabelValues::new_with_labels(labels.values().cloned().collect()), + value, + ) + .with_label_keys_override(labels.into_keys().collect()) + }) + .collect(), + at, + ), + evaluator.stats, + )) +} + +struct Evaluator<'a, F> { + entry: &'a QueryPlanEntry, + series: &'a [Series], + stats: ExecutionStats, + callback: F, + memo: BTreeMap<(QueryNodeId, i64), Value>, + active: BTreeSet<(QueryNodeId, i64)>, + regexes: BTreeMap, +} +impl Result> Evaluator<'_, F> { + fn eval(&mut self, id: QueryNodeId, at: i64) -> Result { + if let Some(value) = self.memo.get(&(id, at)) { + self.stats.memo_hits += 1; + return Ok(value.clone()); + } + if self.active.len() >= 256 || !self.active.insert((id, at)) { + return Err(miss("cyclic or excessively deep installed DAG")); + } + if self.memo.len() >= 200_000 { + return Err(miss("installed DAG evaluation budget exceeded")); + } + let node = self + .entry + .nodes + .get(&id) + .ok_or_else(|| miss("missing installed node"))? + .clone(); + let value = match node { + QueryPlanNode::Scalar { value } => Value::Scalar(value), + QueryPlanNode::Logical { operator, inputs } => self.logical(operator, &inputs, at)?, + _ => { + self.stats.summary_readout_evaluations += 1; + from_result((self.callback)( + id, + u64::try_from(at).map_err(|_| miss("summary timestamp predates epoch"))?, + )?)? + } + }; + self.active.remove(&(id, at)); + self.memo.insert((id, at), value.clone()); + Ok(value) + } + fn logical( + &mut self, + operator: LogicalOperator, + inputs: &[QueryNodeId], + at: i64, + ) -> Result { + let input = |index: usize| { + inputs + .get(index) + .copied() + .ok_or_else(|| miss("missing logical input")) + }; + match operator { + LogicalOperator::Scan { + metric, + matchers, + range_ms, + offset_ms, + } => self.scan(metric.as_deref(), &matchers, range_ms, offset_ms, at), + LogicalOperator::UnaryNegate => match self.eval(input(0)?, at)? { + Value::Scalar(value) => Ok(Value::Scalar(-value)), + Value::Vector(values) => Ok(Value::Vector( + values + .into_iter() + .map(|(labels, value)| (labels, -value)) + .collect(), + )), + _ => Err(miss("cannot negate range vector")), + }, + LogicalOperator::Aggregate { + operation, + grouping, + } => { + let values = vector(self.eval(input(0)?, at)?)?; + Ok(Value::Vector(aggregate(operation, &grouping, values))) + } + LogicalOperator::Binary { + operation, + return_bool, + } => { + let left = self.eval(input(0)?, at)?; + let right = self.eval(input(1)?, at)?; + binary(operation, return_bool, left, right) + } + LogicalOperator::Temporal { operation } => { + let Value::Matrix(values, start, end) = self.eval(input(0)?, at)? else { + return Err(miss("temporal operator requires range vector")); + }; + Ok(Value::Vector( + values + .into_iter() + .filter_map(|(labels, points)| { + if points.is_empty() { + return None; + } + let value = match operation { + TemporalOperation::Rate => rate(&points, start, end), + TemporalOperation::Increase => rate(&points, start, end) + .map(|r| r * (end - start) as f64 / 1000.), + TemporalOperation::Sum => Some(points.iter().map(|p| p.1).sum()), + TemporalOperation::Avg => Some( + points.iter().map(|p| p.1).sum::() / points.len() as f64, + ), + TemporalOperation::Count => Some(points.len() as f64), + TemporalOperation::Max => { + Some(points.iter().fold(f64::NAN, |a, p| { + if a.is_nan() || p.1 > a { + p.1 + } else { + a + } + })) + } + TemporalOperation::Min => { + Some(points.iter().fold(f64::NAN, |a, p| { + if a.is_nan() || p.1 < a { + p.1 + } else { + a + } + })) + } + }; + value.map(|v| (no_name(labels), v)) + }) + .collect(), + )) + } + LogicalOperator::Sort { descending } => { + let mut values = vector(self.eval(input(0)?, at)?)?; + values.sort_by(|a, b| { + if a.1.is_nan() && b.1.is_nan() { + std::cmp::Ordering::Equal + } else if a.1.is_nan() { + std::cmp::Ordering::Greater + } else if b.1.is_nan() { + std::cmp::Ordering::Less + } else if descending { + b.1.total_cmp(&a.1) + } else { + a.1.total_cmp(&b.1) + } + }); + Ok(Value::Vector(values)) + } + LogicalOperator::HistogramQuantile => { + let Value::Scalar(quantile) = self.eval(input(0)?, at)? else { + return Err(miss("quantile requires scalar")); + }; + let mut groups: BTreeMap> = BTreeMap::new(); + for (mut labels, value) in vector(self.eval(input(1)?, at)?)? { + if let Some(le) = labels.remove("le").and_then(|s| s.parse::().ok()) { + groups.entry(no_name(labels)).or_default().push((le, value)); + } + } + Ok(Value::Vector( + groups + .into_iter() + .map(|(labels, buckets)| (labels, bucket_quantile(quantile, buckets))) + .collect(), + )) + } + LogicalOperator::Subquery { + range_ms, + step_ms, + offset_ms, + } => { + let end = at + .checked_sub(offset_ms) + .ok_or_else(|| miss("offset overflow"))?; + let range = i64::try_from(range_ms).map_err(|_| miss("range overflow"))?; + let step = i64::try_from(step_ms).map_err(|_| miss("step overflow"))?; + if step <= 0 || range / step > 100_000 { + return Err(miss("invalid or excessive subquery steps")); + } + let start = end + .checked_sub(range) + .ok_or_else(|| miss("range overflow"))?; + let mut t = start + .div_euclid(step) + .checked_add(1) + .and_then(|n| n.checked_mul(step)) + .ok_or_else(|| miss("subquery grid overflow"))?; + let mut values: BTreeMap> = BTreeMap::new(); + while t <= end { + for (labels, value) in vector(self.eval(input(0)?, t)?)? { + values.entry(labels).or_default().push((t, value)); + } + t = t + .checked_add(step) + .ok_or_else(|| miss("subquery time overflow"))?; + } + Ok(Value::Matrix(values.into_iter().collect(), start, end)) + } + } + } + fn scan( + &mut self, + metric: Option<&str>, + matchers: &[LabelMatcher], + range_ms: Option, + offset_ms: i64, + at: i64, + ) -> Result { + self.stats.raw_scan_evaluations += 1; + for matcher in matchers { + if matches!(matcher.operation, LabelMatch::Regex | LabelMatch::NotRegex) + && !self.regexes.contains_key(&matcher.value) + { + let pattern = regex::Regex::new(&format!("(?s)^(?:{})$", matcher.value)) + .map_err(|e| miss(format!("unsupported regex: {e}")))?; + self.regexes.insert(matcher.value.clone(), pattern); + } + } + let end = at + .checked_sub(offset_ms) + .ok_or_else(|| miss("offset overflow"))?; + let range = + i64::try_from(range_ms.unwrap_or(300_000)).map_err(|_| miss("range overflow"))?; + let start = end + .checked_sub(range) + .ok_or_else(|| miss("range overflow"))?; + let mut instant = Vec::new(); + let mut matrix = Vec::new(); + for series in self.series { + if metric.is_some_and(|m| series.labels.get("__name__").map(String::as_str) != Some(m)) + || !matchers.iter().all(|matcher| { + let value = series + .labels + .get(&matcher.name) + .map(String::as_str) + .unwrap_or(""); + match matcher.operation { + LabelMatch::Equal => value == matcher.value, + LabelMatch::NotEqual => value != matcher.value, + LabelMatch::Regex => self.regexes[&matcher.value].is_match(value), + LabelMatch::NotRegex => !self.regexes[&matcher.value].is_match(value), + } + }) + { + continue; + } + let hi = series + .points + .partition_point(|point| point.timestamp_ms <= end); + if range_ms.is_some() { + let lo = series + .points + .partition_point(|point| point.timestamp_ms <= start); + let points: Vec<_> = series.points[lo..hi] + .iter() + .filter_map(|point| { + point + .value + .filter(|v| v.to_bits() != 0x7ff0000000000002) + .map(|v| (point.timestamp_ms, v)) + }) + .collect(); + if !points.is_empty() { + matrix.push((series.labels.clone(), points)); + } + } else if hi > 0 { + let point = &series.points[hi - 1]; + if point.timestamp_ms >= start { + if let Some(value) = point.value.filter(|v| v.to_bits() != 0x7ff0000000000002) { + instant.push((series.labels.clone(), value)); + } + } + } + } + Ok(if range_ms.is_some() { + Value::Matrix(matrix, start, end) + } else { + Value::Vector(instant) + }) + } +} + +fn aggregate(operation: Aggregation, grouping: &Grouping, values: Vector) -> Vector { + let mut groups: BTreeMap> = BTreeMap::new(); + for (labels, value) in values { + let key = labels + .into_iter() + .filter(|(key, _)| { + if grouping.without { + key != "__name__" && !grouping.labels.contains(key) + } else { + grouping.labels.contains(key) + } + }) + .collect(); + groups.entry(key).or_default().push(value); + } + groups + .into_iter() + .map(|(labels, values)| { + let value = match operation { + Aggregation::Sum => values.iter().sum(), + Aggregation::Avg => values.iter().sum::() / values.len() as f64, + Aggregation::Count => values.len() as f64, + Aggregation::Max => { + values + .into_iter() + .fold(f64::NAN, |a, b| if a.is_nan() || b > a { b } else { a }) + } + Aggregation::Min => { + values + .into_iter() + .fold(f64::NAN, |a, b| if a.is_nan() || b < a { b } else { a }) + } + }; + (labels, value) + }) + .collect() +} + +fn binary( + operation: BinaryOperation, + boolean: bool, + left: Value, + right: Value, +) -> Result { + let arithmetic = matches!( + operation, + BinaryOperation::Add + | BinaryOperation::Sub + | BinaryOperation::Mul + | BinaryOperation::Div + | BinaryOperation::Mod + | BinaryOperation::Pow + ); + let combine = |a: f64, b: f64| -> Option { + Some(match operation { + BinaryOperation::Add => a + b, + BinaryOperation::Sub => a - b, + BinaryOperation::Mul => a * b, + BinaryOperation::Div => a / b, + BinaryOperation::Mod => a % b, + BinaryOperation::Pow => a.powf(b), + _ => { + let pass = match operation { + BinaryOperation::Equal => a == b, + BinaryOperation::NotEqual => a != b, + BinaryOperation::Less => a < b, + BinaryOperation::LessEqual => a <= b, + BinaryOperation::Greater => a > b, + BinaryOperation::GreaterEqual => a >= b, + _ => unreachable!(), + }; + if boolean { + if pass { + 1. + } else { + 0. + } + } else if pass { + a + } else { + return None; + } + } + }) + }; + let values = match (left, right) { + (Value::Scalar(a), Value::Scalar(b)) => { + if !arithmetic && !boolean { + return Err(miss("scalar comparison requires bool")); + } + return Ok(Value::Scalar(combine(a, b).unwrap_or(0.))); + } + (Value::Vector(values), Value::Scalar(scalar)) => vector(Value::Vector(values))? + .into_iter() + .filter_map(|(labels, value)| { + combine(value, scalar).map(|v| { + ( + if arithmetic || boolean { + no_name(labels) + } else { + labels + }, + v, + ) + }) + }) + .collect(), + (Value::Scalar(scalar), Value::Vector(values)) => vector(Value::Vector(values))? + .into_iter() + .filter_map(|(labels, value)| { + combine(scalar, value).map(|v| { + ( + if arithmetic || boolean { + no_name(labels) + } else { + labels + }, + if arithmetic || boolean { v } else { value }, + ) + }) + }) + .collect(), + (Value::Vector(left), Value::Vector(right)) => { + let mut rhs = BTreeMap::new(); + for (labels, value) in right { + if rhs.insert(no_name(labels), value).is_some() { + return Err(miss("duplicate vector matching labels")); + } + } + let mut seen = BTreeSet::new(); + let mut out = Vec::new(); + for (labels, value) in left { + let key = no_name(labels.clone()); + if !seen.insert(key.clone()) { + return Err(miss("duplicate vector matching labels")); + } + if let Some(right) = rhs.get(&key) { + if let Some(v) = combine(value, *right) { + out.push((if arithmetic || boolean { key } else { labels }, v)); + } + } + } + out + } + _ => return Err(miss("binary matrix unsupported")), + }; + Ok(Value::Vector(vector(Value::Vector(values))?)) +} + +fn rate(points: &[(i64, f64)], start: i64, end: i64) -> Option { + if points.len() < 2 { + return None; + } + let (first_t, first) = points[0]; + let (last_t, last) = *points.last()?; + let span = (last_t - first_t) as f64 / 1000.; + if span <= 0. { + return None; + } + let mut delta = last - first; + for pair in points.windows(2) { + if pair[1].1 < pair[0].1 { + delta += pair[0].1; + } + } + let average = span / (points.len() - 1) as f64; + let mut to_start = (first_t - start) as f64 / 1000.; + let mut to_end = (end - last_t) as f64 / 1000.; + if to_start >= average * 1.1 { + to_start = average / 2.; + } + // Apply the zero bound after the sparse-window half-interval cap. + if delta > 0. && first >= 0. { + to_start = to_start.min(span * first / delta); + } + if to_end >= average * 1.1 { + to_end = average / 2.; + } + Some(delta * (span + to_start + to_end) / span / ((end - start) as f64 / 1000.)) +} + +fn bucket_quantile(q: f64, mut b: Vec<(f64, f64)>) -> f64 { + if q.is_nan() { + return f64::NAN; + } + if q < 0. { + return f64::NEG_INFINITY; + } + if q > 1. { + return f64::INFINITY; + } + b.retain(|p| !p.0.is_nan()); + b.sort_by(|a, b| a.0.total_cmp(&b.0)); + let mut buckets: Vec<(f64, f64)> = Vec::new(); + for p in b { + if let Some(last) = buckets.last_mut() { + if last.0 == p.0 { + last.1 += p.1; + continue; + } + } + buckets.push(p); + } + if buckets.len() < 2 || buckets.last().unwrap().0 != f64::INFINITY { + return f64::NAN; + } + let mut prev = buckets[0].1; + for p in buckets.iter_mut().skip(1) { + if p.1 < prev || (p.1 - prev).abs() <= 1e-12 * (p.1.abs() + prev.abs()) { + p.1 = prev; + } + prev = p.1; + } + let count = buckets.last().unwrap().1; + if count == 0. { + return f64::NAN; + } + let rank = q * count; + let idx = buckets[..buckets.len() - 1].partition_point(|p| p.1 < rank); + if idx == buckets.len() - 1 { + return buckets[idx - 1].0; + } + if idx == 0 && buckets[0].0 <= 0. { + return buckets[0].0; + } + let (start, base) = if idx == 0 { (0., 0.) } else { buckets[idx - 1] }; + let (end, upper) = buckets[idx]; + start + (end - start) * (rank - base) / (upper - base) +} + +#[cfg(test)] +mod tests { + use super::*; + use control_plane::query_plan::{FallbackPolicy, InstantExecution}; + fn entry(query: &str) -> QueryPlanEntry { + QueryPlanEntry::compile_logical( + "q".into(), + query.into(), + InstantExecution { + lookback_ms: 300_000, + full_history: false, + cumulative_readout: false, + }, + FallbackPolicy::Reject, + ) + .unwrap() + } + fn sample(metric: &str, job: &str, timestamp_ms: i64, value: Option) -> CanonicalSample { + CanonicalSample { + metric: metric.into(), + labels: [("job".into(), job.into())].into(), + series_key: format!("{metric}:{job}"), + timestamp_ms, + value, + } + } + fn values(query: &str, samples: &[CanonicalSample], at: u64) -> Vec { + let QueryResult::Vector(result) = execute(&entry(query), samples, at).unwrap() else { + panic!() + }; + result.values.into_iter().map(|p| p.value).collect() + } + // Regex alternation remains fully anchored and missing labels compare as empty. + #[test] + fn anchored_matchers_and_missing_labels() { + let samples = [ + sample("up", "xorder", 1000, Some(9.)), + sample("up", "user", 1000, Some(2.)), + ]; + assert_eq!( + values(r#"sum(up{job=~"user|order",absent=""})"#, &samples, 1000), + vec![2.] + ); + } + // Stale markers stop instant lookup; range windows are left-open and ignore stale points. + #[test] + fn stale_and_range_boundaries() { + let samples = [ + sample("up", "a", 0, Some(99.)), + sample("up", "a", 1000, Some(2.)), + sample("up", "a", 2000, None), + ]; + assert!(values("up", &samples, 2000).is_empty()); + assert_eq!(values("sum_over_time(up[2s])", &samples, 2000), vec![2.]); + } + // A shared child must be evaluated separately at each subquery grid time. + #[test] + fn shared_subquery_uses_time_in_memo_key() { + let samples = [ + sample("up", "a", 1000, Some(2.)), + sample("up", "a", 2000, Some(4.)), + ]; + assert_eq!( + values("avg_over_time((up + up)[2s:1s])", &samples, 2000), + vec![6.] + ); + } + // Scalar-left filtering returns the vector's original value and metric name. + #[test] + fn scalar_left_comparison_preserves_vector_value() { + let samples = [sample("up", "a", 1000, Some(4.))]; + assert_eq!(values("2 < up", &samples, 1000), vec![4.]); + assert_eq!(values("2 < bool up", &samples, 1000), vec![1.]); + } + // Materialized readouts can feed residual operators without parsing their query text. + #[test] + fn summary_callback_is_memoized_and_composed() { + let mut graph = entry("up + up"); + let leaf = graph + .nodes + .iter() + .find_map(|(id, node)| { + matches!( + node, + QueryPlanNode::Logical { + operator: LogicalOperator::Scan { .. }, + .. + } + ) + .then_some(*id) + }) + .unwrap(); + graph + .nodes + .insert(leaf, QueryPlanNode::SummaryMerge { inputs: vec![] }); + let mut calls = 0; + let result = execute_with_summary(&graph, &[], 1000, |id, at| { + assert_eq!(id, leaf); + assert_eq!(at, 1000); + calls += 1; + Ok(QueryResult::vector( + vec![InstantVectorElement::new( + KeyByLabelValues::new_with_labels(vec!["a".into()]), + 3., + ) + .with_label_keys_override(vec!["job".into()])], + at, + )) + }) + .unwrap(); + let QueryResult::Vector(result) = result else { + panic!() + }; + assert_eq!(result.values[0].value, 6.); + assert_eq!(calls, 1); + } + // Reusing the immutable index keeps raw provenance and shares the same-time leaf. + #[test] + fn prepared_snapshot_preserves_route_counts() { + let data = PreparedSamples::new(&[sample("up", "a", 1000, Some(2.))]).unwrap(); + let (_, stats) = execute_prepared_with_stats(&entry("up + up"), &data, 1000, |_, _| { + Err(miss("unexpected summary")) + }) + .unwrap(); + assert_eq!(stats.raw_scan_evaluations, 1); + assert_eq!(stats.summary_readout_evaluations, 0); + assert_eq!(stats.memo_hits, 1); + assert!(data.estimated_bytes() > 0); + } + // Conflicting values cannot be hidden by different transport series-key strings. + #[test] + fn duplicate_identity_rejected() { + let first = sample("up", "a", 1000, Some(1.)); + let mut second = sample("up", "a", 1000, Some(2.)); + second.series_key = "different".into(); + assert!(execute(&entry("up"), &[first, second], 1000).is_err()); + } +} diff --git a/data_plane/src/query_engines/asap_query_engine/mod.rs b/data_plane/src/query_engines/asap_query_engine/mod.rs index 6736eaca..963d74bc 100644 --- a/data_plane/src/query_engines/asap_query_engine/mod.rs +++ b/data_plane/src/query_engines/asap_query_engine/mod.rs @@ -11,6 +11,7 @@ pub mod engine; pub mod live_serve; +pub mod logical_dag; pub mod physical_dag; pub mod post_asap_planner; pub mod post_asap_readout; diff --git a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs index 1a0cc1dd..b06573b1 100644 --- a/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs +++ b/data_plane/src/query_engines/asap_query_engine/post_asap_readout.rs @@ -251,6 +251,9 @@ impl QueryNodeRuntime for PhysicalQueryRuntime<'_> { .collect::, _>>() .map(PhysicalQueryOutput::State) } + QueryPlanNode::Logical { .. } => Err(PhysicalNodeError::Fallback( + "logical node requires installed logical runtime".into(), + )), QueryPlanNode::ExactFallback { reason } => { Err(PhysicalNodeError::Fallback(reason.clone())) } @@ -793,6 +796,7 @@ mod tests { sid_grouping: vec!["service".into()], output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, + readout_lookback_ms: Some(60_000), }) }, ) @@ -961,6 +965,7 @@ mod tests { sid_grouping: vec![], output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, window_ms: 10_000, + readout_lookback_ms: Some(60_000), }, }, ), @@ -1054,6 +1059,7 @@ mod tests { sid_grouping: vec![], output_grouping: control_plane::query_plan::PhysicalGrouping::PerEntity, window_ms: 60_000, + readout_lookback_ms: Some(60_000), }, }, ), @@ -1069,5 +1075,81 @@ mod tests { .expect("execute exact rate DAG"); let value = outcome.series[0].1[0].1; assert!((value - 0.575).abs() < 1e-12, "reset-aware rate={value}"); + + // The installed residual combines an actual bound store readout with raw input. + use crate::query_engines::asap_query_engine::logical_dag; + use crate::query_engines::query_result::{InstantVectorElement, QueryResult}; + use control_plane::query_plan::{ + logical::{BinaryOperation, LogicalOperator}, + QueryNodeId, + }; + let mut hybrid = entry.clone(); + hybrid.root = QueryNodeId(4); + hybrid.nodes.insert( + QueryNodeId(3), + QueryPlanNode::Logical { + operator: LogicalOperator::Scan { + metric: Some("up".into()), + matchers: vec![], + range_ms: None, + offset_ms: 0, + }, + inputs: vec![], + }, + ); + hybrid.nodes.insert( + QueryNodeId(4), + QueryPlanNode::Logical { + operator: LogicalOperator::Binary { + operation: BinaryOperation::Add, + return_bool: false, + }, + inputs: vec![entry.root, QueryNodeId(3)], + }, + ); + let samples = logical_dag::PreparedSamples::new(&[ + crate::drivers::ingest::prometheus_remote_write::CanonicalSample { + metric: "up".into(), + labels: Default::default(), + series_key: "up".into(), + timestamp_ms: 60_000, + value: Some(2.), + }, + ]) + .unwrap(); + let (result, stats) = + logical_dag::execute_prepared_with_stats(&hybrid, &samples, 60_000, |root, at| { + assert_eq!(root, entry.root); + let result = execute_query_plan_readout(&idx, &entry, at - 60_000, at, true) + .map_err(|error| { + crate::query_engines::EngineError::capability_miss( + "test", + format!("{error:?}"), + ) + })?; + Ok(QueryResult::vector( + result + .series + .into_iter() + .map(|(labels, values)| { + InstantVectorElement::new( + crate::storage_engines::types::KeyByLabelValues::new_with_labels( + labels.values().cloned().collect(), + ), + values.last().unwrap().1, + ) + .with_label_keys_override(labels.into_keys().collect()) + }) + .collect(), + at, + )) + }) + .unwrap(); + let QueryResult::Vector(result) = result else { + panic!("instant vector required") + }; + assert!((result.values[0].value - 2.575).abs() < 1e-12); + assert_eq!(stats.raw_scan_evaluations, 1); + assert_eq!(stats.summary_readout_evaluations, 1); } } diff --git a/data_plane/src/query_engines/mod.rs b/data_plane/src/query_engines/mod.rs index 722fc96d..d27dedbb 100644 --- a/data_plane/src/query_engines/mod.rs +++ b/data_plane/src/query_engines/mod.rs @@ -20,6 +20,7 @@ pub mod asap_query_engine; pub mod no_data_archive; pub mod query_result; +pub mod raw_store; pub mod routing; pub mod thanos_query_engine; diff --git a/data_plane/src/query_engines/raw_store.rs b/data_plane/src/query_engines/raw_store.rs new file mode 100644 index 00000000..99890bf5 --- /dev/null +++ b/data_plane/src/query_engines/raw_store.rs @@ -0,0 +1,159 @@ +//! Retained input for installed typed scan nodes, shared across query roots. +use crate::drivers::ingest::prometheus_remote_write::CanonicalSample; +use crate::query_engines::asap_query_engine::logical_dag::PreparedSamples; +use crate::query_engines::EngineError; +use std::collections::{BTreeMap, BTreeSet}; +use std::sync::{Arc, Mutex}; + +type Labels = BTreeMap; +#[derive(Default)] +struct Generation { + plan: Option<(u64, u64)>, + series: BTreeMap)>>, + prepared: Option>, +} + +#[derive(Default)] +pub struct RawSampleStore { + generation: Mutex, +} + +impl RawSampleStore { + /// Called only after the receiver admits the complete batch to its workers. + /// Receiver deduplication serializes this operation with all other admissions. + pub fn append_admitted( + &self, + plan_id: u64, + plan_version: u64, + samples: &[CanonicalSample], + metrics: &BTreeSet, + all_metrics: bool, + ) { + let mut state = self.generation.lock().unwrap_or_else(|e| e.into_inner()); + if state.plan != Some((plan_id, plan_version)) { + *state = Generation { + plan: Some((plan_id, plan_version)), + ..Default::default() + }; + } + if metrics.is_empty() && !all_metrics { + state.series.clear(); + state.prepared = None; + return; + } + let mut changed = false; + for sample in samples { + if !all_metrics && !metrics.contains(&sample.metric) { + continue; + } + let mut labels: Labels = sample + .labels + .iter() + .map(|(k, v)| (k.clone(), v.clone())) + .collect(); + labels.insert("__name__".into(), sample.metric.clone()); + state + .series + .entry(labels) + .or_default() + .push((sample.timestamp_ms, sample.value)); + changed = true; + } + if changed { + state.prepared = None; + } + } + + /// One immutable indexed snapshot is built per admitted input generation. + /// Holding the lock during preparation prevents concurrent readers from + /// redundantly indexing the same input or observing a partially admitted batch. + pub fn snapshot( + &self, + plan_id: u64, + plan_version: u64, + ) -> Result, EngineError> { + let mut state = self.generation.lock().unwrap_or_else(|e| e.into_inner()); + if state + .plan + .is_some_and(|plan| plan != (plan_id, plan_version)) + { + return Err(EngineError::capability_miss( + "local_raw_store", + "retained input belongs to another plan generation", + )); + } + if let Some(prepared) = &state.prepared { + return Ok(prepared.clone()); + } + let prepared = Arc::new(PreparedSamples::from_series( + state + .series + .iter() + .map(|(labels, points)| (labels.clone(), points.clone())), + )?); + state.prepared = Some(prepared.clone()); + Ok(prepared) + } + + pub fn sample_count(&self) -> usize { + self.generation + .lock() + .unwrap_or_else(|e| e.into_inner()) + .series + .values() + .map(Vec::len) + .sum() + } + + pub fn estimated_bytes(&self) -> usize { + let state = self.generation.lock().unwrap_or_else(|e| e.into_inner()); + state + .series + .iter() + .map(|(labels, points)| { + std::mem::size_of::() + + labels + .iter() + .map(|(k, v)| { + k.capacity() + v.capacity() + std::mem::size_of::<(String, String)>() + }) + .sum::() + + points.capacity() * std::mem::size_of::<(i64, Option)>() + }) + .sum::() + + state + .prepared + .as_ref() + .map_or(0, |snapshot| snapshot.estimated_bytes()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn sample(metric: &str, timestamp_ms: i64) -> CanonicalSample { + CanonicalSample { + metric: metric.into(), + labels: Default::default(), + series_key: metric.into(), + timestamp_ms, + value: Some(1.0), + } + } + // Repeated queries share indexing; new admitted input publishes a coherent replacement. + #[test] + fn raw_store_caches_once_and_isolates_plan_generations() { + let store = RawSampleStore::default(); + let metrics = BTreeSet::from(["a".into()]); + store.append_admitted(1, 1, &[sample("a", 1), sample("b", 1)], &metrics, false); + assert_eq!(store.sample_count(), 1); + let first = store.snapshot(1, 1).unwrap(); + assert!(Arc::ptr_eq(&first, &store.snapshot(1, 1).unwrap())); + store.append_admitted(1, 1, &[sample("a", 2)], &metrics, false); + assert!(!Arc::ptr_eq(&first, &store.snapshot(1, 1).unwrap())); + assert!(store.snapshot(1, 2).is_err()); + store.append_admitted(1, 2, &[], &BTreeSet::new(), false); + assert_eq!(store.sample_count(), 0); + assert!(store.snapshot(1, 1).is_err()); + } +} diff --git a/data_plane/tests/asapquery_compatibility_process_e2e.rs b/data_plane/tests/asapquery_compatibility_process_e2e.rs index 24ed0e0d..206356bd 100644 --- a/data_plane/tests/asapquery_compatibility_process_e2e.rs +++ b/data_plane/tests/asapquery_compatibility_process_e2e.rs @@ -1259,6 +1259,7 @@ async fn collector_free_profile_serves_complete_matrix_and_falls_back_exactly() .time_selection .scope = planner_types::workload::QueryTimeScope::Unknown; let (mut legacy_request, mut environment) = snapshot.planning_request().unwrap(); + legacy_request.local_raw_execution = false; let query = &mut legacy_request.queries[0]; let parsed = control_plane::query_parser::parse_query_expr_canonical( &query.query_string, diff --git a/tools/o11y-execution/CALIBRATION.md b/tools/o11y-execution/CALIBRATION.md new file mode 100644 index 00000000..720d1886 --- /dev/null +++ b/tools/o11y-execution/CALIBRATION.md @@ -0,0 +1,62 @@ +# Measured CPU cost provider + +This developer workflow prices whole candidates in CPU nanoseconds. Memory and +storage remain separate resource measurements; they are not converted to CPU +with arbitrary weights. The provider never chooses a winner. The control plane +compares its complete quotes and installs the selected artifact. + +1. `discover_snapshot.py --corpus CORPUS --metrics METRICS --template + docs/examples/asapquery-planning-snapshot.json --output DISCOVERY` registers + every unique original query with exact accuracy. `--interval-ms` is an + explicitly declared experimental demand (default 60 seconds). The bounded logical cost horizon is `--repetitions` (default 20) times that + interval; it is separate from input timestamps and measured wall time. + Duplicate corpus occurrences increase their registered query frequency, so + the manifest accounts for all 28 occurrences, not merely 24 unique strings. Provenance preserves input hashes and source counts. + This discovery snapshot contains an **uncalibrated unit enumeration seed**, + not cost evidence; version 2 with no quotes cannot deploy it. +2. Run `calibration_candidates DISCOVERY`. It calls the control plane and Planner, + and exports every bindable candidate with its manifest and install request. + Errors remain in the output. These artifacts are solely for calibration. +3. Measure each candidate in a fresh isolated backend, including its exact service + if used. Feed the complete original input and execute every original query. + Record installation, ingestion/build/update, residency, and retirement CPU, + and each query's inclusive process CPU over enough repetitions to exceed the + operating system's accounting resolution. Validate correctness and record + `warm`, `exact_fallback`, or failure. Record memory and storage separately. + Retain all raw artifacts. Do not reuse calibration timings as the independent + held-out performance evaluation. +4. Run `update_global_profile.py --snapshot DISCOVERY --measurements MEASUREMENTS + --sample-count COUNT --output PROFILE` to replace discovery evidence with a + conservative shared CPU-only measured profile. Network/scan zero fields are + explicitly excluded model dimensions, not measured zero traffic. Re-export the + candidates, and ensure their implementation/manifests match what was measured. + The snapshot currently has one shared implementation model, so this is a + coarse global model; complete candidate quotes carry measured whole costs. +5. Supply measurements to `calibrate.py --candidates CANDIDATES --measurements + MEASUREMENTS --metrics METRICS --snapshot SNAPSHOT --output COSTED_SNAPSHOT`. + The output includes a separate attribution audit. Run the normal control-plane + compiler on this costed snapshot, then perform an independent evaluation. + +Measurement JSON uses this structure (numbers below describe fields, not quotes): + +- `units`: `cpu_ns`; `data_snapshot_id`: `sha256:` followed by the input file hash. +- `candidates`: one record per measured `plan_id`, with the exact `manifest`, + `executable`, and matching `horizon_seconds`. +- Each record's `horizon_phases` has `install`, `ingest_and_build`, `residency`, + and `retirement`, each containing measured `cpu_ns` and `raw_measurement_file`. +- Each record's `queries` maps every manifest query ID to `cpu_ns`, `evaluations`, + `classification`, `correct`, and `raw_measurement_file`. +- Optional `resources` preserves separately measured memory and storage. + +Inclusive horizon CPU is assigned once to the first horizon component, and +inclusive per-query CPU once to the first component for that query. Other +components explicitly receive zero **because their work is included in that +measured total**, not because state update, output, or fallback is free. The +attribution audit records these groups. Missing measurements make a candidate +unavailable; failed or incorrect execution cannot obtain an executable quote. +This is attribution of inclusive measured costs, not individual operator timing. + +The backend currently exposes its Planner-selected forest and a whole-workload +exact alternative. This workflow does not claim exhaustive algorithm or lifecycle +search, and its unit discovery seed can influence which forest becomes available. +A calibrated comparison is only between the candidates actually exported. diff --git a/tools/o11y-execution/README.md b/tools/o11y-execution/README.md index f235b8e4..071c2b1a 100644 --- a/tools/o11y-execution/README.md +++ b/tools/o11y-execution/README.md @@ -1,8 +1,7 @@ # Real-workload execution replay -Developer acceptance tooling, stacked on #524 and the deployment/cost-selection -foundation through #505. This is an executable harness, not a published benefit -result. It invokes the **control-plane compiler**, which calls the pinned Planner, +Developer acceptance tooling. A completed run does not establish a benefit. +It invokes the **control-plane compiler**, which calls the pinned Planner, then boots the production data plane with that compiler's atomic install request. No family override, candidate index, or benchmark-selected winner is accepted. @@ -25,8 +24,10 @@ No family override, candidate index, or benchmark-selected winner is accepted. configured for the input's timestamp range. This process is also the fallback service; do not point the runner at a production/shared instance. -The harness does not provision Prometheus or calibrate the cost provider. These -are required run inputs, not completed end-to-end acceptance evidence. +The lower-level `replay.py` uses externally managed Prometheus processes. +`run_comparison.py` provisions fresh baseline and fallback processes for each +trial. See [CALIBRATION.md](CALIBRATION.md) for measured cost evidence; calibration +enumerates candidates, while the replay compiler chooses the winner normally. ## Run @@ -57,6 +58,12 @@ classification. Forwarded responses carry a backend-owned `x-asap-execution` header; an unmarked success is not counted as warm or fallback. `ingestion.json` records accepted batches; acceptance does not prove worker completion. +`execution_provenance` distinguishes summary-only `asap`, `hybrid`, `local_raw` +and `external_exact`, and records actual raw scans and summary readouts. Both +hybrid and raw-only execution remain under `exact_fallback`; they are never +reported as summary-only warm execution. A typed residual DAG can retain selected +summary siblings, but a particular corpus may still produce no materializations. + The runner waits for the finite-input completion barrier. The first traversal is called `first_pass`, not “cold cache”; later traversals are `repeat`. All failures and fallback responses remain in the denominator. A completion file means the @@ -78,9 +85,10 @@ failures, first-pass/repeat latency distributions and sequential service rate. Duplicate series, failed responses, unsupported response types and warnings make a result uncomparable. Matching one dataset is not a formal confidence guarantee. -The latency ratio is emitted only when **all** matched responses are exact-equal -and successful. Approximate discrepancies are still reported, but no arbitrary -error threshold is substituted for each query's accuracy contract. Fallbacks +The latency ratio is emitted only when **all** matched responses pass the recorded +comparison and succeed. Tolerances default to zero; explicit absolute/relative +tolerances preserve strict equality and raw errors alongside the decision. These +numeric tolerances do not establish a sketch's accuracy guarantee. Fallbacks remain in the totals. The ratio measures query service time only, never amortized end-to-end savings. Raw per-request timings allow other analyses without hiding the unsuccessful portion of the workload. @@ -105,14 +113,35 @@ separate fallback service, the report preserves the shared-cache limitation. `--exact-storage` and `--fallback-storage` record logical file sizes separately; backend output file sizes include logs and are not retained summary heap sizes. -The provider's estimated costs are preserved next to measured quantities without -pretending abstract model units are CPU nanoseconds. End-to-end benefit and -estimated/measured cost ratios remain null until their units, lifecycle scope, -exact-service startup/storage costs and resource budgets are matched. Separate -fresh-process/cache-controlled trials, a retained-state measurement, calibration -provenance and real-corpus execution evidence are still required before declaring -the five #524 acceptance criteria complete. A shared fallback/baseline service -can transfer cache warmth; alternating order does not eliminate this confound. +`summarize.py` compares estimated and measured query CPU only after verifying the +CPU model, data hash, selected manifest and priced query multiplicities. It +excludes lifecycle components from that ratio. Full lifecycle benefit remains +unavailable until matching setup, update, residency and retirement measurements +exist. A shared fallback/baseline service can transfer cache warmth; alternating +order does not eliminate this confound. + +## Fresh repeated trials + +```sh +python3 tools/o11y-execution/run_comparison.py \ + --prometheus /path/prometheus \ + --metrics /path/metrics.txt --queries /path/queries.json \ + --snapshot /path/o11y-costed-snapshot.json \ + --compiler target/release/examples/compile_workload_artifact \ + --data-plane target/release/data_plane \ + --cpu-affinity 4,5 --trials 3 --repetitions 20 \ + --output /path/new-trials +python3 tools/o11y-execution/summarize.py /path/new-trials/trial-1/replay \ + --output /path/new-trials/trial-1/summary.json +``` + +Each trial uses new empty TSDB directories and three distinct ports. The wrapper +records commands and Prometheus startup/lifetime resources, then stops its own +services. `backend-lifecycle.json` records backend lifetime CPU and peak RSS using +per-PID `wait4`. Process exit is not a summary-retirement measurement. Fresh +processes do not imply evicted OS caches, fixed memory quotas, or a concurrent +throughput test. Every repetition preserves the original evaluation timestamps; +this run does not simulate a live dashboard's advancing query windows. ## Finite-input completion @@ -126,5 +155,5 @@ backend process for another input generation. This endpoint is for a finite replay, not a live ingestion watermark. Closing a trailing pane does not by itself prove its coverage matches every query window; -unsupported or incomplete readouts must still follow exact fallback. This change -adds no local raw-query storage or hybrid operator execution. +unsupported or incomplete readouts must still follow exact fallback. Typed raw +residual plans prepare their raw index during drain so setup costs remain visible. diff --git a/tools/o11y-execution/calibrate.py b/tools/o11y-execution/calibrate.py new file mode 100644 index 00000000..3028c94e --- /dev/null +++ b/tools/o11y-execution/calibrate.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python3 +"""Turn isolated, inclusive candidate CPU measurements into complete cost quotes. + +This provider selects no winner. All candidate measurements include backend and +its exact service, when used. Horizon CPU includes installation, raw ingestion, +state building/updating, residency and retirement. Query CPU includes every +operator, fallback, and response encoding. Inclusive totals are attributed once; +zero entries mean subsumed work, never an assertion of free execution. Memory +and storage remain separately reported quantities, not invented CPU conversions. +""" +import argparse +import hashlib +import json +import math +from pathlib import Path + + +def nonnegative(value, name): + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value) or value < 0: + raise ValueError(f"{name} must be finite nonnegative measured CPU nanoseconds") + return value + + +def digest(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def calibrate(candidates, measurements, data_snapshot_id, observed_at_unix_ms, valid_for_ms): + if measurements.get("units") != "cpu_ns" or measurements.get("data_snapshot_id") != data_snapshot_id: + raise ValueError("measurement units or data identity mismatch") + by_id = {} + for row in measurements["candidates"]: + key = row["plan_id"] + if key in by_id: + raise ValueError("duplicate measured candidate") + by_id[key] = row + quotes, attribution, unavailable = [], [], [] + for artifact in candidates["candidates"]: + if "manifest" not in artifact: + unavailable.append(artifact) + continue + manifest = artifact["manifest"] + pid = manifest["plan_id"] + measured = by_id.get(pid) + if not measured or not measured.get("executable"): + unavailable.append({"plan_id": pid, "reason": (measured or {}).get("unavailable_reason", "not measured")}) + continue + if measured.get("manifest") != manifest: + raise ValueError(f"candidate {pid}: measured manifest differs") + if measured.get("horizon_seconds") != manifest["horizon_seconds"]: + raise ValueError(f"candidate {pid}: measured horizon differs") + required_phases = {"install", "ingest_and_build", "residency", "retirement"} + phases = measured.get("horizon_phases", {}) + if set(phases) != required_phases: + raise ValueError(f"candidate {pid}: incomplete horizon phases") + horizon_cpu = sum(nonnegative(phases[p]["cpu_ns"], p) for p in required_phases) + if any(not phases[p].get("raw_measurement_file") for p in required_phases): + raise ValueError("each phase requires a raw measurement artifact") + costs = {key: 0.0 for key in manifest["components"]} + horizon_keys = sorted(k for k, v in manifest["components"].items() if v["unit"] == "horizon") + if not horizon_keys: + raise ValueError("no horizon component for inclusive setup/upkeep CPU") + costs[horizon_keys[0]] = horizon_cpu + allocation = {horizon_keys[0]: {"inclusive_of": horizon_keys, "cpu_ns": horizon_cpu}} + query_measurements = measured.get("queries", {}) + if set(query_measurements) != set(manifest["workload"]): + raise ValueError(f"candidate {pid}: incomplete query execution coverage") + for qid in manifest["workload"]: + row = query_measurements[qid] + count = row["evaluations"] + if isinstance(count, bool) or not isinstance(count, int) or count <= 0: + raise ValueError("query evaluation count must be positive") + if row.get("classification") not in ("warm", "exact_fallback") or not row.get("correct"): + raise ValueError(f"candidate {pid}: {qid} failed execution/correctness validation") + if not row.get("raw_measurement_file"): + raise ValueError("each query requires a raw measurement artifact") + keys = sorted(k for k in costs if k.startswith(f"query:{qid}:") or k == f"result:{qid}") + if not keys or any(manifest["components"][k]["unit"] != "query_evaluation" for k in keys): + raise ValueError("missing per-query components") + cpu = nonnegative(row["cpu_ns"], qid) / count + costs[keys[0]] = cpu + allocation[keys[0]] = {"inclusive_of": keys, "cpu_ns_per_evaluation": cpu} + quotes.append({"manifest": manifest, "executable": True, "unit_costs": costs}) + attribution.append({"plan_id": pid, "allocation": allocation, + "resources": measured.get("resources"), + "scope": "backend plus any exact fallback service; complete inclusive CPU; memory/storage reported separately"}) + return ({"data_snapshot_id": data_snapshot_id, "model_version": "measured-inclusive-cpu-ns-v1", + "observed_at_unix_ms": observed_at_unix_ms, "valid_for_ms": valid_for_ms, "quotes": quotes}, + {"attribution": attribution, "unavailable": unavailable}) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--candidates", type=Path, required=True) + parser.add_argument("--measurements", type=Path, required=True) + parser.add_argument("--metrics", type=Path, required=True) + parser.add_argument("--snapshot", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + args = parser.parse_args() + snapshot = json.loads(args.snapshot.read_text()) + evidence, audit = calibrate(json.loads(args.candidates.read_text()), json.loads(args.measurements.read_text()), + "sha256:" + digest(args.metrics), snapshot["environment"]["observed_at_unix_ms"], + snapshot["environment"]["max_evidence_age_ms"]) + snapshot["snapshot_version"] = 2 + snapshot["workload_cost_evidence"] = evidence + args.output.write_text(json.dumps(snapshot, indent=2, allow_nan=False) + "\n") + args.output.with_suffix(".calibration.json").write_text(json.dumps(audit, indent=2, allow_nan=False) + "\n") + if not evidence["quotes"]: + raise SystemExit("no completely measured executable candidates; audit saved, selection must fail closed") + + +if __name__ == "__main__": + main() diff --git a/tools/o11y-execution/calibrate_runtime.py b/tools/o11y-execution/calibrate_runtime.py new file mode 100644 index 00000000..6225f5e3 --- /dev/null +++ b/tools/o11y-execution/calibrate_runtime.py @@ -0,0 +1,208 @@ +#!/usr/bin/env python3 +"""Measure unselected candidate artifacts in isolated backend/fallback processes. + +This produces calibration observations, never selection overrides. Accelerated +input replay is explicitly distinct from a wall-clock deployment horizon. +""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import resource +import subprocess +import time +import urllib.parse + +import replay as runner +from compare import compare_results, process_delta, process_snapshot + + +def wait_ready(url, child): + deadline = time.monotonic() + 60 + while time.monotonic() < deadline: + if child.poll() is not None: + raise RuntimeError(f"process exited {child.returncode}: {url}") + if runner._http_request(url)["http_status"] == 200: + return + time.sleep(.1) + raise RuntimeError(f"readiness timeout: {url}") + + +def snapshots(children): + return {name: process_snapshot(child.pid) for name, child in children.items()} + + +def phase(folder, name, before, after, elapsed_ns): + deltas = {key: process_delta(before.get(key), after.get(key)) for key in before} + if not all(value is not None for value in deltas.values()): + raise RuntimeError(f"unreadable process counters in {name}") + raw = folder / f"phase-{name}.json" + runner.write_json(raw, {"before": before, "after": after, "deltas": deltas, "wall_ns": elapsed_ns}) + return {"cpu_ns": sum(value["cpu_ns"] for value in deltas.values()), "raw_measurement_file": str(raw.resolve())} + + +def file_bytes(root): + return sum(path.stat().st_size for path in root.rglob("*") if path.is_file()) + + +def measure(args, artifact, samples, corpus, snapshot, folder): + folder.mkdir() + manifest = artifact["manifest"] + row = {"plan_id": manifest["plan_id"], "manifest": manifest, "executable": False, + "horizon_seconds": manifest["horizon_seconds"], "horizon_phases": {}, "queries": {}} + children, logs = {}, [] + cpus = {int(value) for value in args.cpu_affinity.split(",")} + def limits(): + os.sched_setaffinity(0, cpus) + def launch(name, command): + log = (folder / f"{name}.log").open("w") + logs.append(log) + child = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, preexec_fn=limits) + children[name] = child + return child + fallback = f"http://127.0.0.1:{args.fallback_port}" + backend = f"http://127.0.0.1:{args.backend_port}" + install = folder / "install.json" + runner.write_json(install, artifact["install_request"]) + config = folder / "prometheus.yml" + config.write_text('global:\n scrape_interval: 1h\nscrape_configs: []\n') + children_cpu_before = resource.getrusage(resource.RUSAGE_CHILDREN) + try: + start = time.perf_counter_ns() + prom = launch("fallback", [str(args.prometheus.resolve()), f"--config.file={config.resolve()}", + f"--storage.tsdb.path={(folder / 'prometheus-data').resolve()}", + f"--web.listen-address=127.0.0.1:{args.fallback_port}", "--web.enable-remote-write-receiver", + "--storage.tsdb.retention.time=1000000h"]) + # Backend startup validates its exact service immediately. + wait_ready(fallback + "/-/ready", prom) + dp = launch("backend", [str(args.data_plane.resolve()), "--profile", "asapquery", "--physical-plan", str(install.resolve()), + "--prometheus-server", fallback, "--forward-unsupported-queries", "--http-port", str(args.backend_port), + "--output-dir", str((folder / "backend-data").resolve()), "--precompute-allowed-lateness-ms", "0", + "--precompute-flush-interval-ms", "25"]) + wait_ready(backend + "/api/v1/health", dp) + after = snapshots(children) + # Fresh processes: cumulative CPU from exec includes all install/startup work. + raw = folder / "phase-install.json" + runner.write_json(raw, {"after": after, "wall_ns": time.perf_counter_ns() - start}) + row["horizon_phases"]["install"] = {"cpu_ns": sum(v["cpu_ns"] for v in after.values()), "raw_measurement_file": str(raw.resolve())} + before, start = after, time.perf_counter_ns() + runner.PROCESS_IDS.clear() + runner.PROCESS_IDS.update({"backend": dp.pid, "fallback_service": prom.pid}) + runner.ingest(samples, [fallback, backend], folder) + drained = runner.request(backend + "/api/v1/precompute/drain", b"") + runner.write_json(folder / "drain.json", drained) + if drained["http_status"] != 200 or drained["response"].get("complete") is not True: + raise RuntimeError("precompute drain did not complete") + after = snapshots(children) + row["horizon_phases"]["ingest_and_build"] = phase(folder, "ingest_and_build", before, after, time.perf_counter_ns() - start) + before, start = after, time.perf_counter_ns() + time.sleep(args.residency_seconds) + after = snapshots(children) + row["horizon_phases"]["residency"] = phase(folder, "residency", before, after, time.perf_counter_ns() - start) + original_by_id = {f"compat-query-{i}": entry["query"] for i, entry in enumerate(snapshot["query_workload"]["repeating_queries"])} + for qid in manifest["workload"]: + occurrences = [q for q in corpus["queries"] if q["query"] == original_by_id[qid]] + if not occurrences: + raise RuntimeError(f"no original corpus occurrences for {qid}") + exact = {} + for occurrence in occurrences: + params = urllib.parse.urlencode({"query": occurrence["query"], "time": f'{occurrence["eval_timestamp_ms"] / 1000:.3f}'}) + exact[occurrence["id"]] = runner._http_request(args.reference_url.rstrip("/") + "/api/v1/query?" + params) + records, before, start = [], snapshots(children), time.perf_counter_ns() + repeat = 0 + measured_cpu = 0 + while repeat < args.repetitions or (measured_cpu < args.minimum_query_cpu_ns and repeat < args.max_repetitions): + for occurrence in occurrences: + params = urllib.parse.urlencode({"query": occurrence["query"], "time": f'{occurrence["eval_timestamp_ms"] / 1000:.3f}'}) + answer = runner._http_request(backend + "/api/v1/query?" + params) + reference = exact[occurrence["id"]] + route = runner.classify(answer["response"], answer["headers"]) if answer["http_status"] == 200 else "failed" + comparison = compare_results(answer["response"], reference["response"], args.relative_tolerance, args.absolute_tolerance) + records.append({**occurrence, "repetition": repeat, "execution": route, + "execution_provenance": runner.execution_provenance(answer["response"], answer["headers"]), **answer, + "exact": reference, "comparison": comparison}) + repeat += 1 + if repeat % 10 == 0 or repeat >= args.repetitions: + current = snapshots(children) + measured_cpu = sum(current[key]["cpu_ns"] - before[key]["cpu_ns"] for key in before) + after = snapshots(children) + query_phase = phase(folder, "query-" + qid, before, after, time.perf_counter_ns() - start) + raw = folder / f"queries-{qid}.json" + runner.write_json(raw, records) + routes = {record["execution"] for record in records} + correct = all(record["comparison"]["equal"] and record["exact"]["http_status"] == 200 for record in records) + row["queries"][qid] = {"cpu_ns": query_phase["cpu_ns"], "evaluations": len(records), + "classification": next(iter(routes)) if len(routes) == 1 else "mixed", "correct": correct, + "raw_measurement_file": str(raw.resolve()), "resource_measurement_file": query_phase["raw_measurement_file"], + "cpu_resolution_censored": query_phase["cpu_ns"] < args.minimum_query_cpu_ns} + state = runner.request(backend + "/api/v1/store/metrics") + runner.write_json(folder / "store.json", state) + final = snapshots(children) + row["resources"] = {"peak_memory_bytes": sum(v["process_lifetime_peak_rss_bytes"] for v in final.values()), + "storage_bytes": file_bytes(folder / "prometheus-data") + file_bytes(folder / "backend-data"), + "backend_state": state, "processes": final, "source_scan_bytes": None, "network_bytes": None, + "residency_wall_seconds": args.residency_seconds, + "scope": "accelerated finite-input replay; no extrapolation of short idle residency to logical data horizon; process HWM sum is conservative"} + before_total = sum(v["cpu_ns"] for v in final.values()) + for child in children.values(): + child.terminate() + for child in children.values(): + child.wait(timeout=30) + usage = resource.getrusage(resource.RUSAGE_CHILDREN) + total_cpu = int((usage.ru_utime + usage.ru_stime - children_cpu_before.ru_utime - children_cpu_before.ru_stime) * 1e9) + retirement = folder / "phase-retirement.json" + runner.write_json(retirement, {"wait4_children_cpu_ns": total_cpu, "proc_before_retirement_cpu_ns": before_total, + "note": "difference includes /proc tick rounding; nonnegative clamp below tick resolution"}) + row["horizon_phases"]["retirement"] = {"cpu_ns": max(0, total_cpu - before_total), "raw_measurement_file": str(retirement.resolve())} + invalid = [qid for qid, q in row["queries"].items() if not q["correct"] or q["classification"] not in ("warm", "exact_fallback") or q["cpu_resolution_censored"]] + if invalid: + row["unavailable_reason"] = "failed/mixed/incorrect or CPU below measurement resolution: " + ",".join(invalid) + else: + row["executable"] = True + except Exception as error: + row["unavailable_reason"] = str(error) + finally: + for child in children.values(): + if child.poll() is None: + child.kill() + child.wait() + for log in logs: + log.close() + runner.PROCESS_IDS.clear() + runner.write_json(folder / "measurement.json", row) + return row + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ["candidates", "metrics", "queries", "snapshot", "data-plane", "prometheus", "output"]: + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--reference-url", required=True, help="separate already-loaded exact Prometheus using identical input") + parser.add_argument("--cpu-affinity", required=True) + parser.add_argument("--backend-port", type=int, default=19210) + parser.add_argument("--fallback-port", type=int, default=19211) + parser.add_argument("--repetitions", type=int, default=50) + parser.add_argument("--max-repetitions", type=int, default=1000) + parser.add_argument("--minimum-query-cpu-ns", type=int, default=100000000, help="ten Linux 100Hz CPU ticks by default") + parser.add_argument("--residency-seconds", type=float, default=1) + parser.add_argument("--relative-tolerance", type=float, default=0.0) + parser.add_argument("--absolute-tolerance", type=float, default=0.0) + args = parser.parse_args() + if args.repetitions < 1 or args.max_repetitions < args.repetitions or args.minimum_query_cpu_ns <= 0 or args.residency_seconds < 0: + parser.error("positive repetitions and nonnegative residency required") + args.output.mkdir(parents=True, exist_ok=False) + samples = runner.parse_samples(args.metrics.read_text().splitlines()) + corpus, snapshot = json.loads(args.queries.read_text()), json.loads(args.snapshot.read_text()) + runner.validate_workload(snapshot, corpus) + result = {"units": "cpu_ns", "data_snapshot_id": "sha256:" + hashlib.sha256(args.metrics.read_bytes()).hexdigest(), + "scope": "accelerated finite-input calibration; measured wall residency is not full logical-horizon residency", "candidates": []} + for index, candidate in enumerate(json.loads(args.candidates.read_text())["candidates"]): + if "manifest" not in candidate or "install_request" not in candidate: + continue + result["candidates"].append(measure(args, candidate, samples, corpus, snapshot, args.output / f"candidate-{index}")) + runner.write_json(args.output / "measurements.json", result) + + +if __name__ == "__main__": + main() diff --git a/tools/o11y-execution/compare.py b/tools/o11y-execution/compare.py index 09d7da22..520eee0f 100644 --- a/tools/o11y-execution/compare.py +++ b/tools/o11y-execution/compare.py @@ -31,7 +31,9 @@ def result_samples(response): return kind, groups, result -def compare_results(actual, expected): +def compare_results(actual, expected, relative_tolerance=0.0, absolute_tolerance=0.0): + if any(not math.isfinite(v) or v < 0 for v in (relative_tolerance, absolute_tolerance)): + raise ValueError("comparison tolerances must be finite and nonnegative") try: ak, ag, a = result_samples(actual) ek, eg, e = result_samples(expected) @@ -39,7 +41,7 @@ def compare_results(actual, expected): raise ValueError("different result types") except (ValueError, TypeError, KeyError) as error: return {"comparable": False, "equal": False, "reason": str(error)} - absolute, relative, zero, nonfinite = [], [], 0, 0 + absolute, relative, zero, nonfinite, outside_tolerance = [], [], 0, 0, 0 for key in a.keys() & e.keys(): x, y = a[key], e[key] if not math.isfinite(x) or not math.isfinite(y): @@ -51,6 +53,8 @@ def compare_results(actual, expected): nonfinite += 1 continue absolute.append(error) + if not math.isclose(x, y, rel_tol=relative_tolerance, abs_tol=absolute_tolerance): + outside_tolerance += 1 if y: ratio = error / abs(y) if math.isfinite(ratio): @@ -61,7 +65,10 @@ def compare_results(actual, expected): zero += 1 missing, extra = len(e.keys() - a.keys()), len(a.keys() - e.keys()) return {"comparable": True, - "equal": ag == eg and not (missing or extra or zero or nonfinite or any(absolute)), + "equal": ag == eg and not (missing or extra or nonfinite or outside_tolerance), + "strict_equal": ag == eg and not (missing or extra or zero or nonfinite or any(absolute)), + "relative_tolerance": relative_tolerance, "absolute_tolerance": absolute_tolerance, + "samples_outside_tolerance": outside_tolerance, "missing_series": len(eg - ag), "extra_series": len(ag - eg), "missing_samples": missing, "extra_samples": extra, "completeness": (len(a.keys() & e.keys()) / len(e)) if e else (1.0 if not a else 0.0), @@ -80,7 +87,9 @@ def distribution(values): def summarize(rows): - comparisons = [compare_results(row["response"], row.get("exact", {}).get("response", {})) for row in rows] + comparisons = [compare_results(row["response"], row.get("exact", {}).get("response", {}), + row.get("comparison", {}).get("relative_tolerance", 0.0), + row.get("comparison", {}).get("absolute_tolerance", 0.0)) for row in rows] eligible = bool(rows) and all( c["equal"] and r["execution"] in ("warm", "exact_fallback") and r["exact"].get("http_status") == 200 for r, c in zip(rows, comparisons)) @@ -99,6 +108,8 @@ def cpu_total(requests, names): "cpu_scope": "request intervals, whole processes including background work; fallback CPU charged to backend; /proc tick granularity", "occurrences": len(rows), "execution_counts": {k: sum(r["execution"] == k for r in rows) for k in ("warm", "exact_fallback", "failed")}, + "execution_detail_counts": {k: sum(r.get("execution_provenance", {}).get("detail") == k for r in rows) for k in ("asap", "hybrid", "local_raw", "external_exact")}, + "summary_acceleration_scope": "warm excludes every request using raw execution; hybrid is reported separately under exact_fallback", "equal_results": sum(c["equal"] for c in comparisons), "uncomparable_results": sum(not c["comparable"] for c in comparisons), "comparisons": comparisons, diff --git a/tools/o11y-execution/discover_snapshot.py b/tools/o11y-execution/discover_snapshot.py new file mode 100644 index 00000000..1f2fa0ec --- /dev/null +++ b/tools/o11y-execution/discover_snapshot.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Register the complete real corpus for candidate discovery, never cost selection. + +Unit discovery costs are an explicit uncalibrated enumeration seed. Version 2 +without quotes cannot select/deploy. Replace implementation evidence with measured +calibration and re-export candidates before producing final deployment quotes. +""" +import argparse +from collections import Counter +import hashlib +import json +from pathlib import Path +import re +import time +from replay import parse_samples + + +def duration_ms(text): + units = {"ms": 1, "s": 1000, "m": 60000, "h": 3600000, "d": 86400000, "w": 604800000, "y": 31536000000} + return sum(int(n) * units[u] for n, u in re.findall(r"(\d+)(ms|[smhdwy])", text)) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--corpus", type=Path, required=True) + parser.add_argument("--metrics", type=Path, required=True) + parser.add_argument("--template", type=Path, required=True) + parser.add_argument("--output", type=Path, required=True) + parser.add_argument("--interval-ms", type=int, default=60000, help="declared repeated-query experiment demand") + parser.add_argument("--repetitions", type=int, default=20, help="query evaluations per bounded replay batch") + args = parser.parse_args() + corpus = json.loads(args.corpus.read_text()) + rows = parse_samples(args.metrics.read_text().splitlines()) + snapshot = json.loads(args.template.read_text()) + now = int(time.time() * 1000) + input_span = (rows[-1][2] - rows[0][2]) / 1000 + horizon = args.repetitions * args.interval_ms / 1000 + if input_span <= 0 or horizon <= 0 or args.interval_ms <= 0: + raise ValueError("positive input horizon and recurrence required") + def evidence(value): + return {"value": value, "source": "observed", "observed_at_ms": now, "valid_for_ms": 86400000} + data = snapshot["data_workload"] + data.update(ingestion_volume=evidence(len(rows)), ingestion_rate=evidence(len(rows)/horizon), + input_cardinality=evidence(len({tuple(sorted(labels.items())) for labels, _, _ in rows}))) + data["ingestion_rate"]["source"] = "derived" + registrations, query_audit = [], [] + frequencies = Counter(row["query"] for row in corpus["queries"]) + for query in dict.fromkeys(row["query"] for row in corpus["queries"]): + windows = [duration_ms(x) for x in re.findall(r"\[([0-9a-z]+)(?::[^\]]*)?\]", query)] + lookback = max(windows, default=300000) + if args.interval_ms % frequencies[query]: + raise ValueError("base interval must divide exactly by query occurrence frequency") + interval = args.interval_ms // frequencies[query] + registrations.append({"query": query, "demand": {"fixed_interval": interval}, + "requirements": {"accuracy": {"explicit": "Exact"}, "response_latency": "unspecified"}, + "predictability": {"predictable": {"known_at": None}}, + "time_selection": {"scope": "real_time", "lookback": lookback, "as_of": None}}) + query_audit.append({"query": query, "window_lookback_ms": lookback, + "occurrence_count": frequencies[query], "expected_evaluations": frequencies[query] * args.repetitions, + "declared_interval_ms": interval, + "lookback_method": "largest explicit range; instant selector defaults to Prometheus 5m; original offsets/subqueries preserved in query"}) + snapshot["query_workload"].update(repeating_queries=registrations, data_workload=data, query_batch=None) + snapshot["snapshot_version"] = 2 + snapshot.pop("workload_cost_evidence", None) + implementation = snapshot["implementation"] + implementation.update(evidence_observed_at_unix_ms=now, evidence_valid_for_ms=86400000, horizon_seconds=horizon) + implementation["lifecycle_costs"] = dict.fromkeys(("build", "maintenance_per_update", "read", "retention_per_second", "retirement"), 1.0) + implementation["implementation_cost"].update(model_version="UNCALIBRATED-enumeration-only", observed_at_unix_ms=now, + valid_for_ms=86400000, horizon_seconds=horizon, cpu_cost=1.0, peak_memory_bytes=0, network_bytes=0, + storage_bytes=0, source_scan_bytes=0, weighted_cost=1.0) + snapshot["environment"].update(observed_at_unix_ms=now, activation_unix_ms=rows[0][2], max_evidence_age_ms=86400000, + capability_snapshot_id="o11y-backend-local-calibration-v1") + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text(json.dumps(snapshot, indent=2) + "\n") + args.output.with_suffix(".provenance.json").write_text(json.dumps({ + "purpose": "candidate discovery only; uncalibrated costs are NOT execution quotes or benefit evidence", + "input_sha256": hashlib.sha256(args.metrics.read_bytes()).hexdigest(), + "corpus_sha256": hashlib.sha256(args.corpus.read_bytes()).hexdigest(), + "upstream_revision": corpus["upstream_revision"], "sample_count": len(rows), + "series_count": data["input_cardinality"]["value"], "first_timestamp_ms": rows[0][2], "last_timestamp_ms": rows[-1][2], + "declared_query_interval_ms": args.interval_ms, "repetitions": args.repetitions, + "input_span_seconds": input_span, "cost_horizon_seconds": horizon, + "event_time_ingestion_rate": len(rows) / input_span, + "derived_bounded_replay_ingestion_rate": len(rows) / horizon, + "cost_scope": "bounded accelerated replay: entire finite input once plus declared query repetitions; logical horizon is not wall-clock residency", + "accuracy": "exact", + "queries": query_audit}, indent=2) + "\n") + + +if __name__ == "__main__": + main() diff --git a/tools/o11y-execution/process_lifecycle.py b/tools/o11y-execution/process_lifecycle.py new file mode 100644 index 00000000..03a8c0b8 --- /dev/null +++ b/tools/o11y-execution/process_lifecycle.py @@ -0,0 +1,42 @@ +"""Collect per-child lifetime usage; never label aggregate child usage as one PID.""" +import os +import time +from compare import process_snapshot + + +def stop(child, timeout=30): + before = process_snapshot(child.pid) + started = time.perf_counter_ns() + forced = False + usage = None + # Popen.poll()/wait() may already have reaped an exited child. Its usage + # cannot subsequently be recovered with wait4, and is not a measured zero. + if child.returncode is None: + child.terminate() + deadline = time.monotonic() + timeout + while True: + try: + pid, status, collected = os.wait4(child.pid, os.WNOHANG) + except ChildProcessError: + break + if pid: + child.returncode = os.waitstatus_to_exitcode(status) + usage = collected + break + if time.monotonic() >= deadline: + forced = True + child.kill() + try: + _, status, usage = os.wait4(child.pid, 0) + child.returncode = os.waitstatus_to_exitcode(status) + except ChildProcessError: + pass + break + time.sleep(.01) + return {'before_shutdown': before, + 'lifetime_cpu_ns': int((usage.ru_utime + usage.ru_stime) * 1e9) if usage is not None else None, + 'lifetime_peak_rss_bytes': usage.ru_maxrss * 1024 if usage is not None else None, + 'usage_unavailable_reason': 'child was already reaped; per-PID wait4 usage unavailable' if usage is None else None, + 'shutdown_wall_ns': time.perf_counter_ns() - started, + 'forced_kill': forced, 'exit_code': child.returncode, + 'scope': 'Per-PID wait4 CPU and peak RSS cover the whole child lifetime, not shutdown only. Process exit is not a measured summary retirement operation.'} diff --git a/tools/o11y-execution/replay.py b/tools/o11y-execution/replay.py index eeb15588..9045b071 100644 --- a/tools/o11y-execution/replay.py +++ b/tools/o11y-execution/replay.py @@ -37,8 +37,14 @@ def constrain_process(pid, cpus, address_space_bytes=None): def classify(response, headers=None): if response.get("status") != "success": return "failed" - if (headers or {}).get("x-asap-execution") == "exact_fallback": + declared = (headers or {}).get("x-asap-execution") + if declared in ("exact_fallback", "failed"): + return declared + detail = (headers or {}).get("x-asap-execution-detail") + if detail in ("hybrid", "local_raw"): return "exact_fallback" + if detail == "invalid_provenance": + return "failed" sources = {x for x in response.get("infos", []) if isinstance(x, str) and x.startswith("data_source:")} if sources == {"data_source: asap_query"}: return "warm" @@ -47,6 +53,21 @@ def classify(response, headers=None): return "failed" +def execution_provenance(response, headers=None): + headers = headers or {} + route = classify(response, headers) + detail = headers.get("x-asap-execution-detail") + if detail is None: + detail = "asap" if route == "warm" else "external_exact" if route == "exact_fallback" else "failed" + counts = {} + for name, header in (("raw_scan_evaluations", "x-asap-raw-scan-evaluations"), + ("summary_readout_evaluations", "x-asap-summary-readout-evaluations"), + ("memo_hits", "x-asap-memo-hits")): + value = headers.get(header) + counts[name] = int(value) if value is not None and value.isdigit() else None + return {"detail": detail, **counts} + + def validate_workload(snapshot, corpus): rows = corpus["queries"] if not corpus.get("upstream_revision") or not rows: @@ -203,7 +224,7 @@ def ingest(rows, endpoints, output): raise RuntimeError(f"ingestion failed at batch {offset}; inspect partial acceptance before retry") -def replay(queries, backend, output, repetitions, exact_url=None): +def replay(queries, backend, output, repetitions, exact_url=None, relative_tolerance=0.0, absolute_tolerance=0.0): rows = [] for repeat in range(repetitions): for query_index, query in enumerate(queries): @@ -220,10 +241,10 @@ def replay(queries, backend, output, repetitions, exact_url=None): if answer["http_status"] != 200: route = "failed" rows.append({**query, "repetition": repeat, "phase": "first_pass" if repeat == 0 else "repeat", - "execution": route, **answer}) + "execution": route, "execution_provenance": execution_provenance(answer["response"], answer["headers"]), **answer}) if exact is not None: rows[-1]["exact"] = exact - rows[-1]["comparison"] = compare_results(answer["response"], exact["response"]) + rows[-1]["comparison"] = compare_results(answer["response"], exact["response"], relative_tolerance, absolute_tolerance) rows[-1]["pair_order"] = "exact_first" if exact_first else "backend_first" write_json(output / "queries.json", rows) return rows @@ -248,6 +269,8 @@ def main(): parser.add_argument("--port", type=int, default=18089) parser.add_argument("--settle-seconds", type=float, default=0, help="deprecated; completion uses explicit finite-input drain") parser.add_argument("--repetitions", type=int, default=2) + parser.add_argument("--relative-tolerance", type=float, default=0.0) + parser.add_argument("--absolute-tolerance", type=float, default=0.0) parser.add_argument("--output", type=Path, required=True) args = parser.parse_args() if args.repetitions < 1 or not 0 <= args.settle_seconds <= 60: @@ -347,7 +370,8 @@ def limits(): phases["after_ingest_and_drain"] = process_snapshots() write_json(args.output / "store-after-build.json", request(backend + "/api/v1/store/metrics")) write_json(args.output / "process-phases.json", phases) - results = replay(queries, backend, args.output, args.repetitions, args.exact_url if args.compare else None) + results = replay(queries, backend, args.output, args.repetitions, args.exact_url if args.compare else None, + args.relative_tolerance, args.absolute_tolerance) phases["after_queries"] = process_snapshots() write_json(args.output / "process-phases.json", phases) store = request(backend + "/api/v1/store/metrics") @@ -362,6 +386,7 @@ def disk_bytes(path): write_json(args.output / "storage.json", storage) write_json(args.output / "completion.json", {"complete": True, "execution_counts": {k: sum(r["execution"] == k for r in results) for k in ["warm", "exact_fallback", "failed"]}, + "execution_detail_counts": {k: sum(r["execution_provenance"]["detail"] == k for r in results) for k in ["asap", "hybrid", "local_raw", "external_exact", "failed", "invalid_provenance"]}, "benefit_claim": None}) if args.compare: report = {"schema_version": 1, "all_requests": summarize(results), @@ -371,6 +396,8 @@ def disk_bytes(path): for query in queries}, "by_execution": {route: summarize([r for r in results if r["execution"] == route]) for route in ["warm", "exact_fallback", "failed"]}, + "by_execution_detail": {detail: summarize([r for r in results if r["execution_provenance"]["detail"] == detail]) + for detail in ["asap", "hybrid", "local_raw", "external_exact"]}, "estimated_cost": plan["cost_comparison"], "measurement_units": {"latency": "nanoseconds", "cpu": "process CPU nanoseconds", "memory": "bytes"}, "resource_limits": {"cpu_affinity": sorted(cpus) if cpus else None, @@ -398,14 +425,15 @@ def disk_bytes(path): "Raw process RSS is not summary state size; store.json retains backend counters", "Service startup before supplied PID attachment and isolated cold-cache runs remain unmeasured", "Query equality on one dataset is not a formal approximation confidence guarantee"]} + from summarize import query_cost_comparison, STALE + report["query_cost_comparison"] = query_cost_comparison(plan, report, results, provenance, json.loads(args.snapshot.read_text())) + if report["query_cost_comparison"]["available"]: + report["limitations"] = [item for item in report["limitations"] if item != STALE] + report["limitations"].append("Full lifecycle ratio remains unavailable: residency, retirement and service startup scopes are not aligned") write_json(args.output / "comparison.json", report) finally: - child.terminate() - try: - child.wait(timeout=10) - except subprocess.TimeoutExpired: - child.kill() - child.wait() + from process_lifecycle import stop + write_json(args.output / "backend-lifecycle.json", stop(child, timeout=10)) if __name__ == "__main__": diff --git a/tools/o11y-execution/run_comparison.py b/tools/o11y-execution/run_comparison.py new file mode 100644 index 00000000..15578026 --- /dev/null +++ b/tools/o11y-execution/run_comparison.py @@ -0,0 +1,119 @@ +#!/usr/bin/env python3 +"""Own fresh baseline/fallback services and retain repeated comparison evidence.""" +import argparse +import hashlib +import json +import os +from pathlib import Path +import subprocess +import sys +import time +import urllib.request + +from compare import process_snapshot +from process_lifecycle import stop + + +def save(path, value): + path.write_text(json.dumps(value, indent=2) + "\n") + + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for name in ("prometheus", "metrics", "queries", "snapshot", "compiler", "data-plane", "output"): + parser.add_argument("--" + name, type=Path, required=True) + parser.add_argument("--trials", type=int, default=3) + parser.add_argument("--repetitions", type=int, default=20) + parser.add_argument("--cpu-affinity", required=True) + parser.add_argument("--base-port", type=int, default=19410) + args = parser.parse_args() + if args.trials < 1 or args.repetitions < 1: + parser.error("trials and repetitions must be positive") + if args.base_port < 1 or args.base_port + args.trials * 3 - 1 > 65535: + parser.error("trial port range must fit TCP ports") + cpus = {int(value) for value in args.cpu_affinity.split(",")} + if not cpus or not cpus <= os.sched_getaffinity(0): + parser.error("requested CPUs are unavailable") + args.output.mkdir(parents=True, exist_ok=False) + save(args.output / "manifest.json", { + "configuration": {key: str(value) for key, value in vars(args).items()}, + "prometheus_sha256": hashlib.sha256(args.prometheus.read_bytes()).hexdigest(), + "limitations": ["Fresh processes and empty TSDBs; OS caches are not evicted", + "CPU affinity is shared, not an aggregate quota or memory cap", + "Finite-input replay; repeated queries retain original evaluation times"]}) + for trial in range(1, args.trials + 1): + # Separate ports avoid previous trial connections still in TIME_WAIT. + base_port = args.base_port + (trial - 1) * 3 + folder = args.output / f"trial-{trial}" + folder.mkdir() + config = folder / "prometheus.yml" + config.write_text("global:\n scrape_interval: 1h\nscrape_configs: []\n") + children, logs, evidence = {}, [], {} + try: + for index, name in enumerate(("baseline", "fallback")): + port = base_port + index + command = [str(args.prometheus.resolve()), f"--config.file={config.resolve()}", + f"--storage.tsdb.path={(folder / name).resolve()}", + f"--web.listen-address=127.0.0.1:{port}", + "--web.enable-remote-write-receiver", "--storage.tsdb.retention.time=1000000h"] + log = (folder / f"{name}.log").open("w") + logs.append(log) + started = time.perf_counter_ns() + child = subprocess.Popen(command, stdout=log, stderr=subprocess.STDOUT, + preexec_fn=lambda: os.sched_setaffinity(0, cpus)) + children[name] = child + for attempt in range(240): + if child.poll() is not None: + raise RuntimeError(f"{name} exited before readiness") + try: + with urllib.request.urlopen(f"http://127.0.0.1:{port}/-/ready", timeout=1) as response: + if response.status == 200: + break + except OSError: + pass + time.sleep(0.25) + else: + raise RuntimeError(f"{name} readiness timeout") + evidence[name] = {"command": command, "ready": process_snapshot(child.pid), + "startup_wall_ns": time.perf_counter_ns() - started} + command = [sys.executable, str(Path(__file__).with_name("replay.py"))] + for name in ("metrics", "queries", "snapshot", "compiler", "data_plane"): + command += ["--" + name.replace("_", "-"), str(getattr(args, name).resolve())] + command += ["--compare", "--exact-url", f"http://127.0.0.1:{base_port}", + "--fallback-url", f"http://127.0.0.1:{base_port + 1}", + "--exact-pid", str(children["baseline"].pid), + "--fallback-pid", str(children["fallback"].pid), + "--exact-storage", str((folder / "baseline").resolve()), + "--fallback-storage", str((folder / "fallback").resolve()), + "--cpu-affinity", args.cpu_affinity, "--port", str(base_port + 2), + "--repetitions", str(args.repetitions), "--relative-tolerance", "1e-9", + "--absolute-tolerance", "1e-12", "--output", str((folder / "replay").resolve())] + save(folder / "command.json", command) + subprocess.run(command, check=True) + finally: + cleanup_errors = [] + for name, child in children.items(): + try: + evidence.setdefault(name, {})["termination"] = stop(child) + except Exception as error: + # An error collecting one service must not orphan the other. + evidence.setdefault(name, {})["termination_error"] = repr(error) + cleanup_errors.append(f"{name}: {error}") + try: + child.kill() + child.wait(timeout=10) + except Exception as kill_error: + evidence[name]["cleanup_error"] = repr(kill_error) + try: + save(folder / "service-lifecycle.json", evidence) + finally: + for log in logs: + log.close() + if cleanup_errors and sys.exc_info()[0] is None: + raise RuntimeError("Service cleanup failed: " + "; ".join(cleanup_errors)) + print(f"completed trial {trial}: {folder}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/tools/o11y-execution/summarize.py b/tools/o11y-execution/summarize.py new file mode 100644 index 00000000..e4e4fd76 --- /dev/null +++ b/tools/o11y-execution/summarize.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +"""Derive a compact report without modifying immutable replay evidence.""" +import argparse +from collections import Counter +import hashlib +import json +import math +from pathlib import Path + +MODEL = 'measured-inclusive-cpu-ns-v1' +STALE = 'No common conversion from provider cost units to measured resource units' + + +def query_cost_comparison(planning, comparison, queries, run, snapshot=None): + cost = planning['cost_comparison'] + unavailable = {'available': False, 'estimated_over_measured_ratio': None} + def reject(reason): + return dict(unavailable, reason=reason) + if cost.get('model_version') != MODEL: + return reject('Provider model has no verified CPU-nanosecond interpretation') + if cost.get('data_snapshot_id', '').removeprefix('sha256:') not in run.get('inputs', {}).values(): + return reject('Measured input hash does not match the cost data snapshot') + manifest = cost['selected_manifest'] + if manifest['plan_id'] != cost['selected_plan_id']: + return reject('Selected manifest does not match selected plan') + original_by_id = {} + if snapshot is not None: + evidence = snapshot.get('workload_cost_evidence', {}) + if evidence.get('model_version') != MODEL or not any(q.get('manifest') == manifest for q in evidence.get('quotes', [])): + return reject('Snapshot does not contain the selected calibration manifest') + original_by_id = {f'compat-query-{i}': q['query'] for i, q in enumerate(snapshot['query_workload']['repeating_queries'])} + actual = Counter(q['query'] for q in queries) + expected = {} + for qid, query in manifest['workload'].items(): + keys = [k for k in manifest['components'] if k.startswith(f'query:{qid}:') or k == f'result:{qid}'] + counts = {manifest['components'][k]['multiplicity'] for k in keys} + if len(counts) != 1 or any(manifest['components'][k]['unit'] != 'query_evaluation' for k in keys): + return reject('Missing or inconsistent query evaluation multiplicities') + count = counts.pop() + text = original_by_id.get(qid, query['query']) + if text in expected: + return reject('Ambiguous duplicate query in manifest') + expected[text] = count + if actual != expected: + return reject('Measured query multiset differs from priced evaluation demand') + keys = [k for k in manifest['components'] if k.startswith(('query:', 'result:'))] + values = [cost['component_costs'].get(k) for k in keys] + if any(not isinstance(v, (int, float)) or not math.isfinite(v) or v < 0 for v in values): + return reject('Missing or invalid query cost') + measured = comparison['all_requests'].get('backend_plus_fallback_cpu_ns') + if not isinstance(measured, (int, float)) or not math.isfinite(measured) or measured <= 0: + return reject('Query CPU is unavailable or below measurement resolution') + estimated = sum(values) + return {'available': True, 'model_version': MODEL, 'estimated_query_cpu_ns': estimated, + 'measured_query_cpu_ns': measured, 'estimated_over_measured_ratio': estimated / measured, + 'scope': 'Same input hash and query evaluation multiset; backend plus fallback query CPU only. ' + 'Component costs already include multiplicity. Setup, residency and retirement are excluded. ' + 'Calibration and replay cache/background timing can differ; /proc CPU is tick-quantized.'} + + +def summarize(directory): + def read(name): + return json.loads((directory / name).read_text()) + comparison, planning, queries, run = [read(n) for n in ['comparison.json', 'planning.json', 'queries.json', 'run.json']] + snapshot_path = Path(run.get('configuration', {}).get('snapshot', '/missing')) + snapshot = None + if snapshot_path.is_file() and hashlib.sha256(snapshot_path.read_bytes()).hexdigest() == run['inputs'].get(str(snapshot_path.resolve())): + snapshot = json.loads(snapshot_path.read_text()) + aligned = query_cost_comparison(planning, comparison, queries, run, snapshot) + aggregate = {k: v for k, v in comparison['all_requests'].items() if k != 'comparisons'} + limitations = [x for x in comparison['limitations'] if not (aligned['available'] and x == STALE)] + limitations += ['Full lifecycle estimated/measured ratio is unavailable: isolated residency, retirement and service startup scopes are not aligned.', + 'First pass is not a guaranteed cold-cache trial. Sequential service rate is not concurrent throughput.', + 'Summed process lifetime memory peaks are conservative and need not occur simultaneously.'] + return {'schema_version': 1, 'inputs': run['inputs'], 'samples': run['samples'], + 'query_occurrences': run['query_occurrences'], 'aggregate': aggregate, + 'by_phase': {k: {a: b for a, b in v.items() if a != 'comparisons'} for k, v in comparison['by_phase'].items()}, + 'selected_plan_id': planning['cost_comparison']['selected_plan_id'], + 'candidate_costs': planning['cost_comparison']['alternatives'], + 'query_cost_comparison': aligned, 'phase_resources': comparison['phase_resources'], + 'storage': comparison['storage'], 'limitations': limitations, 'acceptance_complete': False, + 'evidence_sha256': {n: hashlib.sha256((directory / n).read_bytes()).hexdigest() for n in + ['comparison.json', 'planning.json', 'queries.json', 'run.json']}} + + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('trial', type=Path) + parser.add_argument('--output', type=Path, required=True) + args = parser.parse_args() + args.output.write_text(json.dumps(summarize(args.trial), indent=2) + '\n') diff --git a/tools/o11y-execution/test_calibrate.py b/tools/o11y-execution/test_calibrate.py new file mode 100644 index 00000000..8ba369cb --- /dev/null +++ b/tools/o11y-execution/test_calibrate.py @@ -0,0 +1,72 @@ +import copy +import unittest +from calibrate import calibrate + + +class CalibrationTests(unittest.TestCase): + def setUp(self): + self.manifest = {"plan_id": 1, "horizon_seconds": 60, + "workload": {"q": {}}, "components": { + "source:x": {"unit": "horizon", "multiplicity": 1}, + "state:x:update": {"unit": "horizon", "multiplicity": 1}, + "query:q:0": {"unit": "query_evaluation", "multiplicity": 6}, + "result:q": {"unit": "query_evaluation", "multiplicity": 6}}} + self.candidates = {"candidates": [{"manifest": self.manifest}]} + self.measurements = {"units": "cpu_ns", "data_snapshot_id": "d", "candidates": [{ + "plan_id": 1, "manifest": copy.deepcopy(self.manifest), "executable": True, + "horizon_seconds": 60, + "horizon_phases": {key: {"cpu_ns": 10, "raw_measurement_file": key + ".json"} + for key in ("install", "ingest_and_build", "residency", "retirement")}, + "queries": {"q": {"cpu_ns": 100, "evaluations": 10, "classification": "warm", + "correct": True, "raw_measurement_file": "q.json"}}}]} + + def run_provider(self): + return calibrate(self.candidates, self.measurements, "d", 1000, 1000) + + def test_inclusive_cpu_is_counted_once(self): + # Horizon and per-evaluation totals preserve measured CPU without double counting. + evidence, audit = self.run_provider() + costs = evidence["quotes"][0]["unit_costs"] + self.assertEqual(sum(costs[k] * v["multiplicity"] for k, v in self.manifest["components"].items()), 100) + self.assertEqual(len(audit["attribution"][0]["allocation"]), 2) + + def test_unknown_candidate_is_not_given_free_quote(self): + # Unmeasured alternatives remain unavailable rather than getting default zeros. + self.measurements["candidates"] = [] + evidence, audit = self.run_provider() + self.assertEqual(evidence["quotes"], []) + self.assertEqual(audit["unavailable"][0]["reason"], "not measured") + + def test_binding_without_valid_execution_is_rejected(self): + # A binding or failed/incorrect response cannot authorize a cost quote. + self.measurements["candidates"][0]["queries"]["q"]["classification"] = "bound" + with self.assertRaises(ValueError): + self.run_provider() + + def test_missing_retirement_is_rejected(self): + # Complete horizon accounting must not silently omit a lifecycle phase. + del self.measurements["candidates"][0]["horizon_phases"]["retirement"] + with self.assertRaises(ValueError): + self.run_provider() + + def test_changed_manifest_cannot_reuse_measurements(self): + # Any change in calibrated candidate identity invalidates the quote. + self.measurements["candidates"][0]["manifest"]["horizon_seconds"] = 30 + with self.assertRaises(ValueError): + self.run_provider() + + def test_shared_profile_preserves_measured_cpu_units(self): + # Inclusive measured phases determine the coarse model; input count only normalizes updates. + from update_global_profile import update + self.measurements["candidates"][0]["resources"] = {"peak_memory_bytes": 123, "storage_bytes": 456} + snapshot = {"implementation": {"horizon_seconds": 60, "implementation_cost": {}}, + "workload_cost_evidence": {"old": True}} + result, audit = update(snapshot, self.measurements, 20) + self.assertEqual(result["implementation"]["lifecycle_costs"]["maintenance_per_update"], 0.5) + self.assertEqual(result["implementation"]["implementation_cost"]["cpu_cost"], 40) + self.assertNotIn("workload_cost_evidence", result) + self.assertIn("not measured zero", " ".join(audit["limitations"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/o11y-execution/test_compare.py b/tools/o11y-execution/test_compare.py index b6d3d4e8..abedef8b 100644 --- a/tools/o11y-execution/test_compare.py +++ b/tools/o11y-execution/test_compare.py @@ -13,6 +13,19 @@ def vector(*values): class ComparisonTests(unittest.TestCase): + def test_explicit_roundoff_tolerance_preserves_errors_and_structure(self): + """Declared floating-point tolerance permits roundoff, never missing series.""" + actual, expected = vector(("a", 1.0 + 1e-15)), vector(("a", 1.0)) + self.assertFalse(compare_results(actual, expected)["equal"]) + result = compare_results(actual, expected, 1e-9, 1e-12) + self.assertTrue(result["equal"]) + self.assertFalse(result["strict_equal"]) + self.assertGreater(result["max_absolute_error"], 0) + self.assertFalse(compare_results(actual, vector(("a", 1.0), ("b", 1.0)), 1e-9, 1e-12)["equal"]) + self.assertFalse(compare_results(vector(("a", 1.1)), expected, 1e-9, 1e-12)["equal"]) + with self.assertRaises(ValueError): + compare_results(actual, expected, float("nan"), 0) + def test_group_matching_is_not_row_position(self): """Permuting a group-by result does not change correctness.""" result = compare_results(vector(("b", 2), ("a", 1)), vector(("a", 1), ("b", 2))) diff --git a/tools/o11y-execution/test_process_lifecycle.py b/tools/o11y-execution/test_process_lifecycle.py new file mode 100644 index 00000000..699cb291 --- /dev/null +++ b/tools/o11y-execution/test_process_lifecycle.py @@ -0,0 +1,38 @@ +import subprocess +import sys +import unittest +from process_lifecycle import stop + + +class LifecycleTests(unittest.TestCase): + def test_lifetime_wait4_not_shutdown_delta(self): + child = subprocess.Popen([sys.executable, '-u', '-c', + 'import time; end=time.process_time()+.05\nwhile time.process_time()