From faedd10cbbb28da2522555a7ce9e3e0e36363898 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 8 Sep 2026 11:36:07 -0600 Subject: [PATCH 1/3] feat(bench): add offline evaluation tools with externally archived results --- Cargo.lock | 2 + crates/devtools/Cargo.toml | 2 + crates/devtools/src/bin/o11y_exact_bench.rs | 367 +++++++++++++ crates/devtools/src/bin/o11y_replay.rs | 495 ++++++++++++++++++ crates/devtools/src/bin/offline_recommend.rs | 28 + .../empirical-o11y-execution-plan.md | 60 +++ docs/offline-o11y-final-2026-09-08.md | 90 ++++ docs/user-guide/o11y-replay.md | 106 ++++ tools/empirical-bench/.gitignore | 3 + tools/empirical-bench/ARTIFACTS.md | 32 ++ tools/empirical-bench/README.md | 113 ++++ tools/empirical-bench/memory_probe.rs | 101 ++++ tools/empirical-bench/resource_probe.rs | 138 +++++ tools/empirical-bench/run.py | 367 +++++++++++++ tools/empirical-bench/test_export.py | 51 ++ 15 files changed, 1955 insertions(+) create mode 100644 crates/devtools/src/bin/o11y_exact_bench.rs create mode 100644 crates/devtools/src/bin/o11y_replay.rs create mode 100644 crates/devtools/src/bin/offline_recommend.rs create mode 100644 docs/design_docs/empirical-o11y-execution-plan.md create mode 100644 docs/offline-o11y-final-2026-09-08.md create mode 100644 docs/user-guide/o11y-replay.md create mode 100644 tools/empirical-bench/.gitignore create mode 100644 tools/empirical-bench/ARTIFACTS.md create mode 100644 tools/empirical-bench/README.md create mode 100644 tools/empirical-bench/memory_probe.rs create mode 100644 tools/empirical-bench/resource_probe.rs create mode 100644 tools/empirical-bench/run.py create mode 100644 tools/empirical-bench/test_export.py diff --git a/Cargo.lock b/Cargo.lock index 34f106da..2fe45ad6 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -320,6 +320,8 @@ dependencies = [ "asap-frontend-promql", "asap-frontend-sql", "asap-types", + "libc", + "regex", "serde", "serde_json", "serde_yaml", diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index 0abc8ccd..748fae18 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -19,4 +19,6 @@ asap-types = { path = "../types" } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" +libc = "0.2" +regex = "1" tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } diff --git a/crates/devtools/src/bin/o11y_exact_bench.rs b/crates/devtools/src/bin/o11y_exact_bench.rs new file mode 100644 index 00000000..e0cd9bbd --- /dev/null +++ b/crates/devtools/src/bin/o11y_exact_bench.rs @@ -0,0 +1,367 @@ +//! Fixed-snapshot exact summary profiling for a deliberately bounded o11y subset. +use std::{hint::black_box, rc::Rc}; + +use asap_aware_mapping::cost_model::Cost; +use asap_aware_mapping::{ + default_strategies_with, search_workload_with_targets, CostModel, DefaultAccuracyModel, + DefaultCostModel, Replacement, ReplacementStrategy, ReplacementSubDAG, SketchAlgorithmStrategy, + TargetSubDAG, +}; +use asap_devtools::lower_promql; +use asap_types::{ + post_asap::SketchAlgorithm, + pre_asap::{AggIntent, QueryExpr}, + types::AccuracyTarget, +}; +use serde_json::{json, Value}; + +const CORPUS: &str = + include_str!("../../../frontend-promql/tests/observability/data/o11y_bench_promql.txt"); + +#[derive(Clone, Copy, Debug)] +enum Kernel { + SumInstant, + MaxWindow, + SumAverageWindow, +} + +struct Fixture { + query: &'static str, + kernel: Kernel, + range_seconds: u64, + job_matcher: Option, +} + +// Admit pinned fixture queries whose full value computation is implemented. +// Regex construction belongs to planning, not execution, on both paths. +fn fixture(query: &'static str) -> Option { + let (kernel, range_seconds, matcher) = match query { + "sum(process_resident_memory_bytes)" | "sum(up)" => (Kernel::SumInstant, 0, None), + "max_over_time(service_cache_refresh_lag_seconds{job=\"user-service\"}[12h])" => { + (Kernel::MaxWindow, 43200, Some("^user-service$")) + } + "max_over_time(service_retry_queue_depth{job=\"order-service\"}[6h])" => { + (Kernel::MaxWindow, 21600, Some("^order-service$")) + } + "max_over_time(service_retry_queue_depth{job=\"payment-service\"}[6h])" => { + (Kernel::MaxWindow, 21600, Some("^payment-service$")) + } + "max_over_time(service_retry_queue_depth{job=~\".+\"}[6h])" => { + (Kernel::MaxWindow, 21600, Some("^.+$")) + } + "sum(avg_over_time(process_resident_memory_bytes{job=~\".+\"}[6h]))" => { + (Kernel::SumAverageWindow, 21600, Some("^.+$")) + } + _ => return None, + }; + Some(Fixture { + query, + kernel, + range_seconds, + job_matcher: matcher.map(|m| regex::Regex::new(m).unwrap()), + }) +} + +struct Series { + job: &'static str, + samples: Vec, +} + +fn data(fixture: &Fixture, series: usize) -> Vec { + let samples = if fixture.range_seconds == 0 { + 1 + } else { + fixture.range_seconds as usize / 60 + }; + (0..series) + .map(|s| Series { + job: ["user-service", "order-service", "payment-service"][s % 3], + samples: (0..samples) + .map(|t| { + if fixture.query == "sum(up)" { + if s % 11 == 0 { + 0.0 + } else { + 1.0 + } + } else { + ((s * 73 + t * 31 + (s * t) % 17) % 10000) as f64 / 100.0 + } + }) + .collect(), + }) + .collect() +} + +fn raw(fixture: &Fixture, data: &[Series]) -> Vec { + let selected = data.iter().filter(|s| { + fixture + .job_matcher + .as_ref() + .is_none_or(|r| r.is_match(s.job)) + }); + match fixture.kernel { + Kernel::SumInstant => vec![selected.map(|s| s.samples[0]).sum()], + Kernel::MaxWindow => selected + .map(|s| s.samples.iter().copied().fold(f64::NEG_INFINITY, f64::max)) + .collect(), + Kernel::SumAverageWindow => vec![selected + .map(|s| s.samples.iter().sum::() / s.samples.len() as f64) + .sum()], + } +} + +#[cfg(unix)] +fn cpu_ns() -> u64 { + let mut ts = libc::timespec { + tv_sec: 0, + tv_nsec: 0, + }; + // A process CPU clock excludes scheduling waits; no wall-time conversion. + assert_eq!( + unsafe { libc::clock_gettime(libc::CLOCK_PROCESS_CPUTIME_ID, &mut ts) }, + 0 + ); + ts.tv_sec as u64 * 1_000_000_000 + ts.tv_nsec as u64 +} + +fn measure(mut op: impl FnMut(), iterations: usize) -> Value { + for _ in 0..2 { + op(); + } + let samples: Vec<_> = (0..5) + .map(|_| { + let start = cpu_ns(); + for _ in 0..iterations { + op(); + } + (cpu_ns() - start) as f64 / iterations as f64 + }) + .collect(); + let mean = samples.iter().sum::() / samples.len() as f64; + let stddev = (samples.iter().map(|x| (x - mean).powi(2)).sum::() + / (samples.len() - 1) as f64) + .sqrt(); + json!({"mean_cpu_ns":mean, "stddev_cpu_ns":stddev, "samples_cpu_ns":samples, "iterations_per_sample":iterations}) +} + +// This reference deployment admits exactly the root it profiled. CPU values +// apply to the complete bounded in-memory kernel, never an interior group. +struct ProfiledModel<'a> { + root: &'a QueryExpr, + approved: Vec>, + raw_cpu: f64, + candidate_cpu: f64, +} + +fn approved_profiles(root: &Rc) -> Vec> { + SketchAlgorithmStrategy::new(&DefaultCostModel) + .replacements(&TargetSubDAG::new(root)) + .into_iter() + .filter_map(|candidate| match candidate.replacement { + Replacement::Summary(node) + if matches!( + node.expr, + asap_types::post_asap::SummaryExpr::SummaryAgg { .. } + ) && node.guarantee.as_ref().is_some_and(|g| g.is_exact()) => + { + Some(node) + } + _ => None, + }) + .collect() +} + +impl CostModel for ProfiledModel<'_> { + fn rank_candidates( + &self, + intent: &AggIntent, + candidates: &[SketchAlgorithm], + ) -> Vec { + DefaultCostModel.rank_candidates(intent, candidates) + } + fn candidate_cost_covers_complete_plan(&self) -> bool { + true + } + fn candidate_cost( + &self, + candidate: &ReplacementSubDAG, + target: &TargetSubDAG<'_>, + ) -> Option { + let Replacement::Summary(node) = &candidate.replacement else { + return None; + }; + if target.root.as_ref() != self.root || !self.approved.contains(node) { + return None; + } + (self.candidate_cpu < self.raw_cpu).then_some(Cost(self.candidate_cpu)) + } + fn estimate_cost(&self, candidate: &ReplacementSubDAG, target: &TargetSubDAG<'_>) -> f64 { + self.candidate_cost(candidate, target) + .map_or(f64::INFINITY, |c| c.0) + } +} + +fn run(fixture: &Fixture, series: usize, evaluations: usize) -> Value { + let root = Rc::new(lower_promql(fixture.query, AccuracyTarget::Exact).unwrap()); + let data = data(fixture, series); + let cached = raw(fixture, &data); + // Same ordered per-series output, or the same scalar; immutable fixed + // windows, no staleness, NaNs, counter resets, or advancing evaluation time. + assert_eq!(raw(fixture, &data), cached.clone()); + let raw_measurement = measure( + || { + black_box(raw(black_box(fixture), black_box(&data))); + }, + 25, + ); + let read_measurement = measure( + || { + black_box(black_box(&cached).clone()); + }, + 10000, + ); + let raw_per_eval = raw_measurement["mean_cpu_ns"].as_f64().unwrap(); + let read_per_eval = read_measurement["mean_cpu_ns"].as_f64().unwrap(); + // Building a fixed-window summary executes the identical raw kernel once. + // Both timers include output allocation and destruction; one-time build + // therefore conservatively includes a destruction absent during retention. + let raw_total = raw_per_eval * evaluations as f64; + let summary_total = raw_per_eval + read_per_eval * evaluations as f64; + let model = ProfiledModel { + root: &root, + approved: approved_profiles(&root), + raw_cpu: raw_total, + candidate_cpu: summary_total, + }; + let candidates = SketchAlgorithmStrategy::new(&model).replacements(&TargetSubDAG::new(&root)); + let accepted = candidates + .iter() + .filter(|c| model.candidate_cost(c, &TargetSubDAG::new(&root)).is_some()) + .count(); + let space = search_workload_with_targets( + vec![(0, root.clone(), Some(AccuracyTarget::Exact))], + &default_strategies_with(&model), + &DefaultAccuracyModel, + ); + let selection = space.global_selection(&model); + let selected_root = &space.roots[0].1; + let selected_plan = selection.materialize(selected_root).unwrap(); + let selected_summary = selected_plan + .as_ref() + .is_some_and(|node| model.approved.contains(node)); + let retained_value_bytes = cached.len() * std::mem::size_of::(); + json!({"query":fixture.query,"status":"profiled_fixed_snapshot", "kernel":format!("{:?}",fixture.kernel), + "profile_model_version":"fixed-snapshot-exact-values-v1", "planner_candidate_count":candidates.len(), + "accepted_measured_candidates":accepted, + "selected":if selected_summary {"retained_exact_summary"} else {"raw_recompute"}, + "selected_planner_graph":selected_plan.as_ref().map(|node|asap_types::dag_export::export_summary(node)), + "input_series":series,"selected_series":data.iter().filter(|s|fixture.job_matcher.as_ref().is_none_or(|r|r.is_match(s.job))).count(), + "samples_per_series":data[0].samples.len(),"sample_interval_seconds":60,"range_seconds":fixture.range_seconds, + "input_value_bytes":data.iter().map(|s|s.samples.len()*8).sum::(), + "retained_value_bytes":retained_value_bytes,"retained_value_bytes_method":"exact Vec length times sizeof(f64), logical values only; label/input storage remains shared", + "raw_per_evaluation":raw_measurement,"summary_read_per_evaluation":read_measurement, + "evaluations":evaluations,"raw_cpu_ns":raw_total,"summary_cpu_ns":summary_total, + "estimated_cpu_reduction_fraction":if selected_summary {Some(1.0-summary_total/raw_total)} else {None}, + "output_equality_verified":true, + "comparison_scope":"same immutable metric snapshot and fixed evaluation time; retained whole-query exact result reference implementation including filtering, reduction, allocation and readout", + "exclusions":["disk/network storage and protocol serialization", "label output materialization shared by both paths", "live updates, eviction, sliding windows, staleness and exceptional samples"]}) +} + +fn main() -> Result<(), Box> { + let args: Vec<_> = std::env::args().skip(1).collect(); + let series: usize = args.first().map_or(Ok(300), |x| x.parse())?; + let evaluations: usize = args.get(1).map_or(Ok(60), |x| x.parse())?; + if args.len() > 2 || series == 0 || series > 10000 || evaluations == 0 { + return Err("usage: o11y_exact_bench [SERIES(1..10000)] [EVALUATIONS>0]".into()); + } + let rows: Vec<_> = CORPUS.lines().map(str::trim).filter(|q|!q.is_empty()&&!q.starts_with('#')).map(|q| + fixture(q).map_or_else(||json!({"query":q,"status":"unavailable","reason":"no complete reference kernel for this query; no borrowed costs"}), |f|run(&f,series,evaluations))).collect(); + let cpu = std::fs::read_to_string("/proc/cpuinfo") + .ok() + .and_then(|text| { + text.lines() + .find(|line| line.starts_with("model name")) + .map(str::to_owned) + }); + let revision = std::process::Command::new("git") + .args(["rev-parse", "HEAD"]) + .output() + .ok() + .filter(|out| out.status.success()) + .map(|out| String::from_utf8_lossy(&out.stdout).trim().to_owned()); + serde_json::to_writer_pretty( + std::io::stdout(), + &json!({"schema_version":1, + "data":"synthetic deterministic finite gauges; three jobs; non-stale finite float samples on (T-window,T] every60s", + "implementation":"o11y_exact_bench Rust reference kernels; not production data-plane runtime", + "build_profile":if cfg!(debug_assertions){"debug"}else{"release"}, + "source_revision":revision,"source_file":"crates/devtools/src/bin/o11y_exact_bench.rs", + "command_arguments":args,"cpu":cpu,"os":std::env::consts::OS,"arch":std::env::consts::ARCH, + "timing_environment":"shared development host; no exclusive core reservation or CPU affinity", + "clock":"CLOCK_PROCESS_CPUTIME_ID","rows":rows}), + )?; + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + #[test] + fn admission_is_explicit_and_unknown_semantics_stay_unavailable() { + assert_eq!(CORPUS.lines().filter(|q| fixture(q).is_some()).count(), 7); + assert!(fixture("sum(rate(http_requests_total[5m]))").is_none()); + } + #[test] + fn kernels_match_independent_small_reference_results() { + let input = vec![ + Series { + job: "order-service", + samples: vec![2.0, 4.0], + }, + Series { + job: "payment-service", + samples: vec![8.0, 10.0], + }, + ]; + let max = + fixture("max_over_time(service_retry_queue_depth{job=\"order-service\"}[6h])").unwrap(); + assert_eq!(raw(&max, &input), vec![4.0]); + let avg = + fixture("sum(avg_over_time(process_resident_memory_bytes{job=~\".+\"}[6h]))").unwrap(); + assert_eq!(raw(&avg, &input), vec![12.0]); + } + #[test] + fn measured_scope_and_break_even_control_candidate_acceptance() { + let root = Rc::new(lower_promql("sum(up)", AccuracyTarget::Exact).unwrap()); + let target = TargetSubDAG::new(&root); + let candidates = SketchAlgorithmStrategy::new(&DefaultCostModel).replacements(&target); + assert!(!candidates.is_empty()); + let slow = ProfiledModel { + root: &root, + approved: approved_profiles(&root), + raw_cpu: 100.0, + candidate_cpu: 101.0, + }; + let fast = ProfiledModel { + root: &root, + approved: approved_profiles(&root), + raw_cpu: 100.0, + candidate_cpu: 90.0, + }; + assert!(slow.candidate_cost(&candidates[0], &target).is_none()); + assert_eq!( + fast.candidate_cost(&candidates[0], &target), + Some(Cost(90.0)) + ); + let other = Rc::new(lower_promql("sum(other)", AccuracyTarget::Exact).unwrap()); + assert!(fast + .candidate_cost(&candidates[0], &TargetSubDAG::new(&other)) + .is_none()); + let mut unprofiled = candidates[0].clone(); + let mut node = fast.approved[0].as_ref().clone(); + node.expr = asap_types::post_asap::SummaryExpr::KeepPreAsap(root.clone()); + unprofiled.replacement = Replacement::Summary(Rc::new(node)); + assert!(fast.candidate_cost(&unprofiled, &target).is_none()); + } +} diff --git a/crates/devtools/src/bin/o11y_replay.rs b/crates/devtools/src/bin/o11y_replay.rs new file mode 100644 index 00000000..579f586d --- /dev/null +++ b/crates/devtools/src/bin/o11y_replay.rs @@ -0,0 +1,495 @@ +//! Offline workload replay through search, selection, and lifecycle planning. +use std::{collections::HashSet, error::Error, rc::Rc, time::Instant}; + +use asap_aware_mapping::empirical_cost::{ + EmpiricalCostModel, EmpiricalEvidenceProvider, EvidenceArtifact, EvidenceContext, +}; +use asap_aware_mapping::{ + default_strategies_with, export_summary_maintenance_plan, + materialize_with_summary_maintenance_lifecycles, search_workload_with_targets, CostModel, + DefaultAccuracyModel, DefaultCostModel, Horizon, Replacement, + SummaryMaintenanceLifecycleCapabilities, WorkloadDemand, +}; +use asap_devtools::lower_promql; +use asap_types::{ + dag_export, + post_asap::{SummaryExpr, SummaryFamilyType, SummaryNode}, + pre_asap::QueryExpr, + types::AccuracyTarget, + workload::*, +}; +use serde_json::{json, Value}; + +const CORPUS: &str = + include_str!("../../../frontend-promql/tests/observability/data/o11y_bench_promql.txt"); +const SUPPLEMENTAL: &str = "quantile_over_time(0.95, service_latency_seconds[5m])\nquantile_over_time(0.99, service_latency_seconds[5m])\ncount_over_time(offline_frequency_metric[5m])"; + +struct Options { + epsilon: f64, + evaluations: u64, + now_ms: u64, + supplemental: bool, + queries: Option, + evidence_path: Option, + context_path: Option, +} + +impl Default for Options { + fn default() -> Self { + Self { + epsilon: 0.01, + evaluations: 60, + now_ms: 1_788_825_600_000, + supplemental: false, + queries: None, + evidence_path: None, + context_path: None, + } + } +} + +fn parse_options(args: impl IntoIterator) -> Result> { + let mut options = Options::default(); + let mut args = args.into_iter(); + while let Some(arg) = args.next() { + match arg.as_str() { + "--supplemental" => options.supplemental = true, + "--epsilon" => options.epsilon = args.next().ok_or("missing epsilon")?.parse()?, + "--evaluations" => options.evaluations = args.next().ok_or("missing evaluations")?.parse()?, + "--now-ms" => options.now_ms = args.next().ok_or("missing now-ms")?.parse()?, + "--queries" => options.queries = Some(std::fs::read_to_string(args.next().ok_or("missing queries path")?)?), + "--evidence" => options.evidence_path = Some(args.next().ok_or("missing evidence path")?), + "--context" => options.context_path = Some(args.next().ok_or("missing context path")?), + _ => return Err(format!("unknown option {arg}; use --epsilon, --evaluations, --now-ms, --queries, --supplemental, --evidence and --context").into()), + } + } + if !options.epsilon.is_finite() + || !(0.0..1.0).contains(&options.epsilon) + || options.epsilon == 0.0 + { + return Err("epsilon must be finite and strictly between 0 and 1".into()); + } + if options.evaluations == 0 { + return Err("evaluations must be positive".into()); + } + if options.evidence_path.is_some() != options.context_path.is_some() { + return Err("--evidence and --context must be supplied together".into()); + } + Ok(options) +} + +fn queries(text: &str) -> Vec { + text.lines() + .map(str::trim) + .filter(|q| !q.is_empty() && !q.starts_with('#')) + .map(str::to_owned) + .collect() +} + +// Attribute memo groups by DAG identity after workload-wide CSE. Scalar +// expressions aren't replacement sites; they remain part of their operator. +fn reachable(node: &QueryExpr, found: &mut HashSet<*const QueryExpr>) { + if !found.insert(node as *const QueryExpr) { + return; + } + use QueryExpr::*; + match node { + PromqlVectorFromScalar(child) + | PromqlScalarFromVector(child) + | PromqlRelabel { child, .. } + | PromqlInfoEnrich { child, .. } + | PromqlSeriesSample { child, .. } + | Filter { child, .. } + | Project { child, .. } + | Aggregate { child, .. } + | Dedup { child, .. } + | PromqlSubquery { child, .. } + | TimeRange { child, .. } + | TimeShift { child, .. } + | SQLWindowFunc { child, .. } + | Sort { child, .. } + | Limit { child, .. } => reachable(child, found), + Concat { children, .. } => { + for child in children { + reachable(child, found); + } + } + Join { left, right, .. } | SetOp { left, right, .. } => { + reachable(left, found); + reachable(right, found); + } + BinaryOp { lhs, rhs, .. } => { + reachable(lhs, found); + reachable(rhs, found); + } + _ => {} + } +} + +fn family_report( + family: &SummaryFamilyType, + provider: Option<&EmpiricalEvidenceProvider>, +) -> Value { + let evidence = if let SummaryFamilyType::Sketch(kind, _) = family { + match provider { + Some(provider) => match provider.lookup(kind.algorithm(), kind.params()) { + Ok(row) => json!({"status": "matched_configuration", "measurement": row, + "scope": "offline benchmark primitive; error/read query semantics are recorded in measurement.error.query, not validated against replay query"}), + Err(error) => json!({"status": "unavailable", "reason": error.to_string()}), + }, + None => { + json!({"status": "unavailable", "reason": "no empirical provider in this mode"}) + } + } + } else { + json!({"status": "unavailable", "reason": "offline sketch evidence does not cost exact operators"}) + }; + let sketch = match family { + SummaryFamilyType::Sketch(kind, _) => { + json!({"algorithm": kind.algorithm(), "params": kind.params()}) + } + _ => Value::Null, + }; + json!({"family": format!("{family:?}"), "sketch": sketch, + "is_sketch": matches!(family, SummaryFamilyType::Sketch(..)), "offline_evidence": evidence}) +} + +fn families( + node: &SummaryNode, + result: &mut Vec, + provider: Option<&EmpiricalEvidenceProvider>, +) { + use SummaryExpr::*; + match &node.expr { + SummaryAgg { family, child, .. } => { + result.push(family_report(family, provider)); + families(child, result, provider); + } + SummaryJoin { + family, + outer, + inner, + .. + } => { + result.push(family_report(family, provider)); + families(outer, result, provider); + families(inner, result, provider); + } + SummaryEstimate { summary_input, .. } | SummaryDelete { summary_input, .. } => { + families(summary_input, result, provider) + } + BinaryOp { lhs, rhs, .. } => { + families(lhs, result, provider); + families(rhs, result, provider); + } + SummarySubtract { left, right } => { + families(left, result, provider); + families(right, result, provider); + } + SummaryMerge { children } => { + for child in children { + families(child, result, provider); + } + } + KeepPreAsap(_) => {} + } +} + +fn replay( + name: &str, + query_texts: &[String], + accuracy: AccuracyTarget, + options: &Options, + model: &dyn CostModel, + provider: Option<&EmpiricalEvidenceProvider>, +) -> Value { + let lowering_start = Instant::now(); + let mut roots = Vec::new(); + let mut rows = Vec::new(); + for (index, query) in query_texts.iter().enumerate() { + match lower_promql(query, accuracy.clone()) { + Ok(root) => roots.push((index, Rc::new(root), Some(accuracy.clone()))), + Err(error) => rows.push(json!({"index": index, "query": query, "coverage": "rejected", "reason": error.to_string()})), + } + } + let lowering_ns = lowering_start.elapsed().as_nanos(); + let search_start = Instant::now(); + let space = search_workload_with_targets( + roots, + &default_strategies_with(model), + &DefaultAccuracyModel, + ); + let search_ns = search_start.elapsed().as_nanos(); + let selection_start = Instant::now(); + let ranked = space.cost_sorted(model); + let selection = space.global_selection(model); + let selection_ns = selection_start.elapsed().as_nanos(); + let workload = QueryWorkload { + language: QueryLanguage::PromQL, + query_batch: Some( + query_texts + .iter() + .map(|q| BatchEntry { + query: Query(q.clone()), + requirements: QueryRequirements { + accuracy: AccuracyRequirement::Explicit(accuracy.clone()), + ..Default::default() + }, + predictability: Predictability::AdHoc, + invocations: options.evaluations, + execute_at: Some(TimestampMs(options.now_ms)), + time_selection: TimeSelection { + as_of: Some(TimestampMs(options.now_ms)), + ..Default::default() + }, + }) + .collect(), + ), + repeating_queries: None, + data_workload: Some(DataWorkload { + arrival: DataArrival::AtRest, + ..Default::default() + }), + }; + for (index, root) in &space.roots { + let started = Instant::now(); + let mut found = HashSet::new(); + reachable(root, &mut found); + let mut candidates = Vec::new(); + let mut rejected = Vec::new(); + for (group_index, group) in ranked + .iter() + .enumerate() + .filter(|(_, g)| found.contains(&Rc::as_ptr(g.target))) + { + let target_intent = asap_aware_mapping::replacement::bindable_intent(group.target) + .map(|intent| format!("{intent:?}")); + for (rank, candidate) in group.candidates.iter().enumerate() { + let mut family_list = Vec::new(); + if let Replacement::Summary(node) = &candidate.replacement { + families(node, &mut family_list, provider); + } + candidates.push( + json!({"group": group_index, "rank": rank, "strategy": candidate.strategy, + "target_intent": target_intent, + "rationale": candidate.rationale, "families": family_list, + "heuristic_score": group.costs[rank].is_finite().then_some(group.costs[rank]), + "score_unit": "dimensionless_not_cpu", "consumer_count": group.consumer_count}), + ); + } + if let Some(memo) = space.group_for(group.target) { + rejected.extend(memo.rejected.iter().map(|r| json!({"group": group_index, "strategy": r.strategy, "description": r.description, "reason": r.error.to_string()}))); + } + } + let has_sketch = candidates.iter().any(|c| { + c["families"] + .as_array() + .unwrap() + .iter() + .any(|f| f["is_sketch"] == true) + }); + let has_summary = candidates + .iter() + .any(|c| !c["families"].as_array().unwrap().is_empty()); + let materialized = selection.materialize(root); + let root_plan = match materialized { + Ok(Some(node)) => { + json!({"graph": dag_export::export_summary(&node), "guarantee": node.guarantee}) + } + Ok(None) => json!({"reason": "no selected root group"}), + Err(error) => json!({"reason": error.to_string()}), + }; + let lifecycle = match materialize_with_summary_maintenance_lifecycles( + &selection, + root, + WorkloadDemand::new(&workload, &[*index]), + options.now_ms, + Some(Horizon(3600.0)), + SummaryMaintenanceLifecycleCapabilities::ALL, + model, + ) { + Ok(Some(plan)) => json!(export_summary_maintenance_plan(&plan)), + Ok(None) => json!({"reason": "no selected root group", "selected_raw_recompute": true}), + Err(error) => json!({"reason": error.to_string(), "selected_raw_recompute": true}), + }; + rows.push(json!({"index": index, "query": query_texts[*index], + "coverage": if has_sketch { "sketch_candidate" } else if has_summary { "exact_summary_candidate" } else { "exact_fallback" }, + "coverage_scope": "reachable_candidate_sites_not_whole_query_execution", + "candidates": candidates, "rejections": rejected, "root_plan": root_plan, "lifecycle_plan": lifecycle, + "report_and_materialization_ns": started.elapsed().as_nanos(), + "estimated_end_to_end_cpu_savings": null, "estimated_end_to_end_memory_savings": null, + "unknown_cost_reason": "complete raw scan, residual operators, grouping cardinalities and deployment costs are unavailable"})); + } + rows.sort_by_key(|row| row["index"].as_u64()); + let mut coverage_counts = std::collections::BTreeMap::new(); + for row in &rows { + *coverage_counts + .entry(row["coverage"].as_str().unwrap_or("unknown")) + .or_insert(0) += 1; + } + let raw_fallback_count = rows + .iter() + .filter(|row| row["lifecycle_plan"]["selected_raw_recompute"] == true) + .count(); + json!({"mode": name, "accuracy": accuracy, "lowering_ns": lowering_ns, "search_ns": search_ns, + "selection_ns": selection_ns, "query_count": query_texts.len(), "memo_groups": space.len(), + "coverage_counts": coverage_counts, "lifecycle_raw_fallback_count": raw_fallback_count, "queries": rows}) +} + +fn report(options: &Options, empirical: Option<&EmpiricalCostModel>) -> Value { + let mut corpora = vec![( + if options.queries.is_some() { + "custom" + } else { + "o11y_bench" + }, + queries(options.queries.as_deref().unwrap_or(CORPUS)), + )]; + if options.supplemental { + corpora.push(("supplemental_sketch_queries", queries(SUPPLEMENTAL))); + } + let corpora: Vec = corpora + .into_iter() + .map(|(name, queries)| { + let mut runs = vec![ + replay( + "exact", + &queries, + AccuracyTarget::Exact, + options, + &DefaultCostModel, + None, + ), + replay( + "default", + &queries, + AccuracyTarget::Epsilon(options.epsilon), + options, + &DefaultCostModel, + None, + ), + ]; + if let Some(model) = empirical { + runs.push(replay( + "empirical", + &queries, + AccuracyTarget::Epsilon(options.epsilon), + options, + model, + Some(&model.provider), + )); + } + json!({"name": name, "source": match name { + "o11y_bench" => "repository-vendored grafana/o11y-bench snapshot", + "custom" => "user-supplied PromQL query file", + _ => "local supplemental examples, not upstream o11y queries" + }, "runs": runs}) + }) + .collect(); + json!({"schema_version": 1, "execution_scope": "offline_planner_search_selection_and_lifecycle_no_data_plane", + "empirical_context": empirical.map(|m| m.provider.context()), + "empirical_artifact": empirical.map(|m| json!({"schema_version": m.provider.artifact().schema_version, + "benchmark_version": m.provider.artifact().benchmark_version, "model_version": m.provider.artifact().model_version})), + "vendored_fixture_source": {"repository": "https://github.com/grafana/o11y-bench", "vendored_snapshot_date": "2026-07-17", "upstream_commit": null, + "note": "existing 27-query local fixture; upstream revision and scenario data were not provided"}, + "assumptions": {"evaluation_time_ms": options.now_ms, "evaluations_per_query": options.evaluations, + "data_arrival": "at_rest", "replay_semantics": "repeated reads at one fixed as-of time; query windows and offsets remain in IR", + "lifecycle_horizon_seconds": 3600, "runtime_capabilities": "hypothetical all supported, no runtime launched", + "timing": "single wall-clock sample; report/materialization timing includes JSON export"}, + "corpora": corpora}) +} + +fn main() -> Result<(), Box> { + let options = parse_options(std::env::args().skip(1))?; + let empirical = match (&options.evidence_path, &options.context_path) { + (Some(evidence), Some(context)) => { + let artifact: EvidenceArtifact = + serde_json::from_str(&std::fs::read_to_string(evidence)?)?; + let context: EvidenceContext = + serde_json::from_str(&std::fs::read_to_string(context)?)?; + Some(EmpiricalCostModel::new(EmpiricalEvidenceProvider::new( + artifact, context, + )?)) + } + _ => None, + }; + println!( + "{}", + serde_json::to_string_pretty(&report(&options, empirical.as_ref()))? + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Invalid CLI numbers must fail before they can poison cost estimates. + #[test] + fn rejects_invalid_configuration() { + for value in ["NaN", "0", "1", "-1"] { + assert!(parse_options(["--epsilon".into(), value.into()]).is_err()); + } + assert!(parse_options(["--evaluations".into(), "0".into()]).is_err()); + } + + /// Real workload lowering alone never counts as sketch or deployment coverage. + #[test] + fn o11y_replay_records_all_queries_and_conservative_costs() { + let result = report(&Options::default(), None); + for run in result["corpora"][0]["runs"].as_array().unwrap() { + assert_eq!(run["query_count"], 27); + assert!(run["memo_groups"].as_u64().unwrap() > 0); + for row in run["queries"].as_array().unwrap() { + assert_ne!(row["coverage"], "rejected"); + assert_ne!(row["coverage"], "sketch_candidate"); + assert!(row["estimated_end_to_end_cpu_savings"].is_null()); + assert_eq!(row["lifecycle_plan"]["selected_raw_recompute"], true); + } + } + } + + /// Supplemental sketches and rejected syntax remain separately identifiable. + #[test] + fn sketches_and_rejections_are_explicit() { + let qs = vec!["quantile_over_time(0.95, latency[5m])".into(), "!!!".into()]; + let result = replay( + "default", + &qs, + AccuracyTarget::Epsilon(0.01), + &Options::default(), + &DefaultCostModel, + None, + ); + assert_eq!(result["queries"][0]["coverage"], "sketch_candidate"); + assert_eq!(result["queries"][1]["coverage"], "rejected"); + assert!(result["queries"][1]["reason"].is_string()); + } + + /// Count-over-time exercises the measured CMS/CountSketch configuration, + /// while its evidence remains absent unless an artifact is supplied. + #[test] + fn supplemental_count_exposes_frequency_sketch_candidates() { + let result = replay( + "default", + &queries("count_over_time(metric[5m])"), + AccuracyTarget::Epsilon(0.01), + &Options::default(), + &DefaultCostModel, + None, + ); + let families: Vec<_> = result["queries"][0]["candidates"] + .as_array() + .unwrap() + .iter() + .flat_map(|c| c["families"].as_array().unwrap()) + .collect(); + for algorithm in ["Cms", "CountSketch"] { + let family = families + .iter() + .find(|f| f["sketch"]["algorithm"] == algorithm) + .unwrap(); + assert_eq!(family["offline_evidence"]["status"], "unavailable"); + } + } +} diff --git a/crates/devtools/src/bin/offline_recommend.rs b/crates/devtools/src/bin/offline_recommend.rs new file mode 100644 index 00000000..3f6333f0 --- /dev/null +++ b/crates/devtools/src/bin/offline_recommend.rs @@ -0,0 +1,28 @@ +//! Evaluate an explicit offline error/resource requirement against measured data. +use asap_aware_mapping::empirical_comparison::{ + recommend_offline, OfflineComparisonEvidence, OfflineComparisonRequest, +}; + +fn main() -> Result<(), Box> { + let args: Vec<_> = std::env::args().skip(1).collect(); + if args.len() != 2 { + return Err("usage: offline_recommend COMPARISON-EVIDENCE.json REQUEST.json".into()); + } + let evidence: OfflineComparisonEvidence = + serde_json::from_str(&std::fs::read_to_string(&args[0])?)?; + let request: OfflineComparisonRequest = + serde_json::from_str(&std::fs::read_to_string(&args[1])?)?; + let recommendation = recommend_offline(&evidence, &request)?; + serde_json::to_writer_pretty( + std::io::stdout(), + &serde_json::json!({ + "schema_version":1, + "benchmark_version":evidence.sketch_evidence.benchmark_version, + "model_version":evidence.sketch_evidence.model_version, + "request":request, + "recommendation":recommendation, + "accuracy_scope":"observed offline mean error on the exact declared probe population; not an unseen-data or realtime guarantee" + }), + )?; + Ok(()) +} diff --git a/docs/design_docs/empirical-o11y-execution-plan.md b/docs/design_docs/empirical-o11y-execution-plan.md new file mode 100644 index 00000000..0471529e --- /dev/null +++ b/docs/design_docs/empirical-o11y-execution-plan.md @@ -0,0 +1,60 @@ +# Offline evidence and o11y planning evaluation + +Audience: developers reproducing issue #322 and the planner/control-plane evaluation. + +## Scope + +Use offline sketch-bench CPU, elapsed time, state/memory, disk (when measured), +and errors against offline ground truth. No runtime ground truth, posterior +feedback, or self-estimated accuracy is required. Offline accuracy observations +do not establish formal guarantees on unseen distributions. + +## Execution and ownership + +1. Pin isolated planner and control-plane worktrees from fetched main branches. + Preserve existing worktrees, including unresolved cost-model changes. +2. Evidence agent: implement the versioned artifact, validation, compatibility + matching, public cost-model integration, and fallback/decision tests. +3. Benchmark agent: run actual sketch-bench algorithms on uniform and Zipf + inputs, preserve raw output, export artifacts with source and environment + provenance, and document reproducible commands. +4. Replay agent: run the existing o11y PromQL corpus through actual planning, + export candidate coverage and planning latency, and separately identify + supplemental sketch workloads. +5. Integration owner: connect compatible evidence to the downstream control + plane, validate dependency compatibility, review other agents' changes, run + integration checks, and report evidence-supported comparisons. + +Steps 2–4 run concurrently after agreeing the artifact contract. Integration +uses their completed interfaces and measurements. Query-pattern changes are +limited to demonstrated blockers with regression coverage; unsupported query +semantics remain explicit in the report. + +## Acceptance + +- Versioned schema and example; explicit units, algorithm/configuration, + dataset/distribution, environment, collection/validity times, sample count, + dispersion, and benchmark/model provenance. +- At least two actual sketch algorithms measured offline on uniform and skewed + inputs. CPU is distinct from elapsed time; serialization size is distinct + from heap memory and disk I/O. Unmeasured fields stay unavailable. +- Matching evidence affects a public planning cost/ranking/lifecycle boundary. + Tests cover changed decisions, missing/stale/mismatched evidence, invalid + values, and unchanged accuracy guarantees. +- The planner and control-plane evaluation preserve exact fallbacks and expose + unsupported shapes, measured provenance, and limits on benefit estimates. +- Reproducible measurement and replay commands, raw machine-readable results, + and a concise report distinguish actual measurements from modeled totals. + +## Comparison rules + +Compare equivalent tasks, data, parameters, windows, horizons, and evaluation +cadences. Whole-plan estimates include build/update/readout, retained windows, +sharing, and raw residual work where evidence exists. Missing raw baseline or +physical evidence means an unavailable whole-plan speedup, not zero cost. +Microbenchmark algorithm comparisons are labeled separately. Disk usage and +network savings require their own measurements or explicit models. + +Repeated-query break-even can be computed only when compatible exact baseline, +sketch build/maintenance, and readout measurements are present. It is an offline +estimate and does not establish deployed end-to-end latency improvement. diff --git a/docs/offline-o11y-final-2026-09-08.md b/docs/offline-o11y-final-2026-09-08.md new file mode 100644 index 00000000..25f1b9f4 --- /dev/null +++ b/docs/offline-o11y-final-2026-09-08.md @@ -0,0 +1,90 @@ +# Offline Sketch Evidence and o11y: Final Report + +Audience: developers. This report covers the offline evidence path for #322, +without real-time error feedback. The original dirty working directory was left unchanged. + +## Completed Integration + +- Versioned artifacts, JSON Schema, and a public CostModel provider match parameters, query semantics, distributions, environments, and validity intervals. Unknown measurements remain unavailable. +- Actual CMS/CountSketch parameter sweeps measure offline error, separate construction/update/read/merge CPU, live heap allocations, serialized size, and allocated file blocks. +- Fixed-snapshot point-frequency comparisons use explicit observed-error thresholds, formal parameter minima, CPU/retained byte-seconds weights, and an exact baseline on the same data. +- The backend filters unsupported layouts before recommending and binding frequency configurations. When exact execution wins or evidence is insufficient, it preserves the original query without silently rounding selected parameters. +- o11y planner/control-plane replay and exact fixed-snapshot reference measurements cover seven explicitly supported queries. + +For interfaces and reproduction instructions, see the [evidence contract](developer_docs/offline-sketch-evidence.md), +[benchmark commands](../tools/empirical-bench/README.md), and [replay guide](user-guide/o11y-replay.md). + +## Measured Results + +sketch-bench is pinned to revision `87f619e843fd2e4da784160d4e205a0d0d55f032`. +The uniform and Zipf datasets each contain 20,000 i64 keys, with a key space of +1,000, seed 42, and Zipf exponent 1.1. There are 18 sketch configurations and +two exact baselines; CPU measurements use five trials after two warmups. +Accuracy comes from one offline experiment per fixed dataset, not a guarantee +across distributions or real-time ground truth. The comparison assumes one +build, 1,000 queries, 300 seconds of retention, and no merges. + +| Scenario | Uniform / Zipf results | +| --- | --- | +| CPU-only, with a 1% or 5% observed mean relative-error limit | Both select exact execution; no sketch CPU benefit | +| Generic comparator, with memory weighting | CMS 2720×5; retained heap decreases by 81.68%, while CPU increases by approximately 0.037 / 0.182 ms, respectively | +| Backend-compatible configuration, with memory weighting | CMS 4096×5; retained heap decreases by approximately 72.4%, while CPU also increases | + +The memory-weighted objective is `CPU ns + 0.01 × retained byte-seconds`. These +weights express an illustrative preference, not a conversion measured from +hardware. Exact retained heap is 296,976 B; CMS 2720×5 uses 54,400 B, and +4096×5 uses 81,920 B. Memory figures measure requested allocation bytes using +a separate System allocator probe; CPU measurements use jemalloc. These memory +figures are not process RSS. Serialized size and allocated file blocks do not +establish disk-throughput benefits. + +See `results-sweep/MEASUREMENTS.md` in the [verified experiment archive](../tools/empirical-bench/ARTIFACTS.md). +Earlier frequency measurements and control-plane replay in `results/` remain +as the initial baseline. Final frequency comparisons and control-plane results +use `results-sweep/`; costs from the two runs must not be mixed. + +## o11y Coverage and Limitations + +The replay reuses the repository's existing 27 PromQL fixtures. It does not run +upstream LLM-agent scoring or use real scenario data. + +| Check | Result | +| --- | --- | +| Planner candidate coverage | 26/27 have exact summary candidates; 1/27 uses raw fallback; no sketch candidates | +| Complete lifecycle costs in the original replay | Insufficient evidence; all 27/27 conservatively fall back to raw execution | +| Control-plane exact/default/empirical binding | All three modes bind 27/27 successfully; each retains the original query in 5 root plans | +| Seven exact reference queries | Raw execution and cached-result reads are measured on fixed synthetic gauge snapshots, using the public CostModel and actual selection flow | +| Remaining 20 queries | No complete reference implementation; benefits remain unavailable | + +New fallbacks preserve the original IR for selector, sort, and comparison/filter +roots, increasing successful binding from 24/27 to 27/27. Binding does not mean +every plan is executable by the summary executor. Column updates in the new IR +are supported; keyed/non-column updates and warm-tier BinaryOp remain explicitly rejected. + +The seven reference queries cover instant sums, window maxima, and sums of +window averages: 300 series, three jobs, one finite gauge sample per minute, +and 60 repeated reads of the same snapshot. The retained state is the complete +exact query result, not a sliding window with advancing time. Measurements +exclude network, disk, protocol serialization, and output-label materialization; +memory figures count logical value bytes. Benefits from repeated reads cannot +be extrapolated to production end-to-end speedups. Machine-readable results are +available as `results/o11y-exact-snapshot.json` in that archive. + +Quantile measurements, real o11y traces, time drift, post-merge error, online +update and retirement costs, and complete production benefits remain outside +this coverage. Point-frequency error cannot be applied to quantiles or count_over_time. + +## Validation and Delivery + +- Planner mapping: 349 unit tests pass. +- Devtools: 3 exact benchmark tests and 4 replay tests pass; the recommendation CLI is verified with six actual requests. +- Benchmark export: 5 Python tests pass. +- Backend: 633 control-plane and 973 data-plane library tests pass; 7 new integration tests pass. +- Agents cross-reviewed the provider, measurement, and binding code; artifact text, JSON, source hashes, and provenance were checked successfully. + +The backend pins the published planner core commit +`dfdf6b5c1f7d667394a4ea1f56fb3786af04228d`. Normal builds and final replay do not +require local Cargo source patches. Core planner code, benchmark tools, and backend +integration are reviewed in separate PRs. Full generated results are distributed +as an experiment artifact, not embedded in the source diff. No PR is automatically +merged and the issue is not automatically closed. diff --git a/docs/user-guide/o11y-replay.md b/docs/user-guide/o11y-replay.md new file mode 100644 index 00000000..1a3692b3 --- /dev/null +++ b/docs/user-guide/o11y-replay.md @@ -0,0 +1,106 @@ +# Offline o11y workload replay + +For users evaluating planner coverage using offline sketch-bench evidence. +`o11y_replay` runs the existing 27-query o11y fixture through lowering, +workload-wide candidate search, accuracy checks, selection and summary lifecycle +planning. It launches no data plane or downstream control-plane server. + +```sh +cargo run -p asap-devtools --bin o11y_replay -- --supplemental > replay.json +cargo run -p asap-devtools --bin o11y_replay -- \ + --supplemental --evidence planner-evidence.json --context context.json > empirical-replay.json +``` + +The first command compares exact requirements with the default approximate +planner (`--epsilon 0.01`). The second adds an empirical planning run using a +versioned offline evidence artifact and an explicit applicability context. +The context JSON contains `distribution`, `environment`, and +`now_unix_seconds`; copy the full descriptors of the intended offline dataset +and execution environment, and choose the evidence evaluation time explicitly. +Configuration, distribution, environment and validity dates must match. +An absent measurement stays unavailable and falls back to default planning. + +`--queries FILE` accepts one PromQL query per nonblank, noncomment line. +`--evaluations 60` models repeated reads of the same fixed historical input; +`--now-ms 1788825600000` fixes the as-of time. These are scenario assumptions, +not timings extracted from o11y task execution. Query windows, subqueries and +offsets remain encoded in the query IR. The lifecycle horizon is one hour and +the assumed data is at rest. `--supplemental` adds two quantile queries and a +`count_over_time` query as a separate corpus. Its total sample count has different +readout semantics from integer-stream point-frequency sketch-bench data; matching +sketch configuration does not establish matching query readout/error semantics. + +The JSON distinguishes: + +- `coverage`: whether a reachable site has a sketch candidate, exact summary + candidate, exact fallback, or a lowering rejection. Candidate coverage does + not establish that the whole query can execute from summaries. +- `root_plan` and `lifecycle_plan`: selected root structure and deployable + lifecycle result. Missing complete costs can make lifecycle planning retain + exact recomputation despite available sketch candidates. +- `offline_evidence`: matched primitive measurements and provenance, or an + explicit missing/incompatible/stale reason. Error statistics remain offline + observations for the measurement's recorded query, separate from the + planner's formal result guarantee. +- `heuristic_score`: a dimensionless planner score, never CPU nanoseconds. + Search and selection wall-clock timings are one local sample. Reporting and + materialization timings include JSON export work. +- End-to-end resource savings: `null` until complete raw execution, residual + operators, grouping cardinalities, sharing and deployment costs exist. Do not + divide heuristic scores to claim runtime speedup, or sum nested candidate + costs as a whole-query cost. + +The fixture identifies an upstream snapshot date, not an upstream commit; this +tool retains that limitation explicitly. It reuses the repository's existing +fixture, whose provenance is documented there. It does not claim to run the +upstream agent benchmark or its scenario data. + +## Measure supported exact-query reference implementations + +```sh +cargo run --release -p asap-devtools --bin o11y_exact_bench -- 300 60 > exact-snapshot.json +``` + +This profiles seven explicitly admitted o11y queries: instantaneous sums, +per-series window maxima, and a sum of per-series window averages. The synthetic +dataset has 300 series across three job labels, finite gauge values, and one +sample per minute inside the selected window. The 60 invocations repeat the +identical immutable snapshot and evaluation time. They are not advancing live +windows. Unsupported queries are reported as unavailable. + +The reference deployment retains the complete exact result of an admitted root +summary. It measures the raw value kernel and the retained result read separately +with a process CPU clock, and estimates one build plus repeated reads against +repeated raw execution. Both timed kernels include result allocation/destruction; +charging the build this way is conservative because its result is actually +retained. Only the profiled exact SummaryAgg signatures receive costs through +the public CostModel boundary; actual search, global selection, and root +materialization determine the reported choice. Unprofiled candidates and raw +fallback nodes never inherit those costs. + +The output identifies this as a Rust reference implementation, not the deployed +backend. It includes filtering and value reduction/readout but excludes disk, +network, protocol serialization and shared output-label materialization. Memory +is logical stored value bytes, not measured process RSS or a claim that raw data +can be deleted. Large savings from reusing identical results do not establish +the benefit of live sliding-window maintenance. + +## Compare query-matched sketch configurations + +First restore the saved inputs using the [artifact download instructions](../../tools/empirical-bench/ARTIFACTS.md), +or generate them with the benchmark driver. Result files are not checked in. + +```sh +cargo run -p asap-devtools --bin offline_recommend -- \ + tools/empirical-bench/results-sweep/comparison-evidence.json \ + tools/empirical-bench/results-sweep/request-uniform-are001.json +``` + +The companion comparison artifact includes disjoint construction, ingestion, +prepare and read CPU evidence for the exact frequency index and sketch rungs. +The request declares its offline observed-error criterion, fixed snapshot, +probe population, formal parameter minima if applicable, and resource weights. +The JSON reports accepted and rejected configurations, the exact reference, +selection, and dimensional tradeoffs. CPU-only and memory-weighted requests can +choose different plans. These point-frequency observations do not price or +bound error for the o11y gauge queries above. diff --git a/tools/empirical-bench/.gitignore b/tools/empirical-bench/.gitignore new file mode 100644 index 00000000..81c34145 --- /dev/null +++ b/tools/empirical-bench/.gitignore @@ -0,0 +1,3 @@ +/results/ +/results-sweep/ +__pycache__/ diff --git a/tools/empirical-bench/ARTIFACTS.md b/tools/empirical-bench/ARTIFACTS.md new file mode 100644 index 00000000..84a7495f --- /dev/null +++ b/tools/empirical-bench/ARTIFACTS.md @@ -0,0 +1,32 @@ +# Archived experiment results + +Full raw measurements and expanded query plans are distributed as an +[experiment artifact](https://github.com/ProjectASAP/ASAPPlanner/releases/tag/offline-evidence-2026-09-08), +not checked into the tool source tree. This prerelease is an experiment archive, +not a product release. The concise conclusions remain in +[the final report](../../docs/offline-o11y-final-2026-09-08.md). + +Download and verify from the repository root: + +```sh +gh release download offline-evidence-2026-09-08 --repo ProjectASAP/ASAPPlanner \ + --pattern offline-evidence-2026-09-08.tar.gz --dir /tmp +echo '180f3768203635c31188d19b500455290b96e19a4df4fdac7245af0944bd1b34 /tmp/offline-evidence-2026-09-08.tar.gz' | sha256sum --check +tar -xzf /tmp/offline-evidence-2026-09-08.tar.gz +``` + +Extract only into a checkout without existing result files; extraction restores +`tools/empirical-bench/results/` and `results-sweep/`. These generated directories +are ignored by Git. The archive is 460,710 bytes and contains 43 text files. +It preserves the original source snapshot +`152ca6cb71fc11a8c38d93e69c9fd7dca682f03b`, per-run commands, environment metadata, +raw timing/error reports, and measurement-source checksums. + +- `results/`: historical partial frequency measurements and initial planner/control-plane replay; also the later `o11y-exact-snapshot.json` reference evaluation. +- `results-sweep/`: final frequency matrix, disjoint CPU and heap evidence, six recommendation scenarios, four frequency binding reports, and final o11y control-plane replay. +- Each directory contains `MEASUREMENTS.md` explaining its measurement scope. Do not mix costs from the two runs. + +To produce new results instead of downloading these measurements, follow +[the benchmark guide](README.md) and [the replay guide](../../docs/user-guide/o11y-replay.md). +The archive preserves offline evidence, not production speedups or real-time +accuracy guarantees. diff --git a/tools/empirical-bench/README.md b/tools/empirical-bench/README.md new file mode 100644 index 00000000..e5d5b0b2 --- /dev/null +++ b/tools/empirical-bench/README.md @@ -0,0 +1,113 @@ +# Offline sketch-bench evidence (developer guide) + +This driver runs ProjectASAP/sketch-bench's real `approxbench` CLI and adapts its +schema-v5 reports to the planner's versioned evidence schema. It measures CMS and +CountSketch using `asap_sketchlib` RegularPath/Vector2D plus an exact Polars +frequency baseline, on deterministic uniform and Zipf streams. No collector, +query backend, runtime ground truth, or runtime error feedback is involved. + +Full historical and final outputs are stored in the [experiment archive](ARTIFACTS.md), +not in this source tree. Download and verify that archive to inspect saved results. + +```bash +git clone https://github.com/ProjectASAP/sketch-bench.git /path/to/sketch-bench +git -C /path/to/sketch-bench checkout 87f619e843fd2e4da784160d4e205a0d0d55f032 +(cd /path/to/sketch-bench && cargo build --release --locked -p aqpbm-cli) +python3 tools/empirical-bench/run.py --bench-repo /path/to/sketch-bench --output /tmp/offline-evidence --sweep +python3 -m unittest discover -s tools/empirical-bench -p 'test_*.py' +``` + +Build from the sketch-bench directory so its `.cargo/config.toml` applies +`target-cpu=native`, as in the archived run. The driver records the source +revision, dirty status, executable checksum, full invocations, compiler, CPU, +OS, implementation, and parameters. It pins Polars to one worker. Five measured +runs follow two warmups per timed operation by default. Upstream accuracy executes +one offline truth comparison for the fixed generated dataset, recorded as +`error.trials=1`; it is run separately because accuracy of insert is undefined. +These timings are repeated observations +within one process, not independent experiments: standard deviations are reported, +but no confidence interval or cross-distribution guarantee is inferred. +`manifest.json.runtime_provenance` distinguishes observed host metadata and the +driver's enforced thread setting from declared build preconditions. The runtime +string's release/native/jemalloc settings describe the documented required build; +the driver does not independently recover compiler or allocator flags from the +executable. Its SHA-256 identifies that binary without proving those build flags. +The archived run used a shared development host without CPU affinity or an +exclusive core reservation; timing estimates should be remeasured on deployment +hardware before using them for capacity decisions. + +Outputs: + +- `raw.json`: unmodified per-invocation sketch-bench flat reports, including wall + times, CPU user/system samples, process RSS/allocator readings, and all errors. +- `manifest.json`: execution and environment provenance. +- `operation-reports.json`: original reports before upstream flattening; preserve + per-operation memory, including exact prepared-index state. +- `memory-probe.json`: requested-allocation samples with each live sketch, using + the same library and upstream data generator on identical inputs. +- `planner-evidence.json`: normalized sketch measurements, with unknowns null. +- `exact-baselines.json`: exact frequency index costs on the same input. +- `resource-probe.json`: disjoint live-state construction/update/read/merge CPU, + actual serialized snapshot size and allocated filesystem blocks. +- `comparison-evidence.json`: query-bound sketch records and exact baselines; + its required `disjoint_live_state_v1` timing contract excludes duplicate setup + and destruction charges. +- `request-*.json`: fixed-snapshot CPU-only comparisons at 1%/5% observed mean + relative error, plus an explicitly illustrative memory-weighted comparison. + +The default sketch parameters match formal planner sizing for epsilon=0.01, delta=0.01: +CMS width 272, depth 5; CountSketch width 30,000, depth 83. Input defaults to +20,000 i64 keys, key-space size 1,000, seed 42; Zipf exponent is 1.1. Actual +distinct-key counts come from the ground-truth probe population. Errors are +average absolute relative point-frequency errors over **all distinct keys**. +They are not error of a total COUNT, or a formal epsilon bound. Aggregate error +over repeated deterministic inputs does not establish a confidence interval. +`--sweep` additionally measures CMS widths 512/2720/4096/27200/32768 (depth 5) +and CountSketch widths 300/3000 (depth 83). The power-of-two CMS widths support +the backend frequency extension's actual parameter constraints. An offline +accuracy threshold does not authorize deployment below formal minimum sizes. + +Original upstream CPU reports use paired user+system samples and preserve wrapper +allocation/destruction overhead. The current comparison exporter instead uses +`resource_probe.rs`, linked to the exact release libraries and jemalloc. It times +live-state phases separately with `CLOCK_PROCESS_CPUTIME_ID`: empty constructor, +updates, exact prepare, reads, and sketch merge. Holder allocation and destruction +are outside the constructor timer; constructors/destructors are outside the other +phase timers. Queries cover every distinct key ten times against an unchanged +snapshot, in a reproducible rotated sorted order. The exact implementation is +the upstream public `PolarsFrequencyCore`, not a replacement algorithm. +The model's scope ends with state retained after reading; retirement is excluded +on both sides. Wall time remains in upstream raw reports and is never called CPU. + +`retained_bytes` and `peak_bytes` use the companion `memory_probe.rs`, linked to +the exact release libraries built by sketch-bench. Its counting System allocator +measures requested heap allocation bytes across construction and insertion, +keeping the sketch alive at the final snapshot; input generation is outside +the measurement. This is independent of allocator-resident pages and does not +measure jemalloc overhead. Five runs must release every sketch allocation after +destruction. CPU results come from a separate uninstrumented jemalloc probe. +The exact heap probe warms Polars twice, then measures construction/ingestion/ +prepare with the exact state alive. Its readings are accepted only when every +post-destruction allocation balance returns to zero; otherwise the exact footprint +stays explicitly formula based. `--export-only --refresh-memory` refreshes this +heap evidence without rerunning or replacing CPU timings. +Upstream counter-storage formula, process RSS and allocator readings remain in +raw reports only. The resource probe calls the actual `serialize_to_bytes` API +and verifies estimate-equivalent deserialization for every input key. It writes +one snapshot to a fresh local temporary file, flushes it, and records allocated +blocks times 512 separately from logical byte length; the file is then removed. +Directory/inode overhead and a runtime write schedule are outside this disk +measurement. State bytes are never substituted for disk usage. Validity is a +30-day reproducibility policy, not a statistical claim of future applicability. + +The exact baseline inserts the input into a buffer, then `prepare` executes +Polars group-by/count and creates a HashMap. Reads query that prepared exact +index. Compare `empty_build + (insert_per_item * N) + prepare + Q * read` with +sketch `empty_build + (update_per_item * N) + Q * read`. All components use +disjoint timed regions. A sketch can save state while losing CPU to this exact index. +These comparisons are offline point-frequency microbenchmarks; they do not +measure an o11y PromQL query or end-to-end deployed speedup. + +`results/` preserves the earlier partial run whose constructor/disk/serialization +were unknown. `results-sweep/` contains the complete frequency component run; +its additional probe must not retroactively change the earlier measurements. diff --git a/tools/empirical-bench/memory_probe.rs b/tools/empirical-bench/memory_probe.rs new file mode 100644 index 00000000..7564fbe4 --- /dev/null +++ b/tools/empirical-bench/memory_probe.rs @@ -0,0 +1,101 @@ +//! Counts requested allocation bytes with each sketch alive; input generation is excluded. +use std::alloc::{GlobalAlloc, Layout, System}; +use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; +use asap_sketchlib::{Count, CountMin, DataInput, RegularPath, Vector2D}; + +struct Tracking; +static LIVE: AtomicUsize = AtomicUsize::new(0); +static PEAK: AtomicUsize = AtomicUsize::new(0); +fn account(bytes: usize) { + let live = LIVE.fetch_add(bytes, Relaxed) + bytes; + PEAK.fetch_max(live, Relaxed); +} +unsafe impl GlobalAlloc for Tracking { + unsafe fn alloc(&self, layout: Layout) -> *mut u8 { + let ptr = unsafe { System.alloc(layout) }; + if !ptr.is_null() { account(layout.size()); } + ptr + } + unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { + let ptr = unsafe { System.alloc_zeroed(layout) }; + if !ptr.is_null() { account(layout.size()); } + ptr + } + unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { + unsafe { System.dealloc(ptr, layout) }; + LIVE.fetch_sub(layout.size(), Relaxed); + } + unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, size: usize) -> *mut u8 { + let next = unsafe { System.realloc(ptr, layout, size) }; + if !next.is_null() { + if size >= layout.size() { account(size - layout.size()); } + else { LIVE.fetch_sub(layout.size() - size, Relaxed); } + } + next + } +} +#[global_allocator] +static ALLOCATOR: Tracking = Tracking; + +fn main() { + let path = std::env::args().nth(1).expect("raw.json path"); + let rows: Vec = serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap(); + let mut results = Vec::new(); + for row in rows { + let description: aqpbm_datagen::TableDescription = serde_json::from_value(row["workload"]["synthetic"]["description"].clone()).unwrap(); + let items = description.generate().unwrap().into_column(0).unwrap().into_i64().unwrap(); + if row["impl"] == "polars" { + use sketch_bench::wrappers::polars_shared::PolarsFrequencyCore; + // Initialize Polars' worker/cache state before attributing bytes to one live index. + for _ in 0..2 { + let mut warm = PolarsFrequencyCore::::default(); + for item in &items { warm.update(item); } + warm.finalize(); + } + let mut samples = Vec::with_capacity(5); + let mut cleanup_deltas = Vec::with_capacity(5); + for _ in 0..5 { + let baseline = LIVE.load(Relaxed); + PEAK.store(baseline, Relaxed); + let mut state = PolarsFrequencyCore::::default(); + for item in &items { state.update(item); } + state.finalize(); + std::hint::black_box(&state); + let sample = (LIVE.load(Relaxed).saturating_sub(baseline), PEAK.load(Relaxed).saturating_sub(baseline)); + drop(state); + cleanup_deltas.push(LIVE.load(Relaxed) as i64 - baseline as i64); + samples.push(sample); + } + results.push(serde_json::json!({"sketch": row["sketch"], "impl": "polars", + "workload": row["workload"], "sketch_config": row["sketch_config"], "samples": samples, + "cleanup_deltas": cleanup_deltas, + "method": "requested allocation bytes using counting System allocator; exact upstream PolarsFrequencyCore construction + insertion + prepare; state alive; input and two startup warmups excluded; valid only when cleanup deltas are zero; not allocator-resident pages"})); + continue; + } + let depth = row["sketch_config"]["params"]["rows"].as_u64().unwrap() as usize; + let width = row["sketch_config"]["params"]["cols"].as_u64().unwrap() as usize; + let cms = row["sketch"].as_str().unwrap().starts_with("cms-"); + let mut samples = Vec::with_capacity(5); + for _ in 0..5 { + let baseline = LIVE.load(Relaxed); + PEAK.store(baseline, Relaxed); + // The sketch remains alive at both snapshots, unlike a consuming benchmark closure. + let (retained, peak) = if cms { + let mut sketch = CountMin::, RegularPath>::with_dimensions(depth, width); + for item in &items { sketch.insert(&DataInput::I64(*item)); } + std::hint::black_box(&sketch); + (LIVE.load(Relaxed) - baseline, PEAK.load(Relaxed) - baseline) + } else { + let mut sketch = Count::, RegularPath>::with_dimensions(depth, width); + for item in &items { sketch.insert(&DataInput::I64(*item)); } + std::hint::black_box(&sketch); + (LIVE.load(Relaxed) - baseline, PEAK.load(Relaxed) - baseline) + }; + assert_eq!(LIVE.load(Relaxed), baseline, "sketch destruction must release its allocations"); + samples.push((retained, peak)); + } + results.push(serde_json::json!({"sketch": row["sketch"], "impl": "lib", "workload": row["workload"], "sketch_config": row["sketch_config"], + "samples": samples, "method": "requested allocation bytes using counting System allocator; construction + insertion; sketch alive; dataset excluded; not allocator-resident pages"})); + } + println!("{}", serde_json::to_string_pretty(&results).unwrap()); +} diff --git a/tools/empirical-bench/resource_probe.rs b/tools/empirical-bench/resource_probe.rs new file mode 100644 index 00000000..e1a17629 --- /dev/null +++ b/tools/empirical-bench/resource_probe.rs @@ -0,0 +1,138 @@ +//! Complements sketch-bench with empty construction CPU and actual persisted snapshot size. +use std::hint::black_box; +use std::io::Write; +use std::os::unix::fs::MetadataExt; +use asap_sketchlib::{Count, CountMin, DataInput, RegularPath, Vector2D}; + +#[global_allocator] +static ALLOCATOR: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; + +fn cpu_ns() -> u64 { + #[repr(C)] + struct Timespec { tv_sec: std::os::raw::c_long, tv_nsec: std::os::raw::c_long } + extern "C" { fn clock_gettime(clock_id: std::os::raw::c_int, time: *mut Timespec) -> std::os::raw::c_int; } + // Linux CLOCK_PROCESS_CPUTIME_ID; this companion also uses Linux allocated blocks. + assert!(cfg!(target_os = "linux")); + let mut time = Timespec { tv_sec: 0, tv_nsec: 0 }; + assert_eq!(unsafe { clock_gettime(2, &mut time) }, 0); + time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 +} + +fn construction(mut build: impl FnMut() -> T, batch: usize) -> Vec { + let mut samples = Vec::with_capacity(5); + for run in 0..7 { + let mut alive = Vec::with_capacity(batch); + let start = cpu_ns(); + for _ in 0..batch { alive.push(black_box(build())); } + let elapsed = cpu_ns() - start; + if run >= 2 { samples.push(elapsed as f64 / batch as f64); } + // Empty objects remain alive until after the timed construction batch. + black_box(&alive); + drop(alive); + } + samples +} + +fn live_phases(items: &[i64], build: impl Fn() -> T, insert: impl Fn(&mut T, i64), + read: impl Fn(&T, i64) -> f64, prepare: impl Fn(&mut T), + merge: Option) -> serde_json::Value { + let mut keys: Vec<_> = items.iter().copied().collect::>().into_iter().collect(); + // Fixed rotation avoids sorted probe order without introducing another RNG definition. + if !keys.is_empty() { let mid = keys.len() / 2; keys.rotate_left(mid); } + let mut updates = Vec::with_capacity(5); + let mut prepares = Vec::with_capacity(5); + let mut reads = Vec::with_capacity(5); + let mut merges = Vec::with_capacity(5); + for run in 0..7 { + let mut state = build(); + let start = cpu_ns(); + for item in items { insert(&mut state, black_box(*item)); } + let update = (cpu_ns() - start) as f64 / items.len() as f64; + let start = cpu_ns(); + prepare(&mut state); + let prepare = (cpu_ns() - start) as f64; + let start = cpu_ns(); + for _ in 0..10 { for key in &keys { black_box(read(&state, black_box(*key))); } } + let query = (cpu_ns() - start) as f64 / (keys.len() * 10) as f64; + let merge_time = merge.map(|merge| { + let mut other = build(); + for item in items { insert(&mut other, *item); } + let start = cpu_ns(); + merge(&mut state, &other); + (cpu_ns() - start) as f64 + }); + if run >= 2 { + updates.push(update); prepares.push(prepare); reads.push(query); + if let Some(time) = merge_time { merges.push(time); } + } + black_box(&state); + } + serde_json::json!({"update_cpu_ns_samples": updates, "prepare_cpu_ns_samples": prepares, + "read_cpu_ns_samples": reads, "merge_cpu_ns_samples": merges, + "query_distinct_keys": keys.len(), "query_passes": 10, + "method": "CLOCK_PROCESS_CPUTIME_ID; live state with constructor/destructor excluded; five runs after two warmups; update per input item; read per all-distinct-key lookup across ten repeated fixed-snapshot probe passes; merge per binary merge; fixed rotated sorted key order"}) +} + +fn main() { + let args: Vec = std::env::args().collect(); + let rows: Vec = serde_json::from_slice(&std::fs::read(&args[1]).unwrap()).unwrap(); + let directory = std::path::Path::new(&args[2]); + let mut results = Vec::new(); + for (index, row) in rows.into_iter().enumerate() { + let description: aqpbm_datagen::TableDescription = serde_json::from_value(row["workload"]["synthetic"]["description"].clone()).unwrap(); + let items = description.generate().unwrap().into_column(0).unwrap().into_i64().unwrap(); + if row["impl"] == "polars" { + use sketch_bench::wrappers::polars_shared::PolarsFrequencyCore; + let samples = construction(PolarsFrequencyCore::::default, 4096); + let phases = live_phases(&items, PolarsFrequencyCore::::default, + |s, x| s.update(&x), |s, x| s.query(&x) as f64, |s| s.finalize(), None); + results.push(serde_json::json!({"sketch": row["sketch"], "impl": row["impl"], + "workload": row["workload"], "sketch_config": row["sketch_config"], + "build_cpu_ns_samples": samples, "build_batch": 4096, + "phases": phases, + "build_method": "process CPU around upstream PolarsFrequencyCore::::default(); output holder allocation and destruction excluded"})); + continue; + } + let depth = row["sketch_config"]["params"]["rows"].as_u64().unwrap() as usize; + let width = row["sketch_config"]["params"]["cols"].as_u64().unwrap() as usize; + let batch = (16_000_000 / (depth * width * 4)).clamp(2, 256); + let (samples, serialized, phases) = if row["sketch"].as_str().unwrap().starts_with("cms-") { + let samples = construction(|| CountMin::, RegularPath>::with_dimensions(depth, width), batch); + let phases = live_phases(&items, || CountMin::, RegularPath>::with_dimensions(depth, width), + |s,x| s.insert(&DataInput::I64(x)), |s,x| s.estimate(&DataInput::I64(x)) as f64, |_| {}, Some(CountMin::, RegularPath>::merge)); + let mut sketch = CountMin::, RegularPath>::with_dimensions(depth, width); + for item in &items { sketch.insert(&DataInput::I64(*item)); } + let bytes = sketch.serialize_to_bytes().unwrap(); + let restored = CountMin::, RegularPath>::deserialize_from_bytes(&bytes).unwrap(); + for item in &items { assert_eq!(sketch.estimate(&DataInput::I64(*item)), restored.estimate(&DataInput::I64(*item))); } + (samples, bytes, phases) + } else { + let samples = construction(|| Count::, RegularPath>::with_dimensions(depth, width), batch); + let phases = live_phases(&items, || Count::, RegularPath>::with_dimensions(depth, width), + |s,x| s.insert(&DataInput::I64(x)), |s,x| s.estimate(&DataInput::I64(x)), |_| {}, Some(Count::, RegularPath>::merge)); + let mut sketch = Count::, RegularPath>::with_dimensions(depth, width); + for item in &items { sketch.insert(&DataInput::I64(*item)); } + let bytes = sketch.serialize_to_bytes().unwrap(); + let restored = Count::, RegularPath>::deserialize_from_bytes(&bytes).unwrap(); + for item in &items { assert_eq!(sketch.estimate(&DataInput::I64(*item)), restored.estimate(&DataInput::I64(*item))); } + (samples, bytes, phases) + }; + let path = directory.join(format!("snapshot-{index}.msgpack")); + let mut file = std::fs::OpenOptions::new().write(true).create_new(true).open(&path).unwrap(); + file.write_all(&serialized).unwrap(); + file.sync_all().unwrap(); + let metadata = file.metadata().unwrap(); + assert_eq!(metadata.len(), serialized.len() as u64); + let disk_bytes = metadata.blocks() * 512; + drop(file); + std::fs::remove_file(path).unwrap(); + results.push(serde_json::json!({"sketch": row["sketch"], "impl": row["impl"], + "workload": row["workload"], "sketch_config": row["sketch_config"], + "build_cpu_ns_samples": samples, "build_batch": batch, + "phases": phases, + "build_method": "CLOCK_PROCESS_CPUTIME_ID around empty sketch constructors in batched live objects; holder allocation and all destruction excluded; jemalloc; five runs after two warmups", + "serialized_bytes": serialized.len(), "disk_bytes": disk_bytes, + "disk_method": "actual flushed single MsgPack snapshot file allocated blocks*512 on local temporary filesystem; directory/inode overhead excluded; file removed after measurement; not runtime persistence demand"})); + } + println!("{}", serde_json::to_string_pretty(&results).unwrap()); +} diff --git a/tools/empirical-bench/run.py b/tools/empirical-bench/run.py new file mode 100644 index 00000000..cd56d756 --- /dev/null +++ b/tools/empirical-bench/run.py @@ -0,0 +1,367 @@ +#!/usr/bin/env python3 +"""Run the real sketch-bench CLI and export offline planner evidence (stdlib only).""" +import argparse +import datetime +import hashlib +import json +import math +import os +from pathlib import Path +import platform +import shlex +import statistics +import subprocess +import tempfile + + +BUILD_PROVENANCE = { + "runtime_field_status": "declared build preconditions, not introspected from the executable", + "compiler_status": "rustc --version observed on driver host; executable compiler not independently verified", + "required_build": "run cargo build --release --locked -p aqpbm-cli from the pinned sketch-bench directory with default features", + "declared_settings": "release; repository target-cpu=native; default jemalloc", + "verified_execution_setting": "driver sets POLARS_MAX_THREADS=1", + "binary_identity": "SHA-256 recorded; digest identifies executable but does not verify build settings", +} + + +def command(argv, **kwargs): + return subprocess.check_output(argv, text=True, **kwargs).strip() + + +def measurement(values): + if not values or any(not math.isfinite(x) or x < 0 for x in values): + raise ValueError("invalid or missing measurement") + return {"value": statistics.mean(values), + "stddev": statistics.stdev(values) if len(values) > 1 else None, + "samples": len(values)} + + +def cpu_per_op(row, op): + """Recover each run's work from paired rate and elapsed samples, as upstream does.""" + rate = row.get(op + ("_folds_per_sec" if op == "merge" else "_throughput_items_per_sec")) + cpu = row.get(op + "_cpu_time_ms") + wall = row.get(op + "_wall_time_ms") + if not rate or not cpu or not wall: + return None + samples = [rate["samples"], wall["samples"], cpu["user_ms"]["samples"], cpu["sys_ms"]["samples"]] + if len({len(x) for x in samples}) != 1: + raise ValueError("unaligned CPU/rate/time samples") + values = [] + for r, elapsed, user, system in zip(*samples): + work = r * elapsed / 1000 + if work <= 0: + raise ValueError("zero benchmark work") + values.append((user + system) * 1_000_000 / work) + result = measurement(values) + result["method"] = "mean of paired run (process user+system CPU ns)/(throughput*wall seconds); per " + ("binary merge" if op == "merge" else "input item" if op == "insert" else "point-frequency key lookup") + return result + + +def cpu_batch(row, op): + cpu = row.get(op + "_cpu_time_ms") + if not cpu: + return None + return measurement([(u + s) * 1_000_000 for u, s in + zip(cpu["user_ms"]["samples"], cpu["sys_ms"]["samples"])]) + + +def export(raw, manifest, operation_reports, memory_rows, resource_rows=None): + records, baselines = [], [] + if not (len(raw) == len(manifest["invocations"]) == len(operation_reports)): + raise ValueError("misaligned invocation reports") + for row, invocation, operations in zip(raw, manifest["invocations"], operation_reports): + if row["schema_version"] != 5: + raise ValueError("adapter requires sketch-bench schema 5") + accuracy = row["query_accuracy"] + resource = next((r for r in (resource_rows or []) if r["sketch"] == row["sketch"] + and r["impl"] == row["impl"] and r["sketch_config"] == row["sketch_config"] + and r["workload"] == row["workload"]), None) + if resource_rows is not None and (resource is None or "phases" not in resource): + raise ValueError("complete comparison requires matched disjoint live-state phase measurements for every record") + def phase_metric(phase): + return (dict(measurement(resource["phases"][phase + "_cpu_ns_samples"]), method=resource["phases"]["method"]) + if resource and resource.get("phases") else None) + shape = row["workload"]["synthetic"]["description"] + distribution = {"id": invocation["distribution_id"], + "family": invocation["distribution"], + "sample_count": shape["row_num"], + "distinct_count": int(accuracy["probes_all"]), + "parameters": shape["column_spec"][0]["distribution"]} + if invocation["algorithm"] == "Exact": + exact_memory = next((m for m in memory_rows if m.get("impl") == "polars" and m["workload"] == row["workload"]), None) + baselines.append({"distribution": distribution, "implementation": "polars exact group_by + HashMap", + "insert_cpu_ns_per_item": phase_metric("update") or cpu_per_op(row, "insert"), + "prepare_cpu_ns_per_dataset": phase_metric("prepare") or cpu_batch(row, "prepare"), + "read_cpu_ns_per_key": phase_metric("read") or cpu_per_op(row, "query"), + "retained_bytes": max(r["bench"]["memory_bytes"] for r in operations + if r["bench"].get("operation") == "query"), + "retained_bytes_method": "upstream prepared-query memory_bytes formula: buffer capacity plus HashMap capacity footprint; not measured allocated heap or peak", + "accuracy": accuracy}) + baselines[-1]["empty_build_cpu_ns"] = (dict(measurement(resource["build_cpu_ns_samples"]), method=resource["build_method"]) + if resource else None) + if exact_memory and all(delta == 0 for delta in exact_memory["cleanup_deltas"]): + baselines[-1]["retained_heap_bytes"] = dict(measurement([s[0] for s in exact_memory["samples"]]), method=exact_memory["method"]) + baselines[-1]["peak_heap_bytes"] = dict(measurement([s[1] for s in exact_memory["samples"]]), method=exact_memory["method"]) + continue + algorithm = invocation["algorithm"] + memory = next(m for m in memory_rows if m["sketch"] == row["sketch"] and m["workload"] == row["workload"] + and (m.get("sketch_config", row["sketch_config"]) == row["sketch_config"])) + params = row["sketch_config"]["params"] + timestamp = row["insert_timestamp"] + if not timestamp.endswith("Z"): + raise ValueError("expected upstream UTC timestamp") + measured = int(datetime.datetime.fromisoformat(timestamp[:19] + "+00:00").timestamp()) + environment = dict(manifest["environment"]) + environment["implementation"] = "asap_sketchlib RegularPath Vector2D" + # The algorithm distinguishes families; storage/hash path must also match. + environment["id"] = hashlib.sha256(json.dumps(environment, sort_keys=True).encode()).hexdigest()[:16] + records.append({"id": invocation["id"], "algorithm": algorithm, + "params": {algorithm: {"width": params["cols"], "depth": params["rows"]}}, + "distribution": distribution, "environment": environment, + "measured_at_unix_seconds": measured, + "valid_until_unix_seconds": measured + 30 * 86400, + "provenance": {"command": shlex.join(invocation["argv"]), + "dataset": "deterministic synthetic i64 frequency stream, seed 42", + "source_revision": manifest["source_revision"], + "repetitions": row["runs"]}, + "metrics": {"build_cpu_ns": (dict(measurement(resource["build_cpu_ns_samples"]), method=resource["build_method"]) + if resource else None), + "update_cpu_ns": phase_metric("update") or cpu_per_op(row, "insert"), + "merge_cpu_ns": phase_metric("merge") or cpu_per_op(row, "merge"), + "read_cpu_ns": phase_metric("read") or cpu_per_op(row, "query"), + "retained_bytes": dict(measurement([float(s[0]) for s in memory["samples"]]), method=memory["method"]), + "peak_bytes": dict(measurement([float(s[1]) for s in memory["samples"]]), method=memory["method"]), + "serialized_bytes": (dict(measurement([resource["serialized_bytes"]]), method="actual serialize_to_bytes MsgPack length; estimate-equivalent roundtrip verified for every input key") if resource else None), + "disk_bytes": (dict(measurement([resource["disk_bytes"]]), method=resource["disk_method"]) if resource else None)}, + "error": {"metric": "mean_absolute_relative_frequency_error_all_distinct_keys", + "mean": accuracy["are_all"], "max": None, + "trials": int(accuracy.get("accuracy_runs", 1)), + "ground_truth_method": "exact offline HashMap counts; every distinct key probed", + "query": {"kind": "point_frequency", "value_type": "i64", + "accuracy_stddev": accuracy.get("are_all_stddev"), + "full_accuracy": accuracy}}}) + return {"schema_version": 1, + "benchmark_version": "sketch-bench@" + manifest["source_revision"] + ("; adapter-v2-disjoint" if resource_rows else "; adapter-v1"), + "model_version": "empirical-update-cpu-v1", "records": records}, baselines + + +def comparison_artifact(artifact, baselines, manifest): + query = {"kind": "point_frequency", "value_type": "i64", "probe_set": "all_distinct_keys"} + exact_records = [] + for baseline in baselines: + if baseline["empty_build_cpu_ns"] is None: + continue + reference = next(r for r in artifact["records"] if r["distribution"] == baseline["distribution"]) + environment = dict(manifest["environment"]) + environment["implementation"] = "polars group_by + std HashMap" + environment["implementation_version"] = "polars 0.46.0" + environment["id"] = hashlib.sha256(json.dumps(environment, sort_keys=True).encode()).hexdigest()[:16] + invocation = next(i for i in manifest["invocations"] if i["algorithm"] == "Exact" + and i["distribution_id"] == baseline["distribution"]["id"]) + exact_records.append({"id": invocation["id"], "distribution": baseline["distribution"], + "environment": environment, "query": query, + "measured_at_unix_seconds": reference["measured_at_unix_seconds"], + "valid_until_unix_seconds": reference["valid_until_unix_seconds"], + "provenance": {"command": shlex.join(invocation["argv"]), + "dataset": "deterministic synthetic i64 frequency stream, seed 42; exact zero-error comparison verified offline", + "source_revision": manifest["source_revision"], "repetitions": 5}, + "metrics": {"empty_build_cpu_ns": baseline["empty_build_cpu_ns"], + "update_cpu_ns": baseline["insert_cpu_ns_per_item"], + "prepare_cpu_ns": baseline["prepare_cpu_ns_per_dataset"], + "read_cpu_ns": baseline["read_cpu_ns_per_key"], + "retained_bytes": baseline.get("retained_heap_bytes") or dict(measurement([baseline["retained_bytes"]]), method=baseline["retained_bytes_method"]), + "peak_bytes": baseline.get("peak_heap_bytes")}}) + return {"schema_version": 1, "timing_contract": "disjoint_live_state_v1", "sketch_evidence": artifact, + "query_bindings": [{"record_id": r["id"], "query": query} for r in artifact["records"]], + "exact_records": exact_records} + + +def refresh_memory(bench_repo, raw_path, memory_path): + deps = bench_repo.resolve() / "target/release/deps" + with tempfile.TemporaryDirectory(prefix="asap-memory-probe-") as temporary: + executable = str(Path(temporary) / "memory-probe") + argv = ["rustc", "--edition=2021", "-C", "opt-level=3", "-C", "panic=abort", "-C", "target-cpu=native", + "-L", f"dependency={deps}", str(Path(__file__).with_name("memory_probe.rs")), "-o", executable] + for name in ["asap_sketchlib", "aqpbm_datagen", "serde_json", "sketch_bench"]: + candidates = list(deps.glob(f"lib{name}-*.rlib")) + if len(candidates) != 1: + raise ValueError(f"expected one pinned release library for {name}, got {len(candidates)}") + argv += ["--extern", f"{name}={candidates[0]}"] + for pattern in ["*/out", "*/out/lib"]: + for native in (bench_repo.resolve() / "target/release/build").glob(pattern): + argv += ["-L", f"native={native}"] + subprocess.check_call(argv) + memory_path.write_text(command([executable, str(raw_path)], env=dict(os.environ, POLARS_MAX_THREADS="1")) + "\n") + return {"source_sha256": hashlib.sha256(Path(__file__).with_name("memory_probe.rs").read_bytes()).hexdigest(), + "compile_argv": argv, + "allocator": "System + requested-byte tracking; separate from uninstrumented jemalloc CPU measurements"} + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--bench-repo", required=True, type=Path) + parser.add_argument("--output", required=True, type=Path) + parser.add_argument("--size", type=int, default=20000) + parser.add_argument("--cardinality", type=int, default=1000) + parser.add_argument("--runs", type=int, default=5) + parser.add_argument("--sweep", action="store_true", help="measure three widths per frequency family") + parser.add_argument("--export-only", action="store_true") + parser.add_argument("--refresh-memory", action="store_true", help="with --export-only, remeasure heap without rerunning CPU timings") + parser.add_argument("--resume", action="store_true", help="continue a partially completed run with identical parameters") + args = parser.parse_args() + if args.refresh_memory and not args.export_only: + parser.error("--refresh-memory requires --export-only") + args.output.mkdir(parents=True, exist_ok=True) + manifest_path = args.output / "manifest.json" + raw_path = args.output / "raw.json" + operations_path = args.output / "operation-reports.json" + if args.export_only: + manifest, raw = json.loads(manifest_path.read_text()), json.loads(raw_path.read_text()) + operation_reports = json.loads(operations_path.read_text()) + else: + if (raw_path.exists() or manifest_path.exists()) and not args.resume: + parser.error("output already contains a run; use a new directory or --export-only") + repo = args.bench_repo.resolve() + if 'name = "asap_sketchlib"\nversion = "0.2.2"' not in (repo / "Cargo.lock").read_text(): + parser.error("adapter is validated with asap_sketchlib 0.2.2; do not relabel another version") + binary = repo / "target/release/approxbench" + cpu = next((line.split(":", 1)[1].strip() for line in Path("/proc/cpuinfo").read_text().splitlines() + if line.startswith("model name")), platform.processor()) + manifest = {"source_revision": command(["git", "rev-parse", "HEAD"], cwd=repo), + "source_dirty": bool(command(["git", "status", "--porcelain"], cwd=repo)), + "environment": {"cpu": cpu, "os": platform.platform(), + "runtime": command(["rustc", "--version"]) + "; release; target-cpu=native; jemalloc; POLARS_MAX_THREADS=1", + "implementation_version": "asap_sketchlib 0.2.2"}, + "binary_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), + "runtime_provenance": BUILD_PROVENANCE, + "invocations": []} + raw, operation_reports = [], [] + if args.resume: + previous = json.loads(manifest_path.read_text()) + if previous["binary_sha256"] != manifest["binary_sha256"] or previous["environment"] != manifest["environment"]: + parser.error("resume requires the same binary and environment") + manifest, raw = previous, json.loads(raw_path.read_text()) + operation_reports = json.loads(operations_path.read_text()) + for invocation in manifest["invocations"]: + argv = invocation["argv"] + if any(argv[argv.index(flag) + 1] != str(value) for flag, value in + [("--size", args.size), ("--cardinality", args.cardinality), ("--runs", args.runs)]): + parser.error("resume requires the same size, cardinality, and runs") + env = dict(os.environ, POLARS_MAX_THREADS="1") + for distribution in ["uniform", "zipf"]: + targets = [ + ("Cms", "cms-regularpath-vector2d", "lib", 5, 272), + ("CountSketch", "countsketch-regularpath-vector2d", "lib", 83, 30000), + ("Exact", "cms", "polars", 5, 272)] + if args.sweep: + targets += [("Cms", "cms-regularpath-vector2d", "lib", 5, width) for width in [2720, 27200, 512, 4096, 32768]] + targets += [("CountSketch", "countsketch-regularpath-vector2d", "lib", 83, width) for width in [300, 3000]] + for algorithm, variant, library, rows, cols in targets: + record_id = f"{algorithm.lower()}-{distribution}-seed42" + (f"-w{cols}-d{rows}" if args.sweep else "") + if any(i["id"] == record_id for i in manifest["invocations"]): + continue + argv = [str(binary), "sketchbench", "--variant", variant, "--library", library, + "--config", f"rows={rows} cols={cols}", "--dataset", distribution, + "--size", str(args.size), "--cardinality", str(args.cardinality), + "--dtype", "i64", "--seed", "42", "--runs", str(args.runs), + "--warmup-runs", "2", "--operations", + "insert,query" if algorithm == "Exact" else "insert,query,merge", + "--metrics", "throughput,cpu,memory"] + if distribution == "zipf": + argv += ["--zipf-s", "1.1"] + print(shlex.join(argv), flush=True) + output = command(argv, env=env) + accuracy_argv = list(argv) + accuracy_argv[accuracy_argv.index("--operations") + 1] = "query" + accuracy_argv[accuracy_argv.index("--metrics") + 1] = "accuracy" + output += "\n" + command(accuracy_argv, env=env) + prepare_argv = None + if algorithm == "Exact": + prepare_argv = list(argv) + prepare_argv[prepare_argv.index("--operations") + 1] = "prepare" + prepare_argv[prepare_argv.index("--metrics") + 1] = "latency,cpu,memory" + output += "\n" + command(prepare_argv, env=env) + operations = [json.loads(line) for line in output.splitlines() if line.strip()] + row = json.loads(command([str(binary), "flatten"], input=output, env=env)) + raw.append(row) + operation_reports.append(operations) + manifest["invocations"].append({"id": record_id, + "distribution_id": f"{distribution}-n{args.size}-c{args.cardinality}-seed42", + "distribution": distribution, "algorithm": algorithm, "argv": argv, + "accuracy_argv": accuracy_argv, "prepare_argv": prepare_argv}) + raw_path.write_text(json.dumps(raw, indent=2) + "\n") + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + operations_path.write_text(json.dumps(operation_reports, indent=2) + "\n") + memory_path = args.output / "memory-probe.json" + resources_path = args.output / "resource-probe.json" + if not args.export_only: + # Older partial runs may lack prepare: CPU/memory alone select no upstream timing pass. + for index, (row, invocation) in enumerate(zip(raw, manifest["invocations"])): + if invocation["algorithm"] == "Exact" and not row.get("prepare_cpu_time_ms"): + prepare_argv = list(invocation["argv"]) + prepare_argv[prepare_argv.index("--operations") + 1] = "prepare" + prepare_argv[prepare_argv.index("--metrics") + 1] = "latency,cpu,memory" + output = command(prepare_argv, env=env) + operation_reports[index].extend(json.loads(line) for line in output.splitlines() if line.strip()) + raw[index] = json.loads(command([str(binary), "flatten"], + input="\n".join(json.dumps(r) for r in operation_reports[index]), env=env)) + invocation["prepare_argv"] = prepare_argv + raw_path.write_text(json.dumps(raw, indent=2) + "\n") + operations_path.write_text(json.dumps(operation_reports, indent=2) + "\n") + deps = args.bench_repo.resolve() / "target/release/deps" + manifest["memory_probe"] = refresh_memory(args.bench_repo, raw_path, memory_path) + with tempfile.TemporaryDirectory(prefix="asap-resource-probe-") as temporary: + executable = str(Path(temporary) / "resource-probe") + argv = ["rustc", "--edition=2021", "-C", "opt-level=3", "-C", "panic=abort", "-C", "target-cpu=native", + "-L", f"dependency={deps}", str(Path(__file__).with_name("resource_probe.rs")), "-o", executable] + for name in ["asap_sketchlib", "aqpbm_datagen", "serde_json", "tikv_jemallocator", "sketch_bench"]: + candidates = list(deps.glob(f"lib{name}-*.rlib")) + if len(candidates) != 1: + raise ValueError(f"expected one pinned release library for {name}, got {len(candidates)}") + argv += ["--extern", f"{name}={candidates[0]}"] + for native in (args.bench_repo.resolve() / "target/release/build").glob("tikv-jemalloc-sys-*/out/lib"): + argv += ["-L", f"native={native}"] + for native in (args.bench_repo.resolve() / "target/release/build").glob("*/out"): + argv += ["-L", f"native={native}"] + subprocess.check_call(argv) + resources_path.write_text(command([executable, str(raw_path), temporary], env=env) + "\n") + manifest["resource_probe"] = {"source_sha256": hashlib.sha256(Path(__file__).with_name("resource_probe.rs").read_bytes()).hexdigest(), + "compile_argv": argv, "allocator": "jemalloc", "disk_filesystem_path": tempfile.gettempdir()} + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + if args.refresh_memory: + if hashlib.sha256((args.bench_repo.resolve() / "target/release/approxbench").read_bytes()).hexdigest() != manifest["binary_sha256"]: + parser.error("memory refresh requires the original benchmark binary") + manifest["memory_probe"] = refresh_memory(args.bench_repo, raw_path, memory_path) + manifest["runtime_provenance"] = BUILD_PROVENANCE + manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") + artifact, baselines = export(raw, manifest, operation_reports, json.loads(memory_path.read_text()), + json.loads(resources_path.read_text()) if resources_path.exists() else None) + (args.output / "planner-evidence.json").write_text(json.dumps(artifact, indent=2) + "\n") + (args.output / "exact-baselines.json").write_text(json.dumps(baselines, indent=2) + "\n") + if resources_path.exists(): + comparison = comparison_artifact(artifact, baselines, manifest) + (args.output / "comparison-evidence.json").write_text(json.dumps(comparison, indent=2) + "\n") + for exact in comparison["exact_records"]: + row = next(r for r in artifact["records"] if r["distribution"] == exact["distribution"]) + for budget, suffix in [(0.01, "001"), (0.05, "005")]: + request = {"context": {"distribution": row["distribution"], "environment": row["environment"], + "now_unix_seconds": max(r["measured_at_unix_seconds"] for r in artifact["records"])}, + "exact_environment": exact["environment"], "query": exact["query"], + "accuracy": {"metric": row["error"]["metric"], "max_observed_mean": budget, "minimum_trials": 1}, + "workload": {"input_items_per_state": row["distribution"]["sample_count"], + "reads_per_state": 1000, "merges_per_state": 0, "state_instances": 1, "horizon_seconds": 300.0}, + "weights": {"cpu_ns_weight": 1.0, "retained_byte_seconds_weight": 0.0}, "formal_minimums": None} + (args.output / ("request-" + row["distribution"]["family"] + "-are" + suffix + ".json")).write_text(json.dumps(request, indent=2) + "\n") + request["accuracy"]["max_observed_mean"] = 0.01 + request["weights"]["retained_byte_seconds_weight"] = 0.01 + request["formal_minimums"] = [{"algorithm": "Cms", "params": {"Cms": {"width": 512, "depth": 5}}}] + (args.output / ("request-" + row["distribution"]["family"] + "-memory-weighted.json")).write_text(json.dumps(request, indent=2) + "\n") + for row in artifact["records"]: + context = {"distribution": row["distribution"], "environment": row["environment"], + "now_unix_seconds": max(r["measured_at_unix_seconds"] for r in artifact["records"])} + (args.output / ("context-" + row["distribution"]["family"] + ".json")).write_text(json.dumps(context, indent=2) + "\n") + print(f"Exported {len(artifact['records'])} sketch records and {len(baselines)} exact baselines.") + + +if __name__ == "__main__": + main() diff --git a/tools/empirical-bench/test_export.py b/tools/empirical-bench/test_export.py new file mode 100644 index 00000000..ab5389b5 --- /dev/null +++ b/tools/empirical-bench/test_export.py @@ -0,0 +1,51 @@ +"""Unit checks for measurement normalization, independent of running the benchmark.""" +import unittest + +from run import cpu_per_op, export + + +class NormalizationTests(unittest.TestCase): + def test_complete_comparison_rejects_missing_live_phase_measurements(self): + """An old constructor-only probe cannot label consuming-wrapper CPU as disjoint.""" + row = {"schema_version": 5, "query_accuracy": {}, "sketch": "cms", "impl": "lib", + "sketch_config": {}, "workload": {}} + manifest = {"invocations": [{}]} + for resources in [[], [dict(row, build_cpu_ns_samples=[1.0])]]: + with self.subTest(resources=resources): + with self.assertRaisesRegex(ValueError, "disjoint live-state"): + export([row], manifest, [[]], [], resources) + + def test_partial_report_files_are_rejected_before_pairing(self): + """A truncated report file must fail rather than silently lose invocations.""" + for raw, invocations, operations in [([{}], [], []), ([], [{}], []), ([], [], [{}])]: + with self.subTest(lengths=(len(raw), len(invocations), len(operations))): + with self.assertRaisesRegex(ValueError, "misaligned invocation reports"): + export(raw, {"invocations": invocations}, operations, []) + + def test_cpu_normalizes_paired_work_not_wall_time(self): + """Different wall durations for identical work must not change the denominator.""" + row = {"query_throughput_items_per_sec": {"samples": [2000, 1000]}, + "query_wall_time_ms": {"samples": [100, 200]}, + "query_cpu_time_ms": {"user_ms": {"samples": [20, 20]}, + "sys_ms": {"samples": [10, 10]}}} + result = cpu_per_op(row, "query") + self.assertEqual(result["value"], 150000) + self.assertEqual(result["stddev"], 0) + self.assertEqual(result["samples"], 2) + + def test_unmeasured_cpu_is_not_zero(self): + """Missing CPU evidence stays absent instead of looking like free work.""" + self.assertIsNone(cpu_per_op({}, "query")) + + def test_misaligned_samples_rejected(self): + """CPU and rates from different run populations cannot be paired.""" + row = {"insert_throughput_items_per_sec": {"samples": [2000, 1000]}, + "insert_wall_time_ms": {"samples": [100]}, + "insert_cpu_time_ms": {"user_ms": {"samples": [20, 20]}, + "sys_ms": {"samples": [10, 10]}}} + with self.assertRaises(ValueError): + cpu_per_op(row, "insert") + + +if __name__ == "__main__": + unittest.main() From 24fb2f98033e2de7971c1299ac84f61cc4b797e9 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 8 Sep 2026 12:37:09 -0600 Subject: [PATCH 2/3] refactor(bench): move experiment execution ownership to backend --- Cargo.lock | 2 - crates/devtools/Cargo.toml | 2 - crates/devtools/src/bin/o11y_exact_bench.rs | 367 ------------- crates/devtools/src/bin/o11y_replay.rs | 495 ------------------ crates/devtools/src/bin/offline_recommend.rs | 28 - .../empirical-o11y-execution-plan.md | 5 +- docs/offline-o11y-final-2026-09-08.md | 18 +- docs/user-guide/o11y-replay.md | 169 +++--- tools/empirical-bench/.gitignore | 3 - tools/empirical-bench/ARTIFACTS.md | 32 -- tools/empirical-bench/README.md | 113 ---- tools/empirical-bench/memory_probe.rs | 101 ---- tools/empirical-bench/resource_probe.rs | 138 ----- tools/empirical-bench/run.py | 367 ------------- tools/empirical-bench/test_export.py | 51 -- 15 files changed, 92 insertions(+), 1799 deletions(-) delete mode 100644 crates/devtools/src/bin/o11y_exact_bench.rs delete mode 100644 crates/devtools/src/bin/o11y_replay.rs delete mode 100644 crates/devtools/src/bin/offline_recommend.rs delete mode 100644 tools/empirical-bench/.gitignore delete mode 100644 tools/empirical-bench/ARTIFACTS.md delete mode 100644 tools/empirical-bench/README.md delete mode 100644 tools/empirical-bench/memory_probe.rs delete mode 100644 tools/empirical-bench/resource_probe.rs delete mode 100644 tools/empirical-bench/run.py delete mode 100644 tools/empirical-bench/test_export.py diff --git a/Cargo.lock b/Cargo.lock index 2fe45ad6..34f106da 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -320,8 +320,6 @@ dependencies = [ "asap-frontend-promql", "asap-frontend-sql", "asap-types", - "libc", - "regex", "serde", "serde_json", "serde_yaml", diff --git a/crates/devtools/Cargo.toml b/crates/devtools/Cargo.toml index 748fae18..0abc8ccd 100644 --- a/crates/devtools/Cargo.toml +++ b/crates/devtools/Cargo.toml @@ -19,6 +19,4 @@ asap-types = { path = "../types" } serde = { version = "1", features = ["derive"] } serde_json = "1" serde_yaml = "0.9" -libc = "0.2" -regex = "1" tokio = { version = "1", features = ["rt", "macros", "rt-multi-thread"] } diff --git a/crates/devtools/src/bin/o11y_exact_bench.rs b/crates/devtools/src/bin/o11y_exact_bench.rs deleted file mode 100644 index e0cd9bbd..00000000 --- a/crates/devtools/src/bin/o11y_exact_bench.rs +++ /dev/null @@ -1,367 +0,0 @@ -//! Fixed-snapshot exact summary profiling for a deliberately bounded o11y subset. -use std::{hint::black_box, rc::Rc}; - -use asap_aware_mapping::cost_model::Cost; -use asap_aware_mapping::{ - default_strategies_with, search_workload_with_targets, CostModel, DefaultAccuracyModel, - DefaultCostModel, Replacement, ReplacementStrategy, ReplacementSubDAG, SketchAlgorithmStrategy, - TargetSubDAG, -}; -use asap_devtools::lower_promql; -use asap_types::{ - post_asap::SketchAlgorithm, - pre_asap::{AggIntent, QueryExpr}, - types::AccuracyTarget, -}; -use serde_json::{json, Value}; - -const CORPUS: &str = - include_str!("../../../frontend-promql/tests/observability/data/o11y_bench_promql.txt"); - -#[derive(Clone, Copy, Debug)] -enum Kernel { - SumInstant, - MaxWindow, - SumAverageWindow, -} - -struct Fixture { - query: &'static str, - kernel: Kernel, - range_seconds: u64, - job_matcher: Option, -} - -// Admit pinned fixture queries whose full value computation is implemented. -// Regex construction belongs to planning, not execution, on both paths. -fn fixture(query: &'static str) -> Option { - let (kernel, range_seconds, matcher) = match query { - "sum(process_resident_memory_bytes)" | "sum(up)" => (Kernel::SumInstant, 0, None), - "max_over_time(service_cache_refresh_lag_seconds{job=\"user-service\"}[12h])" => { - (Kernel::MaxWindow, 43200, Some("^user-service$")) - } - "max_over_time(service_retry_queue_depth{job=\"order-service\"}[6h])" => { - (Kernel::MaxWindow, 21600, Some("^order-service$")) - } - "max_over_time(service_retry_queue_depth{job=\"payment-service\"}[6h])" => { - (Kernel::MaxWindow, 21600, Some("^payment-service$")) - } - "max_over_time(service_retry_queue_depth{job=~\".+\"}[6h])" => { - (Kernel::MaxWindow, 21600, Some("^.+$")) - } - "sum(avg_over_time(process_resident_memory_bytes{job=~\".+\"}[6h]))" => { - (Kernel::SumAverageWindow, 21600, Some("^.+$")) - } - _ => return None, - }; - Some(Fixture { - query, - kernel, - range_seconds, - job_matcher: matcher.map(|m| regex::Regex::new(m).unwrap()), - }) -} - -struct Series { - job: &'static str, - samples: Vec, -} - -fn data(fixture: &Fixture, series: usize) -> Vec { - let samples = if fixture.range_seconds == 0 { - 1 - } else { - fixture.range_seconds as usize / 60 - }; - (0..series) - .map(|s| Series { - job: ["user-service", "order-service", "payment-service"][s % 3], - samples: (0..samples) - .map(|t| { - if fixture.query == "sum(up)" { - if s % 11 == 0 { - 0.0 - } else { - 1.0 - } - } else { - ((s * 73 + t * 31 + (s * t) % 17) % 10000) as f64 / 100.0 - } - }) - .collect(), - }) - .collect() -} - -fn raw(fixture: &Fixture, data: &[Series]) -> Vec { - let selected = data.iter().filter(|s| { - fixture - .job_matcher - .as_ref() - .is_none_or(|r| r.is_match(s.job)) - }); - match fixture.kernel { - Kernel::SumInstant => vec![selected.map(|s| s.samples[0]).sum()], - Kernel::MaxWindow => selected - .map(|s| s.samples.iter().copied().fold(f64::NEG_INFINITY, f64::max)) - .collect(), - Kernel::SumAverageWindow => vec![selected - .map(|s| s.samples.iter().sum::() / s.samples.len() as f64) - .sum()], - } -} - -#[cfg(unix)] -fn cpu_ns() -> u64 { - let mut ts = libc::timespec { - tv_sec: 0, - tv_nsec: 0, - }; - // A process CPU clock excludes scheduling waits; no wall-time conversion. - assert_eq!( - unsafe { libc::clock_gettime(libc::CLOCK_PROCESS_CPUTIME_ID, &mut ts) }, - 0 - ); - ts.tv_sec as u64 * 1_000_000_000 + ts.tv_nsec as u64 -} - -fn measure(mut op: impl FnMut(), iterations: usize) -> Value { - for _ in 0..2 { - op(); - } - let samples: Vec<_> = (0..5) - .map(|_| { - let start = cpu_ns(); - for _ in 0..iterations { - op(); - } - (cpu_ns() - start) as f64 / iterations as f64 - }) - .collect(); - let mean = samples.iter().sum::() / samples.len() as f64; - let stddev = (samples.iter().map(|x| (x - mean).powi(2)).sum::() - / (samples.len() - 1) as f64) - .sqrt(); - json!({"mean_cpu_ns":mean, "stddev_cpu_ns":stddev, "samples_cpu_ns":samples, "iterations_per_sample":iterations}) -} - -// This reference deployment admits exactly the root it profiled. CPU values -// apply to the complete bounded in-memory kernel, never an interior group. -struct ProfiledModel<'a> { - root: &'a QueryExpr, - approved: Vec>, - raw_cpu: f64, - candidate_cpu: f64, -} - -fn approved_profiles(root: &Rc) -> Vec> { - SketchAlgorithmStrategy::new(&DefaultCostModel) - .replacements(&TargetSubDAG::new(root)) - .into_iter() - .filter_map(|candidate| match candidate.replacement { - Replacement::Summary(node) - if matches!( - node.expr, - asap_types::post_asap::SummaryExpr::SummaryAgg { .. } - ) && node.guarantee.as_ref().is_some_and(|g| g.is_exact()) => - { - Some(node) - } - _ => None, - }) - .collect() -} - -impl CostModel for ProfiledModel<'_> { - fn rank_candidates( - &self, - intent: &AggIntent, - candidates: &[SketchAlgorithm], - ) -> Vec { - DefaultCostModel.rank_candidates(intent, candidates) - } - fn candidate_cost_covers_complete_plan(&self) -> bool { - true - } - fn candidate_cost( - &self, - candidate: &ReplacementSubDAG, - target: &TargetSubDAG<'_>, - ) -> Option { - let Replacement::Summary(node) = &candidate.replacement else { - return None; - }; - if target.root.as_ref() != self.root || !self.approved.contains(node) { - return None; - } - (self.candidate_cpu < self.raw_cpu).then_some(Cost(self.candidate_cpu)) - } - fn estimate_cost(&self, candidate: &ReplacementSubDAG, target: &TargetSubDAG<'_>) -> f64 { - self.candidate_cost(candidate, target) - .map_or(f64::INFINITY, |c| c.0) - } -} - -fn run(fixture: &Fixture, series: usize, evaluations: usize) -> Value { - let root = Rc::new(lower_promql(fixture.query, AccuracyTarget::Exact).unwrap()); - let data = data(fixture, series); - let cached = raw(fixture, &data); - // Same ordered per-series output, or the same scalar; immutable fixed - // windows, no staleness, NaNs, counter resets, or advancing evaluation time. - assert_eq!(raw(fixture, &data), cached.clone()); - let raw_measurement = measure( - || { - black_box(raw(black_box(fixture), black_box(&data))); - }, - 25, - ); - let read_measurement = measure( - || { - black_box(black_box(&cached).clone()); - }, - 10000, - ); - let raw_per_eval = raw_measurement["mean_cpu_ns"].as_f64().unwrap(); - let read_per_eval = read_measurement["mean_cpu_ns"].as_f64().unwrap(); - // Building a fixed-window summary executes the identical raw kernel once. - // Both timers include output allocation and destruction; one-time build - // therefore conservatively includes a destruction absent during retention. - let raw_total = raw_per_eval * evaluations as f64; - let summary_total = raw_per_eval + read_per_eval * evaluations as f64; - let model = ProfiledModel { - root: &root, - approved: approved_profiles(&root), - raw_cpu: raw_total, - candidate_cpu: summary_total, - }; - let candidates = SketchAlgorithmStrategy::new(&model).replacements(&TargetSubDAG::new(&root)); - let accepted = candidates - .iter() - .filter(|c| model.candidate_cost(c, &TargetSubDAG::new(&root)).is_some()) - .count(); - let space = search_workload_with_targets( - vec![(0, root.clone(), Some(AccuracyTarget::Exact))], - &default_strategies_with(&model), - &DefaultAccuracyModel, - ); - let selection = space.global_selection(&model); - let selected_root = &space.roots[0].1; - let selected_plan = selection.materialize(selected_root).unwrap(); - let selected_summary = selected_plan - .as_ref() - .is_some_and(|node| model.approved.contains(node)); - let retained_value_bytes = cached.len() * std::mem::size_of::(); - json!({"query":fixture.query,"status":"profiled_fixed_snapshot", "kernel":format!("{:?}",fixture.kernel), - "profile_model_version":"fixed-snapshot-exact-values-v1", "planner_candidate_count":candidates.len(), - "accepted_measured_candidates":accepted, - "selected":if selected_summary {"retained_exact_summary"} else {"raw_recompute"}, - "selected_planner_graph":selected_plan.as_ref().map(|node|asap_types::dag_export::export_summary(node)), - "input_series":series,"selected_series":data.iter().filter(|s|fixture.job_matcher.as_ref().is_none_or(|r|r.is_match(s.job))).count(), - "samples_per_series":data[0].samples.len(),"sample_interval_seconds":60,"range_seconds":fixture.range_seconds, - "input_value_bytes":data.iter().map(|s|s.samples.len()*8).sum::(), - "retained_value_bytes":retained_value_bytes,"retained_value_bytes_method":"exact Vec length times sizeof(f64), logical values only; label/input storage remains shared", - "raw_per_evaluation":raw_measurement,"summary_read_per_evaluation":read_measurement, - "evaluations":evaluations,"raw_cpu_ns":raw_total,"summary_cpu_ns":summary_total, - "estimated_cpu_reduction_fraction":if selected_summary {Some(1.0-summary_total/raw_total)} else {None}, - "output_equality_verified":true, - "comparison_scope":"same immutable metric snapshot and fixed evaluation time; retained whole-query exact result reference implementation including filtering, reduction, allocation and readout", - "exclusions":["disk/network storage and protocol serialization", "label output materialization shared by both paths", "live updates, eviction, sliding windows, staleness and exceptional samples"]}) -} - -fn main() -> Result<(), Box> { - let args: Vec<_> = std::env::args().skip(1).collect(); - let series: usize = args.first().map_or(Ok(300), |x| x.parse())?; - let evaluations: usize = args.get(1).map_or(Ok(60), |x| x.parse())?; - if args.len() > 2 || series == 0 || series > 10000 || evaluations == 0 { - return Err("usage: o11y_exact_bench [SERIES(1..10000)] [EVALUATIONS>0]".into()); - } - let rows: Vec<_> = CORPUS.lines().map(str::trim).filter(|q|!q.is_empty()&&!q.starts_with('#')).map(|q| - fixture(q).map_or_else(||json!({"query":q,"status":"unavailable","reason":"no complete reference kernel for this query; no borrowed costs"}), |f|run(&f,series,evaluations))).collect(); - let cpu = std::fs::read_to_string("/proc/cpuinfo") - .ok() - .and_then(|text| { - text.lines() - .find(|line| line.starts_with("model name")) - .map(str::to_owned) - }); - let revision = std::process::Command::new("git") - .args(["rev-parse", "HEAD"]) - .output() - .ok() - .filter(|out| out.status.success()) - .map(|out| String::from_utf8_lossy(&out.stdout).trim().to_owned()); - serde_json::to_writer_pretty( - std::io::stdout(), - &json!({"schema_version":1, - "data":"synthetic deterministic finite gauges; three jobs; non-stale finite float samples on (T-window,T] every60s", - "implementation":"o11y_exact_bench Rust reference kernels; not production data-plane runtime", - "build_profile":if cfg!(debug_assertions){"debug"}else{"release"}, - "source_revision":revision,"source_file":"crates/devtools/src/bin/o11y_exact_bench.rs", - "command_arguments":args,"cpu":cpu,"os":std::env::consts::OS,"arch":std::env::consts::ARCH, - "timing_environment":"shared development host; no exclusive core reservation or CPU affinity", - "clock":"CLOCK_PROCESS_CPUTIME_ID","rows":rows}), - )?; - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - #[test] - fn admission_is_explicit_and_unknown_semantics_stay_unavailable() { - assert_eq!(CORPUS.lines().filter(|q| fixture(q).is_some()).count(), 7); - assert!(fixture("sum(rate(http_requests_total[5m]))").is_none()); - } - #[test] - fn kernels_match_independent_small_reference_results() { - let input = vec![ - Series { - job: "order-service", - samples: vec![2.0, 4.0], - }, - Series { - job: "payment-service", - samples: vec![8.0, 10.0], - }, - ]; - let max = - fixture("max_over_time(service_retry_queue_depth{job=\"order-service\"}[6h])").unwrap(); - assert_eq!(raw(&max, &input), vec![4.0]); - let avg = - fixture("sum(avg_over_time(process_resident_memory_bytes{job=~\".+\"}[6h]))").unwrap(); - assert_eq!(raw(&avg, &input), vec![12.0]); - } - #[test] - fn measured_scope_and_break_even_control_candidate_acceptance() { - let root = Rc::new(lower_promql("sum(up)", AccuracyTarget::Exact).unwrap()); - let target = TargetSubDAG::new(&root); - let candidates = SketchAlgorithmStrategy::new(&DefaultCostModel).replacements(&target); - assert!(!candidates.is_empty()); - let slow = ProfiledModel { - root: &root, - approved: approved_profiles(&root), - raw_cpu: 100.0, - candidate_cpu: 101.0, - }; - let fast = ProfiledModel { - root: &root, - approved: approved_profiles(&root), - raw_cpu: 100.0, - candidate_cpu: 90.0, - }; - assert!(slow.candidate_cost(&candidates[0], &target).is_none()); - assert_eq!( - fast.candidate_cost(&candidates[0], &target), - Some(Cost(90.0)) - ); - let other = Rc::new(lower_promql("sum(other)", AccuracyTarget::Exact).unwrap()); - assert!(fast - .candidate_cost(&candidates[0], &TargetSubDAG::new(&other)) - .is_none()); - let mut unprofiled = candidates[0].clone(); - let mut node = fast.approved[0].as_ref().clone(); - node.expr = asap_types::post_asap::SummaryExpr::KeepPreAsap(root.clone()); - unprofiled.replacement = Replacement::Summary(Rc::new(node)); - assert!(fast.candidate_cost(&unprofiled, &target).is_none()); - } -} diff --git a/crates/devtools/src/bin/o11y_replay.rs b/crates/devtools/src/bin/o11y_replay.rs deleted file mode 100644 index 579f586d..00000000 --- a/crates/devtools/src/bin/o11y_replay.rs +++ /dev/null @@ -1,495 +0,0 @@ -//! Offline workload replay through search, selection, and lifecycle planning. -use std::{collections::HashSet, error::Error, rc::Rc, time::Instant}; - -use asap_aware_mapping::empirical_cost::{ - EmpiricalCostModel, EmpiricalEvidenceProvider, EvidenceArtifact, EvidenceContext, -}; -use asap_aware_mapping::{ - default_strategies_with, export_summary_maintenance_plan, - materialize_with_summary_maintenance_lifecycles, search_workload_with_targets, CostModel, - DefaultAccuracyModel, DefaultCostModel, Horizon, Replacement, - SummaryMaintenanceLifecycleCapabilities, WorkloadDemand, -}; -use asap_devtools::lower_promql; -use asap_types::{ - dag_export, - post_asap::{SummaryExpr, SummaryFamilyType, SummaryNode}, - pre_asap::QueryExpr, - types::AccuracyTarget, - workload::*, -}; -use serde_json::{json, Value}; - -const CORPUS: &str = - include_str!("../../../frontend-promql/tests/observability/data/o11y_bench_promql.txt"); -const SUPPLEMENTAL: &str = "quantile_over_time(0.95, service_latency_seconds[5m])\nquantile_over_time(0.99, service_latency_seconds[5m])\ncount_over_time(offline_frequency_metric[5m])"; - -struct Options { - epsilon: f64, - evaluations: u64, - now_ms: u64, - supplemental: bool, - queries: Option, - evidence_path: Option, - context_path: Option, -} - -impl Default for Options { - fn default() -> Self { - Self { - epsilon: 0.01, - evaluations: 60, - now_ms: 1_788_825_600_000, - supplemental: false, - queries: None, - evidence_path: None, - context_path: None, - } - } -} - -fn parse_options(args: impl IntoIterator) -> Result> { - let mut options = Options::default(); - let mut args = args.into_iter(); - while let Some(arg) = args.next() { - match arg.as_str() { - "--supplemental" => options.supplemental = true, - "--epsilon" => options.epsilon = args.next().ok_or("missing epsilon")?.parse()?, - "--evaluations" => options.evaluations = args.next().ok_or("missing evaluations")?.parse()?, - "--now-ms" => options.now_ms = args.next().ok_or("missing now-ms")?.parse()?, - "--queries" => options.queries = Some(std::fs::read_to_string(args.next().ok_or("missing queries path")?)?), - "--evidence" => options.evidence_path = Some(args.next().ok_or("missing evidence path")?), - "--context" => options.context_path = Some(args.next().ok_or("missing context path")?), - _ => return Err(format!("unknown option {arg}; use --epsilon, --evaluations, --now-ms, --queries, --supplemental, --evidence and --context").into()), - } - } - if !options.epsilon.is_finite() - || !(0.0..1.0).contains(&options.epsilon) - || options.epsilon == 0.0 - { - return Err("epsilon must be finite and strictly between 0 and 1".into()); - } - if options.evaluations == 0 { - return Err("evaluations must be positive".into()); - } - if options.evidence_path.is_some() != options.context_path.is_some() { - return Err("--evidence and --context must be supplied together".into()); - } - Ok(options) -} - -fn queries(text: &str) -> Vec { - text.lines() - .map(str::trim) - .filter(|q| !q.is_empty() && !q.starts_with('#')) - .map(str::to_owned) - .collect() -} - -// Attribute memo groups by DAG identity after workload-wide CSE. Scalar -// expressions aren't replacement sites; they remain part of their operator. -fn reachable(node: &QueryExpr, found: &mut HashSet<*const QueryExpr>) { - if !found.insert(node as *const QueryExpr) { - return; - } - use QueryExpr::*; - match node { - PromqlVectorFromScalar(child) - | PromqlScalarFromVector(child) - | PromqlRelabel { child, .. } - | PromqlInfoEnrich { child, .. } - | PromqlSeriesSample { child, .. } - | Filter { child, .. } - | Project { child, .. } - | Aggregate { child, .. } - | Dedup { child, .. } - | PromqlSubquery { child, .. } - | TimeRange { child, .. } - | TimeShift { child, .. } - | SQLWindowFunc { child, .. } - | Sort { child, .. } - | Limit { child, .. } => reachable(child, found), - Concat { children, .. } => { - for child in children { - reachable(child, found); - } - } - Join { left, right, .. } | SetOp { left, right, .. } => { - reachable(left, found); - reachable(right, found); - } - BinaryOp { lhs, rhs, .. } => { - reachable(lhs, found); - reachable(rhs, found); - } - _ => {} - } -} - -fn family_report( - family: &SummaryFamilyType, - provider: Option<&EmpiricalEvidenceProvider>, -) -> Value { - let evidence = if let SummaryFamilyType::Sketch(kind, _) = family { - match provider { - Some(provider) => match provider.lookup(kind.algorithm(), kind.params()) { - Ok(row) => json!({"status": "matched_configuration", "measurement": row, - "scope": "offline benchmark primitive; error/read query semantics are recorded in measurement.error.query, not validated against replay query"}), - Err(error) => json!({"status": "unavailable", "reason": error.to_string()}), - }, - None => { - json!({"status": "unavailable", "reason": "no empirical provider in this mode"}) - } - } - } else { - json!({"status": "unavailable", "reason": "offline sketch evidence does not cost exact operators"}) - }; - let sketch = match family { - SummaryFamilyType::Sketch(kind, _) => { - json!({"algorithm": kind.algorithm(), "params": kind.params()}) - } - _ => Value::Null, - }; - json!({"family": format!("{family:?}"), "sketch": sketch, - "is_sketch": matches!(family, SummaryFamilyType::Sketch(..)), "offline_evidence": evidence}) -} - -fn families( - node: &SummaryNode, - result: &mut Vec, - provider: Option<&EmpiricalEvidenceProvider>, -) { - use SummaryExpr::*; - match &node.expr { - SummaryAgg { family, child, .. } => { - result.push(family_report(family, provider)); - families(child, result, provider); - } - SummaryJoin { - family, - outer, - inner, - .. - } => { - result.push(family_report(family, provider)); - families(outer, result, provider); - families(inner, result, provider); - } - SummaryEstimate { summary_input, .. } | SummaryDelete { summary_input, .. } => { - families(summary_input, result, provider) - } - BinaryOp { lhs, rhs, .. } => { - families(lhs, result, provider); - families(rhs, result, provider); - } - SummarySubtract { left, right } => { - families(left, result, provider); - families(right, result, provider); - } - SummaryMerge { children } => { - for child in children { - families(child, result, provider); - } - } - KeepPreAsap(_) => {} - } -} - -fn replay( - name: &str, - query_texts: &[String], - accuracy: AccuracyTarget, - options: &Options, - model: &dyn CostModel, - provider: Option<&EmpiricalEvidenceProvider>, -) -> Value { - let lowering_start = Instant::now(); - let mut roots = Vec::new(); - let mut rows = Vec::new(); - for (index, query) in query_texts.iter().enumerate() { - match lower_promql(query, accuracy.clone()) { - Ok(root) => roots.push((index, Rc::new(root), Some(accuracy.clone()))), - Err(error) => rows.push(json!({"index": index, "query": query, "coverage": "rejected", "reason": error.to_string()})), - } - } - let lowering_ns = lowering_start.elapsed().as_nanos(); - let search_start = Instant::now(); - let space = search_workload_with_targets( - roots, - &default_strategies_with(model), - &DefaultAccuracyModel, - ); - let search_ns = search_start.elapsed().as_nanos(); - let selection_start = Instant::now(); - let ranked = space.cost_sorted(model); - let selection = space.global_selection(model); - let selection_ns = selection_start.elapsed().as_nanos(); - let workload = QueryWorkload { - language: QueryLanguage::PromQL, - query_batch: Some( - query_texts - .iter() - .map(|q| BatchEntry { - query: Query(q.clone()), - requirements: QueryRequirements { - accuracy: AccuracyRequirement::Explicit(accuracy.clone()), - ..Default::default() - }, - predictability: Predictability::AdHoc, - invocations: options.evaluations, - execute_at: Some(TimestampMs(options.now_ms)), - time_selection: TimeSelection { - as_of: Some(TimestampMs(options.now_ms)), - ..Default::default() - }, - }) - .collect(), - ), - repeating_queries: None, - data_workload: Some(DataWorkload { - arrival: DataArrival::AtRest, - ..Default::default() - }), - }; - for (index, root) in &space.roots { - let started = Instant::now(); - let mut found = HashSet::new(); - reachable(root, &mut found); - let mut candidates = Vec::new(); - let mut rejected = Vec::new(); - for (group_index, group) in ranked - .iter() - .enumerate() - .filter(|(_, g)| found.contains(&Rc::as_ptr(g.target))) - { - let target_intent = asap_aware_mapping::replacement::bindable_intent(group.target) - .map(|intent| format!("{intent:?}")); - for (rank, candidate) in group.candidates.iter().enumerate() { - let mut family_list = Vec::new(); - if let Replacement::Summary(node) = &candidate.replacement { - families(node, &mut family_list, provider); - } - candidates.push( - json!({"group": group_index, "rank": rank, "strategy": candidate.strategy, - "target_intent": target_intent, - "rationale": candidate.rationale, "families": family_list, - "heuristic_score": group.costs[rank].is_finite().then_some(group.costs[rank]), - "score_unit": "dimensionless_not_cpu", "consumer_count": group.consumer_count}), - ); - } - if let Some(memo) = space.group_for(group.target) { - rejected.extend(memo.rejected.iter().map(|r| json!({"group": group_index, "strategy": r.strategy, "description": r.description, "reason": r.error.to_string()}))); - } - } - let has_sketch = candidates.iter().any(|c| { - c["families"] - .as_array() - .unwrap() - .iter() - .any(|f| f["is_sketch"] == true) - }); - let has_summary = candidates - .iter() - .any(|c| !c["families"].as_array().unwrap().is_empty()); - let materialized = selection.materialize(root); - let root_plan = match materialized { - Ok(Some(node)) => { - json!({"graph": dag_export::export_summary(&node), "guarantee": node.guarantee}) - } - Ok(None) => json!({"reason": "no selected root group"}), - Err(error) => json!({"reason": error.to_string()}), - }; - let lifecycle = match materialize_with_summary_maintenance_lifecycles( - &selection, - root, - WorkloadDemand::new(&workload, &[*index]), - options.now_ms, - Some(Horizon(3600.0)), - SummaryMaintenanceLifecycleCapabilities::ALL, - model, - ) { - Ok(Some(plan)) => json!(export_summary_maintenance_plan(&plan)), - Ok(None) => json!({"reason": "no selected root group", "selected_raw_recompute": true}), - Err(error) => json!({"reason": error.to_string(), "selected_raw_recompute": true}), - }; - rows.push(json!({"index": index, "query": query_texts[*index], - "coverage": if has_sketch { "sketch_candidate" } else if has_summary { "exact_summary_candidate" } else { "exact_fallback" }, - "coverage_scope": "reachable_candidate_sites_not_whole_query_execution", - "candidates": candidates, "rejections": rejected, "root_plan": root_plan, "lifecycle_plan": lifecycle, - "report_and_materialization_ns": started.elapsed().as_nanos(), - "estimated_end_to_end_cpu_savings": null, "estimated_end_to_end_memory_savings": null, - "unknown_cost_reason": "complete raw scan, residual operators, grouping cardinalities and deployment costs are unavailable"})); - } - rows.sort_by_key(|row| row["index"].as_u64()); - let mut coverage_counts = std::collections::BTreeMap::new(); - for row in &rows { - *coverage_counts - .entry(row["coverage"].as_str().unwrap_or("unknown")) - .or_insert(0) += 1; - } - let raw_fallback_count = rows - .iter() - .filter(|row| row["lifecycle_plan"]["selected_raw_recompute"] == true) - .count(); - json!({"mode": name, "accuracy": accuracy, "lowering_ns": lowering_ns, "search_ns": search_ns, - "selection_ns": selection_ns, "query_count": query_texts.len(), "memo_groups": space.len(), - "coverage_counts": coverage_counts, "lifecycle_raw_fallback_count": raw_fallback_count, "queries": rows}) -} - -fn report(options: &Options, empirical: Option<&EmpiricalCostModel>) -> Value { - let mut corpora = vec![( - if options.queries.is_some() { - "custom" - } else { - "o11y_bench" - }, - queries(options.queries.as_deref().unwrap_or(CORPUS)), - )]; - if options.supplemental { - corpora.push(("supplemental_sketch_queries", queries(SUPPLEMENTAL))); - } - let corpora: Vec = corpora - .into_iter() - .map(|(name, queries)| { - let mut runs = vec![ - replay( - "exact", - &queries, - AccuracyTarget::Exact, - options, - &DefaultCostModel, - None, - ), - replay( - "default", - &queries, - AccuracyTarget::Epsilon(options.epsilon), - options, - &DefaultCostModel, - None, - ), - ]; - if let Some(model) = empirical { - runs.push(replay( - "empirical", - &queries, - AccuracyTarget::Epsilon(options.epsilon), - options, - model, - Some(&model.provider), - )); - } - json!({"name": name, "source": match name { - "o11y_bench" => "repository-vendored grafana/o11y-bench snapshot", - "custom" => "user-supplied PromQL query file", - _ => "local supplemental examples, not upstream o11y queries" - }, "runs": runs}) - }) - .collect(); - json!({"schema_version": 1, "execution_scope": "offline_planner_search_selection_and_lifecycle_no_data_plane", - "empirical_context": empirical.map(|m| m.provider.context()), - "empirical_artifact": empirical.map(|m| json!({"schema_version": m.provider.artifact().schema_version, - "benchmark_version": m.provider.artifact().benchmark_version, "model_version": m.provider.artifact().model_version})), - "vendored_fixture_source": {"repository": "https://github.com/grafana/o11y-bench", "vendored_snapshot_date": "2026-07-17", "upstream_commit": null, - "note": "existing 27-query local fixture; upstream revision and scenario data were not provided"}, - "assumptions": {"evaluation_time_ms": options.now_ms, "evaluations_per_query": options.evaluations, - "data_arrival": "at_rest", "replay_semantics": "repeated reads at one fixed as-of time; query windows and offsets remain in IR", - "lifecycle_horizon_seconds": 3600, "runtime_capabilities": "hypothetical all supported, no runtime launched", - "timing": "single wall-clock sample; report/materialization timing includes JSON export"}, - "corpora": corpora}) -} - -fn main() -> Result<(), Box> { - let options = parse_options(std::env::args().skip(1))?; - let empirical = match (&options.evidence_path, &options.context_path) { - (Some(evidence), Some(context)) => { - let artifact: EvidenceArtifact = - serde_json::from_str(&std::fs::read_to_string(evidence)?)?; - let context: EvidenceContext = - serde_json::from_str(&std::fs::read_to_string(context)?)?; - Some(EmpiricalCostModel::new(EmpiricalEvidenceProvider::new( - artifact, context, - )?)) - } - _ => None, - }; - println!( - "{}", - serde_json::to_string_pretty(&report(&options, empirical.as_ref()))? - ); - Ok(()) -} - -#[cfg(test)] -mod tests { - use super::*; - - /// Invalid CLI numbers must fail before they can poison cost estimates. - #[test] - fn rejects_invalid_configuration() { - for value in ["NaN", "0", "1", "-1"] { - assert!(parse_options(["--epsilon".into(), value.into()]).is_err()); - } - assert!(parse_options(["--evaluations".into(), "0".into()]).is_err()); - } - - /// Real workload lowering alone never counts as sketch or deployment coverage. - #[test] - fn o11y_replay_records_all_queries_and_conservative_costs() { - let result = report(&Options::default(), None); - for run in result["corpora"][0]["runs"].as_array().unwrap() { - assert_eq!(run["query_count"], 27); - assert!(run["memo_groups"].as_u64().unwrap() > 0); - for row in run["queries"].as_array().unwrap() { - assert_ne!(row["coverage"], "rejected"); - assert_ne!(row["coverage"], "sketch_candidate"); - assert!(row["estimated_end_to_end_cpu_savings"].is_null()); - assert_eq!(row["lifecycle_plan"]["selected_raw_recompute"], true); - } - } - } - - /// Supplemental sketches and rejected syntax remain separately identifiable. - #[test] - fn sketches_and_rejections_are_explicit() { - let qs = vec!["quantile_over_time(0.95, latency[5m])".into(), "!!!".into()]; - let result = replay( - "default", - &qs, - AccuracyTarget::Epsilon(0.01), - &Options::default(), - &DefaultCostModel, - None, - ); - assert_eq!(result["queries"][0]["coverage"], "sketch_candidate"); - assert_eq!(result["queries"][1]["coverage"], "rejected"); - assert!(result["queries"][1]["reason"].is_string()); - } - - /// Count-over-time exercises the measured CMS/CountSketch configuration, - /// while its evidence remains absent unless an artifact is supplied. - #[test] - fn supplemental_count_exposes_frequency_sketch_candidates() { - let result = replay( - "default", - &queries("count_over_time(metric[5m])"), - AccuracyTarget::Epsilon(0.01), - &Options::default(), - &DefaultCostModel, - None, - ); - let families: Vec<_> = result["queries"][0]["candidates"] - .as_array() - .unwrap() - .iter() - .flat_map(|c| c["families"].as_array().unwrap()) - .collect(); - for algorithm in ["Cms", "CountSketch"] { - let family = families - .iter() - .find(|f| f["sketch"]["algorithm"] == algorithm) - .unwrap(); - assert_eq!(family["offline_evidence"]["status"], "unavailable"); - } - } -} diff --git a/crates/devtools/src/bin/offline_recommend.rs b/crates/devtools/src/bin/offline_recommend.rs deleted file mode 100644 index 3f6333f0..00000000 --- a/crates/devtools/src/bin/offline_recommend.rs +++ /dev/null @@ -1,28 +0,0 @@ -//! Evaluate an explicit offline error/resource requirement against measured data. -use asap_aware_mapping::empirical_comparison::{ - recommend_offline, OfflineComparisonEvidence, OfflineComparisonRequest, -}; - -fn main() -> Result<(), Box> { - let args: Vec<_> = std::env::args().skip(1).collect(); - if args.len() != 2 { - return Err("usage: offline_recommend COMPARISON-EVIDENCE.json REQUEST.json".into()); - } - let evidence: OfflineComparisonEvidence = - serde_json::from_str(&std::fs::read_to_string(&args[0])?)?; - let request: OfflineComparisonRequest = - serde_json::from_str(&std::fs::read_to_string(&args[1])?)?; - let recommendation = recommend_offline(&evidence, &request)?; - serde_json::to_writer_pretty( - std::io::stdout(), - &serde_json::json!({ - "schema_version":1, - "benchmark_version":evidence.sketch_evidence.benchmark_version, - "model_version":evidence.sketch_evidence.model_version, - "request":request, - "recommendation":recommendation, - "accuracy_scope":"observed offline mean error on the exact declared probe population; not an unseen-data or realtime guarantee" - }), - )?; - Ok(()) -} diff --git a/docs/design_docs/empirical-o11y-execution-plan.md b/docs/design_docs/empirical-o11y-execution-plan.md index 0471529e..d7c55f03 100644 --- a/docs/design_docs/empirical-o11y-execution-plan.md +++ b/docs/design_docs/empirical-o11y-execution-plan.md @@ -18,8 +18,9 @@ do not establish formal guarantees on unseen distributions. 3. Benchmark agent: run actual sketch-bench algorithms on uniform and Zipf inputs, preserve raw output, export artifacts with source and environment provenance, and document reproducible commands. -4. Replay agent: run the existing o11y PromQL corpus through actual planning, - export candidate coverage and planning latency, and separately identify +4. Replay agent: run the existing o11y PromQL corpus through the backend parser, + its ASAPPlanner call, and the backend typed binder (not a planner-only entry), + export binding/fallback coverage and planning latency, and separately identify supplemental sketch workloads. 5. Integration owner: connect compatible evidence to the downstream control plane, validate dependency compatibility, review other agents' changes, run diff --git a/docs/offline-o11y-final-2026-09-08.md b/docs/offline-o11y-final-2026-09-08.md index 25f1b9f4..f361557c 100644 --- a/docs/offline-o11y-final-2026-09-08.md +++ b/docs/offline-o11y-final-2026-09-08.md @@ -1,4 +1,11 @@ -# Offline Sketch Evidence and o11y: Final Report +# Historical Offline Sketch Evidence and o11y Report + +This report records the earlier offline experiment, not the backend-only +prototype's execution results. Its synthetic exact-result cache experiment is +out of the current prototype scope; its source tool was removed from this PR. +Current work uses user-provided OpenMetrics data and queries entering the backend, +followed by ASAPPlanner and actual data-plane execution. No execution speedup or +resource reduction is established by the historical results below. Audience: developers. This report covers the offline evidence path for #322, without real-time error feedback. The original dirty working directory was left unchanged. @@ -12,7 +19,7 @@ without real-time error feedback. The original dirty working directory was left - o11y planner/control-plane replay and exact fixed-snapshot reference measurements cover seven explicitly supported queries. For interfaces and reproduction instructions, see the [evidence contract](developer_docs/offline-sketch-evidence.md), -[benchmark commands](../tools/empirical-bench/README.md), and [replay guide](user-guide/o11y-replay.md). +[backend benchmark commands](https://github.com/ProjectASAP/ASAPQuery-backend/blob/codex/empirical-o11y-322/tools/empirical-bench/README.md), and [backend-entry replay guide](user-guide/o11y-replay.md). ## Measured Results @@ -38,7 +45,7 @@ a separate System allocator probe; CPU measurements use jemalloc. These memory figures are not process RSS. Serialized size and allocated file blocks do not establish disk-throughput benefits. -See `results-sweep/MEASUREMENTS.md` in the [verified experiment archive](../tools/empirical-bench/ARTIFACTS.md). +See `results-sweep/MEASUREMENTS.md` in the [verified experiment archive](https://github.com/ProjectASAP/ASAPQuery-backend/blob/codex/empirical-o11y-322/tools/empirical-bench/ARTIFACTS.md). Earlier frequency measurements and control-plane replay in `results/` remain as the initial baseline. Final frequency comparisons and control-plane results use `results-sweep/`; costs from the two runs must not be mixed. @@ -48,6 +55,11 @@ use `results-sweep/`; costs from the two runs must not be mixed. The replay reuses the repository's existing 27 PromQL fixtures. It does not run upstream LLM-agent scoring or use real scenario data. +The supported evaluation entry point is now the backend's `offline_planner_replay`: +queries enter its parser, call ASAPPlanner, and return through its typed binder. +The standalone planner-only replay was removed. Its candidate/lifecycle counts +below remain historical diagnostics, not backend support coverage. + | Check | Result | | --- | --- | | Planner candidate coverage | 26/27 have exact summary candidates; 1/27 uses raw fallback; no sketch candidates | diff --git a/docs/user-guide/o11y-replay.md b/docs/user-guide/o11y-replay.md index 1a3692b3..b6e4ab71 100644 --- a/docs/user-guide/o11y-replay.md +++ b/docs/user-guide/o11y-replay.md @@ -1,106 +1,85 @@ -# Offline o11y workload replay +# Offline o11y evaluation through the backend -For users evaluating planner coverage using offline sketch-bench evidence. -`o11y_replay` runs the existing 27-query o11y fixture through lowering, -workload-wide candidate search, accuracy checks, selection and summary lifecycle -planning. It launches no data plane or downstream control-plane server. +For developers evaluating the actual control-plane planning integration. +The canonical path is: -```sh -cargo run -p asap-devtools --bin o11y_replay -- --supplemental > replay.json -cargo run -p asap-devtools --bin o11y_replay -- \ - --supplemental --evidence planner-evidence.json --context context.json > empirical-replay.json -``` +`PromQL corpus -> ASAPQuery-backend parser -> ASAPPlanner -> backend typed binder/fallback` + +There is no standalone planner-only `o11y_replay` command. Run the backend's +`offline_planner_replay` example through its reproducibility wrapper instead. + +## Run the control-plane replay -The first command compares exact requirements with the default approximate -planner (`--epsilon 0.01`). The second adds an empirical planning run using a -versioned offline evidence artifact and an explicit applicability context. -The context JSON contains `distribution`, `environment`, and -`now_unix_seconds`; copy the full descriptors of the intended offline dataset -and execution environment, and choose the evidence evaluation time explicitly. -Configuration, distribution, environment and validity dates must match. -An absent measurement stays unavailable and falls back to default planning. - -`--queries FILE` accepts one PromQL query per nonblank, noncomment line. -`--evaluations 60` models repeated reads of the same fixed historical input; -`--now-ms 1788825600000` fixes the as-of time. These are scenario assumptions, -not timings extracted from o11y task execution. Query windows, subqueries and -offsets remain encoded in the query IR. The lifecycle horizon is one hour and -the assumed data is at rest. `--supplemental` adds two quantile queries and a -`count_over_time` query as a separate corpus. Its total sample count has different -readout semantics from integer-stream point-frequency sketch-bench data; matching -sketch configuration does not establish matching query readout/error semantics. - -The JSON distinguishes: - -- `coverage`: whether a reachable site has a sketch candidate, exact summary - candidate, exact fallback, or a lowering rejection. Candidate coverage does - not establish that the whole query can execute from summaries. -- `root_plan` and `lifecycle_plan`: selected root structure and deployable - lifecycle result. Missing complete costs can make lifecycle planning retain - exact recomputation despite available sketch candidates. -- `offline_evidence`: matched primitive measurements and provenance, or an - explicit missing/incompatible/stale reason. Error statistics remain offline - observations for the measurement's recorded query, separate from the - planner's formal result guarantee. -- `heuristic_score`: a dimensionless planner score, never CPU nanoseconds. - Search and selection wall-clock timings are one local sample. Reporting and - materialization timings include JSON export work. -- End-to-end resource savings: `null` until complete raw execution, residual - operators, grouping cardinalities, sharing and deployment costs exist. Do not - divide heuristic scores to claim runtime speedup, or sum nested candidate - costs as a whole-query cost. - -The fixture identifies an upstream snapshot date, not an upstream commit; this -tool retains that limitation explicitly. It reuses the repository's existing -fixture, whose provenance is documented there. It does not claim to run the -upstream agent benchmark or its scenario data. - -## Measure supported exact-query reference implementations +From an ASAPQuery-backend checkout, with its normal sibling dependencies available: ```sh -cargo run --release -p asap-devtools --bin o11y_exact_bench -- 300 60 > exact-snapshot.json +python3 tools/run-offline-planner-replay.py \ + --queries /path/to/ASAPPlanner/crates/frontend-promql/tests/observability/data/o11y_bench_promql.txt \ + --evidence /path/to/planner-evidence.json \ + --context /path/to/context.json \ + --output /tmp/control-plane-o11y.json ``` -This profiles seven explicitly admitted o11y queries: instantaneous sums, -per-series window maxima, and a sum of per-series window averages. The synthetic -dataset has 300 series across three job labels, finite gauge values, and one -sample per minute inside the selected window. The 60 invocations repeat the -identical immutable snapshot and evaluation time. They are not advancing live -windows. Unsupported queries are reported as unavailable. - -The reference deployment retains the complete exact result of an admitted root -summary. It measures the raw value kernel and the retained result read separately -with a process CPU clock, and estimates one build plus repeated reads against -repeated raw execution. Both timed kernels include result allocation/destruction; -charging the build this way is conservative because its result is actually -retained. Only the profiled exact SummaryAgg signatures receive costs through -the public CostModel boundary; actual search, global selection, and root -materialization determine the reported choice. Unprofiled candidates and raw -fallback nodes never inherit those costs. - -The output identifies this as a Rust reference implementation, not the deployed -backend. It includes filtering and value reduction/readout but excludes disk, -network, protocol serialization and shared output-label materialization. Memory -is logical stored value bytes, not measured process RSS or a claim that raw data -can be deleted. Large savings from reusing identical results do not establish -the benefit of live sliding-window maintenance. - -## Compare query-matched sketch configurations - -First restore the saved inputs using the [artifact download instructions](../../tools/empirical-bench/ARTIFACTS.md), -or generate them with the benchmark driver. Result files are not checked in. +The backend uses its pinned planner dependency; a planner source checkout is +only needed here to locate the vendored query fixture. Omit evidence and context +together to run exact/default modes only. Supplying both adds empirical mode. +An optional `--planner /path/to/ASAPPlanner` enables a local source override for +development; it is not the default evaluation path. + +The [backend guide](https://github.com/ProjectASAP/ASAPQuery-backend/blob/codex/empirical-o11y-322/control_plane/docs/offline-sketch-evidence.md) +documents the actual parser/binder and evidence assumptions. +The [benchmark driver](https://github.com/ProjectASAP/ASAPQuery-backend/blob/codex/empirical-o11y-322/tools/empirical-bench/README.md) +and [artifact instructions](https://github.com/ProjectASAP/ASAPQuery-backend/blob/codex/empirical-o11y-322/tools/empirical-bench/ARTIFACTS.md) +also live in the backend repository. + +## Interpret coverage conservatively + +- A parse failure is not a planner or backend execution result. +- A bound plan means the backend accepted a plan returned through its planner integration. +- `root_raw_fallback` means the complete original query remains selected; this is not summary acceleration. +- `raw_subtrees` includes source scans underneath summaries and cannot be counted as root fallback. +- `planning_elapsed_ns` measures the parser/binder path, not query execution. +- Binding does not establish deployable placement, successful data-plane execution, or resource savings. + +The recorded final evaluation bound all 27 fixtures in exact/default/empirical +modes; each mode retained five raw root fallbacks. Earlier standalone planner +candidate counts are historical diagnostics, not backend support coverage. +No collectors, storage services, query servers or upstream agent scoring run. +Point-frequency error measurements do not establish error bounds for these +PromQL queries. + +## Backend-only execution prototype + +The target is real data-plane execution, not just binding coverage: +OpenMetrics data and o11y queries enter ASAPQuery-backend; the backend calls +ASAPPlanner and executes either a valid ASAP plan or a local exact baseline. +No ASAPCollector or Prometheus process is required by the intended standalone +prototype. The implementation and its current limitations belong to +[backend PR #524](https://github.com/ProjectASAP/ASAPQuery-backend/pull/524). + +The supplied dataset is `/mydata/metrics.txt`, an OpenMetrics file with 305,026 +samples across 106 series. Preserve it read-only. The backend adapter owns +timestamp conversion and Remote Write encoding; OpenMetrics text cannot be +posted directly to `/api/v1/write`. The currently pinned upstream task corpus +has 28 query occurrences (24 unique queries); do not equate that corpus with the +older 27-query fixture used for the historical binding report. + +Acceptance requires matching exact/ASAP query results before reporting latency +or resource reductions, with identical data, labels and evaluation times. +Report construction/ingestion costs separately from steady-state queries. +Unsupported queries and exact fallbacks are not accelerated successes. +Do not substitute the earlier synthetic result-cache experiment for this test. + +## Optional backend diagnostic + +The small `offline_recommend` diagnostic also lives in ASAPQuery-backend. +It compares measured fixed-snapshot frequency scenarios, not PromQL execution: ```sh -cargo run -p asap-devtools --bin offline_recommend -- \ - tools/empirical-bench/results-sweep/comparison-evidence.json \ - tools/empirical-bench/results-sweep/request-uniform-are001.json +cargo run -p control_plane --example offline_recommend -- \ + /path/to/comparison-evidence.json /path/to/request.json ``` -The companion comparison artifact includes disjoint construction, ingestion, -prepare and read CPU evidence for the exact frequency index and sketch rungs. -The request declares its offline observed-error criterion, fixed snapshot, -probe population, formal parameter minima if applicable, and resource weights. -The JSON reports accepted and rejected configurations, the exact reference, -selection, and dimensional tradeoffs. CPU-only and memory-weighted requests can -choose different plans. These point-frequency observations do not price or -bound error for the o11y gauge queries above. +All benchmark producers and execution tools live in the backend repository. +This Planner PR contains documentation only; the public evidence and resource +contracts are reviewed separately in #357. diff --git a/tools/empirical-bench/.gitignore b/tools/empirical-bench/.gitignore deleted file mode 100644 index 81c34145..00000000 --- a/tools/empirical-bench/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -/results/ -/results-sweep/ -__pycache__/ diff --git a/tools/empirical-bench/ARTIFACTS.md b/tools/empirical-bench/ARTIFACTS.md deleted file mode 100644 index 84a7495f..00000000 --- a/tools/empirical-bench/ARTIFACTS.md +++ /dev/null @@ -1,32 +0,0 @@ -# Archived experiment results - -Full raw measurements and expanded query plans are distributed as an -[experiment artifact](https://github.com/ProjectASAP/ASAPPlanner/releases/tag/offline-evidence-2026-09-08), -not checked into the tool source tree. This prerelease is an experiment archive, -not a product release. The concise conclusions remain in -[the final report](../../docs/offline-o11y-final-2026-09-08.md). - -Download and verify from the repository root: - -```sh -gh release download offline-evidence-2026-09-08 --repo ProjectASAP/ASAPPlanner \ - --pattern offline-evidence-2026-09-08.tar.gz --dir /tmp -echo '180f3768203635c31188d19b500455290b96e19a4df4fdac7245af0944bd1b34 /tmp/offline-evidence-2026-09-08.tar.gz' | sha256sum --check -tar -xzf /tmp/offline-evidence-2026-09-08.tar.gz -``` - -Extract only into a checkout without existing result files; extraction restores -`tools/empirical-bench/results/` and `results-sweep/`. These generated directories -are ignored by Git. The archive is 460,710 bytes and contains 43 text files. -It preserves the original source snapshot -`152ca6cb71fc11a8c38d93e69c9fd7dca682f03b`, per-run commands, environment metadata, -raw timing/error reports, and measurement-source checksums. - -- `results/`: historical partial frequency measurements and initial planner/control-plane replay; also the later `o11y-exact-snapshot.json` reference evaluation. -- `results-sweep/`: final frequency matrix, disjoint CPU and heap evidence, six recommendation scenarios, four frequency binding reports, and final o11y control-plane replay. -- Each directory contains `MEASUREMENTS.md` explaining its measurement scope. Do not mix costs from the two runs. - -To produce new results instead of downloading these measurements, follow -[the benchmark guide](README.md) and [the replay guide](../../docs/user-guide/o11y-replay.md). -The archive preserves offline evidence, not production speedups or real-time -accuracy guarantees. diff --git a/tools/empirical-bench/README.md b/tools/empirical-bench/README.md deleted file mode 100644 index e5d5b0b2..00000000 --- a/tools/empirical-bench/README.md +++ /dev/null @@ -1,113 +0,0 @@ -# Offline sketch-bench evidence (developer guide) - -This driver runs ProjectASAP/sketch-bench's real `approxbench` CLI and adapts its -schema-v5 reports to the planner's versioned evidence schema. It measures CMS and -CountSketch using `asap_sketchlib` RegularPath/Vector2D plus an exact Polars -frequency baseline, on deterministic uniform and Zipf streams. No collector, -query backend, runtime ground truth, or runtime error feedback is involved. - -Full historical and final outputs are stored in the [experiment archive](ARTIFACTS.md), -not in this source tree. Download and verify that archive to inspect saved results. - -```bash -git clone https://github.com/ProjectASAP/sketch-bench.git /path/to/sketch-bench -git -C /path/to/sketch-bench checkout 87f619e843fd2e4da784160d4e205a0d0d55f032 -(cd /path/to/sketch-bench && cargo build --release --locked -p aqpbm-cli) -python3 tools/empirical-bench/run.py --bench-repo /path/to/sketch-bench --output /tmp/offline-evidence --sweep -python3 -m unittest discover -s tools/empirical-bench -p 'test_*.py' -``` - -Build from the sketch-bench directory so its `.cargo/config.toml` applies -`target-cpu=native`, as in the archived run. The driver records the source -revision, dirty status, executable checksum, full invocations, compiler, CPU, -OS, implementation, and parameters. It pins Polars to one worker. Five measured -runs follow two warmups per timed operation by default. Upstream accuracy executes -one offline truth comparison for the fixed generated dataset, recorded as -`error.trials=1`; it is run separately because accuracy of insert is undefined. -These timings are repeated observations -within one process, not independent experiments: standard deviations are reported, -but no confidence interval or cross-distribution guarantee is inferred. -`manifest.json.runtime_provenance` distinguishes observed host metadata and the -driver's enforced thread setting from declared build preconditions. The runtime -string's release/native/jemalloc settings describe the documented required build; -the driver does not independently recover compiler or allocator flags from the -executable. Its SHA-256 identifies that binary without proving those build flags. -The archived run used a shared development host without CPU affinity or an -exclusive core reservation; timing estimates should be remeasured on deployment -hardware before using them for capacity decisions. - -Outputs: - -- `raw.json`: unmodified per-invocation sketch-bench flat reports, including wall - times, CPU user/system samples, process RSS/allocator readings, and all errors. -- `manifest.json`: execution and environment provenance. -- `operation-reports.json`: original reports before upstream flattening; preserve - per-operation memory, including exact prepared-index state. -- `memory-probe.json`: requested-allocation samples with each live sketch, using - the same library and upstream data generator on identical inputs. -- `planner-evidence.json`: normalized sketch measurements, with unknowns null. -- `exact-baselines.json`: exact frequency index costs on the same input. -- `resource-probe.json`: disjoint live-state construction/update/read/merge CPU, - actual serialized snapshot size and allocated filesystem blocks. -- `comparison-evidence.json`: query-bound sketch records and exact baselines; - its required `disjoint_live_state_v1` timing contract excludes duplicate setup - and destruction charges. -- `request-*.json`: fixed-snapshot CPU-only comparisons at 1%/5% observed mean - relative error, plus an explicitly illustrative memory-weighted comparison. - -The default sketch parameters match formal planner sizing for epsilon=0.01, delta=0.01: -CMS width 272, depth 5; CountSketch width 30,000, depth 83. Input defaults to -20,000 i64 keys, key-space size 1,000, seed 42; Zipf exponent is 1.1. Actual -distinct-key counts come from the ground-truth probe population. Errors are -average absolute relative point-frequency errors over **all distinct keys**. -They are not error of a total COUNT, or a formal epsilon bound. Aggregate error -over repeated deterministic inputs does not establish a confidence interval. -`--sweep` additionally measures CMS widths 512/2720/4096/27200/32768 (depth 5) -and CountSketch widths 300/3000 (depth 83). The power-of-two CMS widths support -the backend frequency extension's actual parameter constraints. An offline -accuracy threshold does not authorize deployment below formal minimum sizes. - -Original upstream CPU reports use paired user+system samples and preserve wrapper -allocation/destruction overhead. The current comparison exporter instead uses -`resource_probe.rs`, linked to the exact release libraries and jemalloc. It times -live-state phases separately with `CLOCK_PROCESS_CPUTIME_ID`: empty constructor, -updates, exact prepare, reads, and sketch merge. Holder allocation and destruction -are outside the constructor timer; constructors/destructors are outside the other -phase timers. Queries cover every distinct key ten times against an unchanged -snapshot, in a reproducible rotated sorted order. The exact implementation is -the upstream public `PolarsFrequencyCore`, not a replacement algorithm. -The model's scope ends with state retained after reading; retirement is excluded -on both sides. Wall time remains in upstream raw reports and is never called CPU. - -`retained_bytes` and `peak_bytes` use the companion `memory_probe.rs`, linked to -the exact release libraries built by sketch-bench. Its counting System allocator -measures requested heap allocation bytes across construction and insertion, -keeping the sketch alive at the final snapshot; input generation is outside -the measurement. This is independent of allocator-resident pages and does not -measure jemalloc overhead. Five runs must release every sketch allocation after -destruction. CPU results come from a separate uninstrumented jemalloc probe. -The exact heap probe warms Polars twice, then measures construction/ingestion/ -prepare with the exact state alive. Its readings are accepted only when every -post-destruction allocation balance returns to zero; otherwise the exact footprint -stays explicitly formula based. `--export-only --refresh-memory` refreshes this -heap evidence without rerunning or replacing CPU timings. -Upstream counter-storage formula, process RSS and allocator readings remain in -raw reports only. The resource probe calls the actual `serialize_to_bytes` API -and verifies estimate-equivalent deserialization for every input key. It writes -one snapshot to a fresh local temporary file, flushes it, and records allocated -blocks times 512 separately from logical byte length; the file is then removed. -Directory/inode overhead and a runtime write schedule are outside this disk -measurement. State bytes are never substituted for disk usage. Validity is a -30-day reproducibility policy, not a statistical claim of future applicability. - -The exact baseline inserts the input into a buffer, then `prepare` executes -Polars group-by/count and creates a HashMap. Reads query that prepared exact -index. Compare `empty_build + (insert_per_item * N) + prepare + Q * read` with -sketch `empty_build + (update_per_item * N) + Q * read`. All components use -disjoint timed regions. A sketch can save state while losing CPU to this exact index. -These comparisons are offline point-frequency microbenchmarks; they do not -measure an o11y PromQL query or end-to-end deployed speedup. - -`results/` preserves the earlier partial run whose constructor/disk/serialization -were unknown. `results-sweep/` contains the complete frequency component run; -its additional probe must not retroactively change the earlier measurements. diff --git a/tools/empirical-bench/memory_probe.rs b/tools/empirical-bench/memory_probe.rs deleted file mode 100644 index 7564fbe4..00000000 --- a/tools/empirical-bench/memory_probe.rs +++ /dev/null @@ -1,101 +0,0 @@ -//! Counts requested allocation bytes with each sketch alive; input generation is excluded. -use std::alloc::{GlobalAlloc, Layout, System}; -use std::sync::atomic::{AtomicUsize, Ordering::Relaxed}; -use asap_sketchlib::{Count, CountMin, DataInput, RegularPath, Vector2D}; - -struct Tracking; -static LIVE: AtomicUsize = AtomicUsize::new(0); -static PEAK: AtomicUsize = AtomicUsize::new(0); -fn account(bytes: usize) { - let live = LIVE.fetch_add(bytes, Relaxed) + bytes; - PEAK.fetch_max(live, Relaxed); -} -unsafe impl GlobalAlloc for Tracking { - unsafe fn alloc(&self, layout: Layout) -> *mut u8 { - let ptr = unsafe { System.alloc(layout) }; - if !ptr.is_null() { account(layout.size()); } - ptr - } - unsafe fn alloc_zeroed(&self, layout: Layout) -> *mut u8 { - let ptr = unsafe { System.alloc_zeroed(layout) }; - if !ptr.is_null() { account(layout.size()); } - ptr - } - unsafe fn dealloc(&self, ptr: *mut u8, layout: Layout) { - unsafe { System.dealloc(ptr, layout) }; - LIVE.fetch_sub(layout.size(), Relaxed); - } - unsafe fn realloc(&self, ptr: *mut u8, layout: Layout, size: usize) -> *mut u8 { - let next = unsafe { System.realloc(ptr, layout, size) }; - if !next.is_null() { - if size >= layout.size() { account(size - layout.size()); } - else { LIVE.fetch_sub(layout.size() - size, Relaxed); } - } - next - } -} -#[global_allocator] -static ALLOCATOR: Tracking = Tracking; - -fn main() { - let path = std::env::args().nth(1).expect("raw.json path"); - let rows: Vec = serde_json::from_slice(&std::fs::read(path).unwrap()).unwrap(); - let mut results = Vec::new(); - for row in rows { - let description: aqpbm_datagen::TableDescription = serde_json::from_value(row["workload"]["synthetic"]["description"].clone()).unwrap(); - let items = description.generate().unwrap().into_column(0).unwrap().into_i64().unwrap(); - if row["impl"] == "polars" { - use sketch_bench::wrappers::polars_shared::PolarsFrequencyCore; - // Initialize Polars' worker/cache state before attributing bytes to one live index. - for _ in 0..2 { - let mut warm = PolarsFrequencyCore::::default(); - for item in &items { warm.update(item); } - warm.finalize(); - } - let mut samples = Vec::with_capacity(5); - let mut cleanup_deltas = Vec::with_capacity(5); - for _ in 0..5 { - let baseline = LIVE.load(Relaxed); - PEAK.store(baseline, Relaxed); - let mut state = PolarsFrequencyCore::::default(); - for item in &items { state.update(item); } - state.finalize(); - std::hint::black_box(&state); - let sample = (LIVE.load(Relaxed).saturating_sub(baseline), PEAK.load(Relaxed).saturating_sub(baseline)); - drop(state); - cleanup_deltas.push(LIVE.load(Relaxed) as i64 - baseline as i64); - samples.push(sample); - } - results.push(serde_json::json!({"sketch": row["sketch"], "impl": "polars", - "workload": row["workload"], "sketch_config": row["sketch_config"], "samples": samples, - "cleanup_deltas": cleanup_deltas, - "method": "requested allocation bytes using counting System allocator; exact upstream PolarsFrequencyCore construction + insertion + prepare; state alive; input and two startup warmups excluded; valid only when cleanup deltas are zero; not allocator-resident pages"})); - continue; - } - let depth = row["sketch_config"]["params"]["rows"].as_u64().unwrap() as usize; - let width = row["sketch_config"]["params"]["cols"].as_u64().unwrap() as usize; - let cms = row["sketch"].as_str().unwrap().starts_with("cms-"); - let mut samples = Vec::with_capacity(5); - for _ in 0..5 { - let baseline = LIVE.load(Relaxed); - PEAK.store(baseline, Relaxed); - // The sketch remains alive at both snapshots, unlike a consuming benchmark closure. - let (retained, peak) = if cms { - let mut sketch = CountMin::, RegularPath>::with_dimensions(depth, width); - for item in &items { sketch.insert(&DataInput::I64(*item)); } - std::hint::black_box(&sketch); - (LIVE.load(Relaxed) - baseline, PEAK.load(Relaxed) - baseline) - } else { - let mut sketch = Count::, RegularPath>::with_dimensions(depth, width); - for item in &items { sketch.insert(&DataInput::I64(*item)); } - std::hint::black_box(&sketch); - (LIVE.load(Relaxed) - baseline, PEAK.load(Relaxed) - baseline) - }; - assert_eq!(LIVE.load(Relaxed), baseline, "sketch destruction must release its allocations"); - samples.push((retained, peak)); - } - results.push(serde_json::json!({"sketch": row["sketch"], "impl": "lib", "workload": row["workload"], "sketch_config": row["sketch_config"], - "samples": samples, "method": "requested allocation bytes using counting System allocator; construction + insertion; sketch alive; dataset excluded; not allocator-resident pages"})); - } - println!("{}", serde_json::to_string_pretty(&results).unwrap()); -} diff --git a/tools/empirical-bench/resource_probe.rs b/tools/empirical-bench/resource_probe.rs deleted file mode 100644 index e1a17629..00000000 --- a/tools/empirical-bench/resource_probe.rs +++ /dev/null @@ -1,138 +0,0 @@ -//! Complements sketch-bench with empty construction CPU and actual persisted snapshot size. -use std::hint::black_box; -use std::io::Write; -use std::os::unix::fs::MetadataExt; -use asap_sketchlib::{Count, CountMin, DataInput, RegularPath, Vector2D}; - -#[global_allocator] -static ALLOCATOR: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc; - -fn cpu_ns() -> u64 { - #[repr(C)] - struct Timespec { tv_sec: std::os::raw::c_long, tv_nsec: std::os::raw::c_long } - extern "C" { fn clock_gettime(clock_id: std::os::raw::c_int, time: *mut Timespec) -> std::os::raw::c_int; } - // Linux CLOCK_PROCESS_CPUTIME_ID; this companion also uses Linux allocated blocks. - assert!(cfg!(target_os = "linux")); - let mut time = Timespec { tv_sec: 0, tv_nsec: 0 }; - assert_eq!(unsafe { clock_gettime(2, &mut time) }, 0); - time.tv_sec as u64 * 1_000_000_000 + time.tv_nsec as u64 -} - -fn construction(mut build: impl FnMut() -> T, batch: usize) -> Vec { - let mut samples = Vec::with_capacity(5); - for run in 0..7 { - let mut alive = Vec::with_capacity(batch); - let start = cpu_ns(); - for _ in 0..batch { alive.push(black_box(build())); } - let elapsed = cpu_ns() - start; - if run >= 2 { samples.push(elapsed as f64 / batch as f64); } - // Empty objects remain alive until after the timed construction batch. - black_box(&alive); - drop(alive); - } - samples -} - -fn live_phases(items: &[i64], build: impl Fn() -> T, insert: impl Fn(&mut T, i64), - read: impl Fn(&T, i64) -> f64, prepare: impl Fn(&mut T), - merge: Option) -> serde_json::Value { - let mut keys: Vec<_> = items.iter().copied().collect::>().into_iter().collect(); - // Fixed rotation avoids sorted probe order without introducing another RNG definition. - if !keys.is_empty() { let mid = keys.len() / 2; keys.rotate_left(mid); } - let mut updates = Vec::with_capacity(5); - let mut prepares = Vec::with_capacity(5); - let mut reads = Vec::with_capacity(5); - let mut merges = Vec::with_capacity(5); - for run in 0..7 { - let mut state = build(); - let start = cpu_ns(); - for item in items { insert(&mut state, black_box(*item)); } - let update = (cpu_ns() - start) as f64 / items.len() as f64; - let start = cpu_ns(); - prepare(&mut state); - let prepare = (cpu_ns() - start) as f64; - let start = cpu_ns(); - for _ in 0..10 { for key in &keys { black_box(read(&state, black_box(*key))); } } - let query = (cpu_ns() - start) as f64 / (keys.len() * 10) as f64; - let merge_time = merge.map(|merge| { - let mut other = build(); - for item in items { insert(&mut other, *item); } - let start = cpu_ns(); - merge(&mut state, &other); - (cpu_ns() - start) as f64 - }); - if run >= 2 { - updates.push(update); prepares.push(prepare); reads.push(query); - if let Some(time) = merge_time { merges.push(time); } - } - black_box(&state); - } - serde_json::json!({"update_cpu_ns_samples": updates, "prepare_cpu_ns_samples": prepares, - "read_cpu_ns_samples": reads, "merge_cpu_ns_samples": merges, - "query_distinct_keys": keys.len(), "query_passes": 10, - "method": "CLOCK_PROCESS_CPUTIME_ID; live state with constructor/destructor excluded; five runs after two warmups; update per input item; read per all-distinct-key lookup across ten repeated fixed-snapshot probe passes; merge per binary merge; fixed rotated sorted key order"}) -} - -fn main() { - let args: Vec = std::env::args().collect(); - let rows: Vec = serde_json::from_slice(&std::fs::read(&args[1]).unwrap()).unwrap(); - let directory = std::path::Path::new(&args[2]); - let mut results = Vec::new(); - for (index, row) in rows.into_iter().enumerate() { - let description: aqpbm_datagen::TableDescription = serde_json::from_value(row["workload"]["synthetic"]["description"].clone()).unwrap(); - let items = description.generate().unwrap().into_column(0).unwrap().into_i64().unwrap(); - if row["impl"] == "polars" { - use sketch_bench::wrappers::polars_shared::PolarsFrequencyCore; - let samples = construction(PolarsFrequencyCore::::default, 4096); - let phases = live_phases(&items, PolarsFrequencyCore::::default, - |s, x| s.update(&x), |s, x| s.query(&x) as f64, |s| s.finalize(), None); - results.push(serde_json::json!({"sketch": row["sketch"], "impl": row["impl"], - "workload": row["workload"], "sketch_config": row["sketch_config"], - "build_cpu_ns_samples": samples, "build_batch": 4096, - "phases": phases, - "build_method": "process CPU around upstream PolarsFrequencyCore::::default(); output holder allocation and destruction excluded"})); - continue; - } - let depth = row["sketch_config"]["params"]["rows"].as_u64().unwrap() as usize; - let width = row["sketch_config"]["params"]["cols"].as_u64().unwrap() as usize; - let batch = (16_000_000 / (depth * width * 4)).clamp(2, 256); - let (samples, serialized, phases) = if row["sketch"].as_str().unwrap().starts_with("cms-") { - let samples = construction(|| CountMin::, RegularPath>::with_dimensions(depth, width), batch); - let phases = live_phases(&items, || CountMin::, RegularPath>::with_dimensions(depth, width), - |s,x| s.insert(&DataInput::I64(x)), |s,x| s.estimate(&DataInput::I64(x)) as f64, |_| {}, Some(CountMin::, RegularPath>::merge)); - let mut sketch = CountMin::, RegularPath>::with_dimensions(depth, width); - for item in &items { sketch.insert(&DataInput::I64(*item)); } - let bytes = sketch.serialize_to_bytes().unwrap(); - let restored = CountMin::, RegularPath>::deserialize_from_bytes(&bytes).unwrap(); - for item in &items { assert_eq!(sketch.estimate(&DataInput::I64(*item)), restored.estimate(&DataInput::I64(*item))); } - (samples, bytes, phases) - } else { - let samples = construction(|| Count::, RegularPath>::with_dimensions(depth, width), batch); - let phases = live_phases(&items, || Count::, RegularPath>::with_dimensions(depth, width), - |s,x| s.insert(&DataInput::I64(x)), |s,x| s.estimate(&DataInput::I64(x)), |_| {}, Some(Count::, RegularPath>::merge)); - let mut sketch = Count::, RegularPath>::with_dimensions(depth, width); - for item in &items { sketch.insert(&DataInput::I64(*item)); } - let bytes = sketch.serialize_to_bytes().unwrap(); - let restored = Count::, RegularPath>::deserialize_from_bytes(&bytes).unwrap(); - for item in &items { assert_eq!(sketch.estimate(&DataInput::I64(*item)), restored.estimate(&DataInput::I64(*item))); } - (samples, bytes, phases) - }; - let path = directory.join(format!("snapshot-{index}.msgpack")); - let mut file = std::fs::OpenOptions::new().write(true).create_new(true).open(&path).unwrap(); - file.write_all(&serialized).unwrap(); - file.sync_all().unwrap(); - let metadata = file.metadata().unwrap(); - assert_eq!(metadata.len(), serialized.len() as u64); - let disk_bytes = metadata.blocks() * 512; - drop(file); - std::fs::remove_file(path).unwrap(); - results.push(serde_json::json!({"sketch": row["sketch"], "impl": row["impl"], - "workload": row["workload"], "sketch_config": row["sketch_config"], - "build_cpu_ns_samples": samples, "build_batch": batch, - "phases": phases, - "build_method": "CLOCK_PROCESS_CPUTIME_ID around empty sketch constructors in batched live objects; holder allocation and all destruction excluded; jemalloc; five runs after two warmups", - "serialized_bytes": serialized.len(), "disk_bytes": disk_bytes, - "disk_method": "actual flushed single MsgPack snapshot file allocated blocks*512 on local temporary filesystem; directory/inode overhead excluded; file removed after measurement; not runtime persistence demand"})); - } - println!("{}", serde_json::to_string_pretty(&results).unwrap()); -} diff --git a/tools/empirical-bench/run.py b/tools/empirical-bench/run.py deleted file mode 100644 index cd56d756..00000000 --- a/tools/empirical-bench/run.py +++ /dev/null @@ -1,367 +0,0 @@ -#!/usr/bin/env python3 -"""Run the real sketch-bench CLI and export offline planner evidence (stdlib only).""" -import argparse -import datetime -import hashlib -import json -import math -import os -from pathlib import Path -import platform -import shlex -import statistics -import subprocess -import tempfile - - -BUILD_PROVENANCE = { - "runtime_field_status": "declared build preconditions, not introspected from the executable", - "compiler_status": "rustc --version observed on driver host; executable compiler not independently verified", - "required_build": "run cargo build --release --locked -p aqpbm-cli from the pinned sketch-bench directory with default features", - "declared_settings": "release; repository target-cpu=native; default jemalloc", - "verified_execution_setting": "driver sets POLARS_MAX_THREADS=1", - "binary_identity": "SHA-256 recorded; digest identifies executable but does not verify build settings", -} - - -def command(argv, **kwargs): - return subprocess.check_output(argv, text=True, **kwargs).strip() - - -def measurement(values): - if not values or any(not math.isfinite(x) or x < 0 for x in values): - raise ValueError("invalid or missing measurement") - return {"value": statistics.mean(values), - "stddev": statistics.stdev(values) if len(values) > 1 else None, - "samples": len(values)} - - -def cpu_per_op(row, op): - """Recover each run's work from paired rate and elapsed samples, as upstream does.""" - rate = row.get(op + ("_folds_per_sec" if op == "merge" else "_throughput_items_per_sec")) - cpu = row.get(op + "_cpu_time_ms") - wall = row.get(op + "_wall_time_ms") - if not rate or not cpu or not wall: - return None - samples = [rate["samples"], wall["samples"], cpu["user_ms"]["samples"], cpu["sys_ms"]["samples"]] - if len({len(x) for x in samples}) != 1: - raise ValueError("unaligned CPU/rate/time samples") - values = [] - for r, elapsed, user, system in zip(*samples): - work = r * elapsed / 1000 - if work <= 0: - raise ValueError("zero benchmark work") - values.append((user + system) * 1_000_000 / work) - result = measurement(values) - result["method"] = "mean of paired run (process user+system CPU ns)/(throughput*wall seconds); per " + ("binary merge" if op == "merge" else "input item" if op == "insert" else "point-frequency key lookup") - return result - - -def cpu_batch(row, op): - cpu = row.get(op + "_cpu_time_ms") - if not cpu: - return None - return measurement([(u + s) * 1_000_000 for u, s in - zip(cpu["user_ms"]["samples"], cpu["sys_ms"]["samples"])]) - - -def export(raw, manifest, operation_reports, memory_rows, resource_rows=None): - records, baselines = [], [] - if not (len(raw) == len(manifest["invocations"]) == len(operation_reports)): - raise ValueError("misaligned invocation reports") - for row, invocation, operations in zip(raw, manifest["invocations"], operation_reports): - if row["schema_version"] != 5: - raise ValueError("adapter requires sketch-bench schema 5") - accuracy = row["query_accuracy"] - resource = next((r for r in (resource_rows or []) if r["sketch"] == row["sketch"] - and r["impl"] == row["impl"] and r["sketch_config"] == row["sketch_config"] - and r["workload"] == row["workload"]), None) - if resource_rows is not None and (resource is None or "phases" not in resource): - raise ValueError("complete comparison requires matched disjoint live-state phase measurements for every record") - def phase_metric(phase): - return (dict(measurement(resource["phases"][phase + "_cpu_ns_samples"]), method=resource["phases"]["method"]) - if resource and resource.get("phases") else None) - shape = row["workload"]["synthetic"]["description"] - distribution = {"id": invocation["distribution_id"], - "family": invocation["distribution"], - "sample_count": shape["row_num"], - "distinct_count": int(accuracy["probes_all"]), - "parameters": shape["column_spec"][0]["distribution"]} - if invocation["algorithm"] == "Exact": - exact_memory = next((m for m in memory_rows if m.get("impl") == "polars" and m["workload"] == row["workload"]), None) - baselines.append({"distribution": distribution, "implementation": "polars exact group_by + HashMap", - "insert_cpu_ns_per_item": phase_metric("update") or cpu_per_op(row, "insert"), - "prepare_cpu_ns_per_dataset": phase_metric("prepare") or cpu_batch(row, "prepare"), - "read_cpu_ns_per_key": phase_metric("read") or cpu_per_op(row, "query"), - "retained_bytes": max(r["bench"]["memory_bytes"] for r in operations - if r["bench"].get("operation") == "query"), - "retained_bytes_method": "upstream prepared-query memory_bytes formula: buffer capacity plus HashMap capacity footprint; not measured allocated heap or peak", - "accuracy": accuracy}) - baselines[-1]["empty_build_cpu_ns"] = (dict(measurement(resource["build_cpu_ns_samples"]), method=resource["build_method"]) - if resource else None) - if exact_memory and all(delta == 0 for delta in exact_memory["cleanup_deltas"]): - baselines[-1]["retained_heap_bytes"] = dict(measurement([s[0] for s in exact_memory["samples"]]), method=exact_memory["method"]) - baselines[-1]["peak_heap_bytes"] = dict(measurement([s[1] for s in exact_memory["samples"]]), method=exact_memory["method"]) - continue - algorithm = invocation["algorithm"] - memory = next(m for m in memory_rows if m["sketch"] == row["sketch"] and m["workload"] == row["workload"] - and (m.get("sketch_config", row["sketch_config"]) == row["sketch_config"])) - params = row["sketch_config"]["params"] - timestamp = row["insert_timestamp"] - if not timestamp.endswith("Z"): - raise ValueError("expected upstream UTC timestamp") - measured = int(datetime.datetime.fromisoformat(timestamp[:19] + "+00:00").timestamp()) - environment = dict(manifest["environment"]) - environment["implementation"] = "asap_sketchlib RegularPath Vector2D" - # The algorithm distinguishes families; storage/hash path must also match. - environment["id"] = hashlib.sha256(json.dumps(environment, sort_keys=True).encode()).hexdigest()[:16] - records.append({"id": invocation["id"], "algorithm": algorithm, - "params": {algorithm: {"width": params["cols"], "depth": params["rows"]}}, - "distribution": distribution, "environment": environment, - "measured_at_unix_seconds": measured, - "valid_until_unix_seconds": measured + 30 * 86400, - "provenance": {"command": shlex.join(invocation["argv"]), - "dataset": "deterministic synthetic i64 frequency stream, seed 42", - "source_revision": manifest["source_revision"], - "repetitions": row["runs"]}, - "metrics": {"build_cpu_ns": (dict(measurement(resource["build_cpu_ns_samples"]), method=resource["build_method"]) - if resource else None), - "update_cpu_ns": phase_metric("update") or cpu_per_op(row, "insert"), - "merge_cpu_ns": phase_metric("merge") or cpu_per_op(row, "merge"), - "read_cpu_ns": phase_metric("read") or cpu_per_op(row, "query"), - "retained_bytes": dict(measurement([float(s[0]) for s in memory["samples"]]), method=memory["method"]), - "peak_bytes": dict(measurement([float(s[1]) for s in memory["samples"]]), method=memory["method"]), - "serialized_bytes": (dict(measurement([resource["serialized_bytes"]]), method="actual serialize_to_bytes MsgPack length; estimate-equivalent roundtrip verified for every input key") if resource else None), - "disk_bytes": (dict(measurement([resource["disk_bytes"]]), method=resource["disk_method"]) if resource else None)}, - "error": {"metric": "mean_absolute_relative_frequency_error_all_distinct_keys", - "mean": accuracy["are_all"], "max": None, - "trials": int(accuracy.get("accuracy_runs", 1)), - "ground_truth_method": "exact offline HashMap counts; every distinct key probed", - "query": {"kind": "point_frequency", "value_type": "i64", - "accuracy_stddev": accuracy.get("are_all_stddev"), - "full_accuracy": accuracy}}}) - return {"schema_version": 1, - "benchmark_version": "sketch-bench@" + manifest["source_revision"] + ("; adapter-v2-disjoint" if resource_rows else "; adapter-v1"), - "model_version": "empirical-update-cpu-v1", "records": records}, baselines - - -def comparison_artifact(artifact, baselines, manifest): - query = {"kind": "point_frequency", "value_type": "i64", "probe_set": "all_distinct_keys"} - exact_records = [] - for baseline in baselines: - if baseline["empty_build_cpu_ns"] is None: - continue - reference = next(r for r in artifact["records"] if r["distribution"] == baseline["distribution"]) - environment = dict(manifest["environment"]) - environment["implementation"] = "polars group_by + std HashMap" - environment["implementation_version"] = "polars 0.46.0" - environment["id"] = hashlib.sha256(json.dumps(environment, sort_keys=True).encode()).hexdigest()[:16] - invocation = next(i for i in manifest["invocations"] if i["algorithm"] == "Exact" - and i["distribution_id"] == baseline["distribution"]["id"]) - exact_records.append({"id": invocation["id"], "distribution": baseline["distribution"], - "environment": environment, "query": query, - "measured_at_unix_seconds": reference["measured_at_unix_seconds"], - "valid_until_unix_seconds": reference["valid_until_unix_seconds"], - "provenance": {"command": shlex.join(invocation["argv"]), - "dataset": "deterministic synthetic i64 frequency stream, seed 42; exact zero-error comparison verified offline", - "source_revision": manifest["source_revision"], "repetitions": 5}, - "metrics": {"empty_build_cpu_ns": baseline["empty_build_cpu_ns"], - "update_cpu_ns": baseline["insert_cpu_ns_per_item"], - "prepare_cpu_ns": baseline["prepare_cpu_ns_per_dataset"], - "read_cpu_ns": baseline["read_cpu_ns_per_key"], - "retained_bytes": baseline.get("retained_heap_bytes") or dict(measurement([baseline["retained_bytes"]]), method=baseline["retained_bytes_method"]), - "peak_bytes": baseline.get("peak_heap_bytes")}}) - return {"schema_version": 1, "timing_contract": "disjoint_live_state_v1", "sketch_evidence": artifact, - "query_bindings": [{"record_id": r["id"], "query": query} for r in artifact["records"]], - "exact_records": exact_records} - - -def refresh_memory(bench_repo, raw_path, memory_path): - deps = bench_repo.resolve() / "target/release/deps" - with tempfile.TemporaryDirectory(prefix="asap-memory-probe-") as temporary: - executable = str(Path(temporary) / "memory-probe") - argv = ["rustc", "--edition=2021", "-C", "opt-level=3", "-C", "panic=abort", "-C", "target-cpu=native", - "-L", f"dependency={deps}", str(Path(__file__).with_name("memory_probe.rs")), "-o", executable] - for name in ["asap_sketchlib", "aqpbm_datagen", "serde_json", "sketch_bench"]: - candidates = list(deps.glob(f"lib{name}-*.rlib")) - if len(candidates) != 1: - raise ValueError(f"expected one pinned release library for {name}, got {len(candidates)}") - argv += ["--extern", f"{name}={candidates[0]}"] - for pattern in ["*/out", "*/out/lib"]: - for native in (bench_repo.resolve() / "target/release/build").glob(pattern): - argv += ["-L", f"native={native}"] - subprocess.check_call(argv) - memory_path.write_text(command([executable, str(raw_path)], env=dict(os.environ, POLARS_MAX_THREADS="1")) + "\n") - return {"source_sha256": hashlib.sha256(Path(__file__).with_name("memory_probe.rs").read_bytes()).hexdigest(), - "compile_argv": argv, - "allocator": "System + requested-byte tracking; separate from uninstrumented jemalloc CPU measurements"} - - -def main(): - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("--bench-repo", required=True, type=Path) - parser.add_argument("--output", required=True, type=Path) - parser.add_argument("--size", type=int, default=20000) - parser.add_argument("--cardinality", type=int, default=1000) - parser.add_argument("--runs", type=int, default=5) - parser.add_argument("--sweep", action="store_true", help="measure three widths per frequency family") - parser.add_argument("--export-only", action="store_true") - parser.add_argument("--refresh-memory", action="store_true", help="with --export-only, remeasure heap without rerunning CPU timings") - parser.add_argument("--resume", action="store_true", help="continue a partially completed run with identical parameters") - args = parser.parse_args() - if args.refresh_memory and not args.export_only: - parser.error("--refresh-memory requires --export-only") - args.output.mkdir(parents=True, exist_ok=True) - manifest_path = args.output / "manifest.json" - raw_path = args.output / "raw.json" - operations_path = args.output / "operation-reports.json" - if args.export_only: - manifest, raw = json.loads(manifest_path.read_text()), json.loads(raw_path.read_text()) - operation_reports = json.loads(operations_path.read_text()) - else: - if (raw_path.exists() or manifest_path.exists()) and not args.resume: - parser.error("output already contains a run; use a new directory or --export-only") - repo = args.bench_repo.resolve() - if 'name = "asap_sketchlib"\nversion = "0.2.2"' not in (repo / "Cargo.lock").read_text(): - parser.error("adapter is validated with asap_sketchlib 0.2.2; do not relabel another version") - binary = repo / "target/release/approxbench" - cpu = next((line.split(":", 1)[1].strip() for line in Path("/proc/cpuinfo").read_text().splitlines() - if line.startswith("model name")), platform.processor()) - manifest = {"source_revision": command(["git", "rev-parse", "HEAD"], cwd=repo), - "source_dirty": bool(command(["git", "status", "--porcelain"], cwd=repo)), - "environment": {"cpu": cpu, "os": platform.platform(), - "runtime": command(["rustc", "--version"]) + "; release; target-cpu=native; jemalloc; POLARS_MAX_THREADS=1", - "implementation_version": "asap_sketchlib 0.2.2"}, - "binary_sha256": hashlib.sha256(binary.read_bytes()).hexdigest(), - "runtime_provenance": BUILD_PROVENANCE, - "invocations": []} - raw, operation_reports = [], [] - if args.resume: - previous = json.loads(manifest_path.read_text()) - if previous["binary_sha256"] != manifest["binary_sha256"] or previous["environment"] != manifest["environment"]: - parser.error("resume requires the same binary and environment") - manifest, raw = previous, json.loads(raw_path.read_text()) - operation_reports = json.loads(operations_path.read_text()) - for invocation in manifest["invocations"]: - argv = invocation["argv"] - if any(argv[argv.index(flag) + 1] != str(value) for flag, value in - [("--size", args.size), ("--cardinality", args.cardinality), ("--runs", args.runs)]): - parser.error("resume requires the same size, cardinality, and runs") - env = dict(os.environ, POLARS_MAX_THREADS="1") - for distribution in ["uniform", "zipf"]: - targets = [ - ("Cms", "cms-regularpath-vector2d", "lib", 5, 272), - ("CountSketch", "countsketch-regularpath-vector2d", "lib", 83, 30000), - ("Exact", "cms", "polars", 5, 272)] - if args.sweep: - targets += [("Cms", "cms-regularpath-vector2d", "lib", 5, width) for width in [2720, 27200, 512, 4096, 32768]] - targets += [("CountSketch", "countsketch-regularpath-vector2d", "lib", 83, width) for width in [300, 3000]] - for algorithm, variant, library, rows, cols in targets: - record_id = f"{algorithm.lower()}-{distribution}-seed42" + (f"-w{cols}-d{rows}" if args.sweep else "") - if any(i["id"] == record_id for i in manifest["invocations"]): - continue - argv = [str(binary), "sketchbench", "--variant", variant, "--library", library, - "--config", f"rows={rows} cols={cols}", "--dataset", distribution, - "--size", str(args.size), "--cardinality", str(args.cardinality), - "--dtype", "i64", "--seed", "42", "--runs", str(args.runs), - "--warmup-runs", "2", "--operations", - "insert,query" if algorithm == "Exact" else "insert,query,merge", - "--metrics", "throughput,cpu,memory"] - if distribution == "zipf": - argv += ["--zipf-s", "1.1"] - print(shlex.join(argv), flush=True) - output = command(argv, env=env) - accuracy_argv = list(argv) - accuracy_argv[accuracy_argv.index("--operations") + 1] = "query" - accuracy_argv[accuracy_argv.index("--metrics") + 1] = "accuracy" - output += "\n" + command(accuracy_argv, env=env) - prepare_argv = None - if algorithm == "Exact": - prepare_argv = list(argv) - prepare_argv[prepare_argv.index("--operations") + 1] = "prepare" - prepare_argv[prepare_argv.index("--metrics") + 1] = "latency,cpu,memory" - output += "\n" + command(prepare_argv, env=env) - operations = [json.loads(line) for line in output.splitlines() if line.strip()] - row = json.loads(command([str(binary), "flatten"], input=output, env=env)) - raw.append(row) - operation_reports.append(operations) - manifest["invocations"].append({"id": record_id, - "distribution_id": f"{distribution}-n{args.size}-c{args.cardinality}-seed42", - "distribution": distribution, "algorithm": algorithm, "argv": argv, - "accuracy_argv": accuracy_argv, "prepare_argv": prepare_argv}) - raw_path.write_text(json.dumps(raw, indent=2) + "\n") - manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") - operations_path.write_text(json.dumps(operation_reports, indent=2) + "\n") - memory_path = args.output / "memory-probe.json" - resources_path = args.output / "resource-probe.json" - if not args.export_only: - # Older partial runs may lack prepare: CPU/memory alone select no upstream timing pass. - for index, (row, invocation) in enumerate(zip(raw, manifest["invocations"])): - if invocation["algorithm"] == "Exact" and not row.get("prepare_cpu_time_ms"): - prepare_argv = list(invocation["argv"]) - prepare_argv[prepare_argv.index("--operations") + 1] = "prepare" - prepare_argv[prepare_argv.index("--metrics") + 1] = "latency,cpu,memory" - output = command(prepare_argv, env=env) - operation_reports[index].extend(json.loads(line) for line in output.splitlines() if line.strip()) - raw[index] = json.loads(command([str(binary), "flatten"], - input="\n".join(json.dumps(r) for r in operation_reports[index]), env=env)) - invocation["prepare_argv"] = prepare_argv - raw_path.write_text(json.dumps(raw, indent=2) + "\n") - operations_path.write_text(json.dumps(operation_reports, indent=2) + "\n") - deps = args.bench_repo.resolve() / "target/release/deps" - manifest["memory_probe"] = refresh_memory(args.bench_repo, raw_path, memory_path) - with tempfile.TemporaryDirectory(prefix="asap-resource-probe-") as temporary: - executable = str(Path(temporary) / "resource-probe") - argv = ["rustc", "--edition=2021", "-C", "opt-level=3", "-C", "panic=abort", "-C", "target-cpu=native", - "-L", f"dependency={deps}", str(Path(__file__).with_name("resource_probe.rs")), "-o", executable] - for name in ["asap_sketchlib", "aqpbm_datagen", "serde_json", "tikv_jemallocator", "sketch_bench"]: - candidates = list(deps.glob(f"lib{name}-*.rlib")) - if len(candidates) != 1: - raise ValueError(f"expected one pinned release library for {name}, got {len(candidates)}") - argv += ["--extern", f"{name}={candidates[0]}"] - for native in (args.bench_repo.resolve() / "target/release/build").glob("tikv-jemalloc-sys-*/out/lib"): - argv += ["-L", f"native={native}"] - for native in (args.bench_repo.resolve() / "target/release/build").glob("*/out"): - argv += ["-L", f"native={native}"] - subprocess.check_call(argv) - resources_path.write_text(command([executable, str(raw_path), temporary], env=env) + "\n") - manifest["resource_probe"] = {"source_sha256": hashlib.sha256(Path(__file__).with_name("resource_probe.rs").read_bytes()).hexdigest(), - "compile_argv": argv, "allocator": "jemalloc", "disk_filesystem_path": tempfile.gettempdir()} - manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") - if args.refresh_memory: - if hashlib.sha256((args.bench_repo.resolve() / "target/release/approxbench").read_bytes()).hexdigest() != manifest["binary_sha256"]: - parser.error("memory refresh requires the original benchmark binary") - manifest["memory_probe"] = refresh_memory(args.bench_repo, raw_path, memory_path) - manifest["runtime_provenance"] = BUILD_PROVENANCE - manifest_path.write_text(json.dumps(manifest, indent=2) + "\n") - artifact, baselines = export(raw, manifest, operation_reports, json.loads(memory_path.read_text()), - json.loads(resources_path.read_text()) if resources_path.exists() else None) - (args.output / "planner-evidence.json").write_text(json.dumps(artifact, indent=2) + "\n") - (args.output / "exact-baselines.json").write_text(json.dumps(baselines, indent=2) + "\n") - if resources_path.exists(): - comparison = comparison_artifact(artifact, baselines, manifest) - (args.output / "comparison-evidence.json").write_text(json.dumps(comparison, indent=2) + "\n") - for exact in comparison["exact_records"]: - row = next(r for r in artifact["records"] if r["distribution"] == exact["distribution"]) - for budget, suffix in [(0.01, "001"), (0.05, "005")]: - request = {"context": {"distribution": row["distribution"], "environment": row["environment"], - "now_unix_seconds": max(r["measured_at_unix_seconds"] for r in artifact["records"])}, - "exact_environment": exact["environment"], "query": exact["query"], - "accuracy": {"metric": row["error"]["metric"], "max_observed_mean": budget, "minimum_trials": 1}, - "workload": {"input_items_per_state": row["distribution"]["sample_count"], - "reads_per_state": 1000, "merges_per_state": 0, "state_instances": 1, "horizon_seconds": 300.0}, - "weights": {"cpu_ns_weight": 1.0, "retained_byte_seconds_weight": 0.0}, "formal_minimums": None} - (args.output / ("request-" + row["distribution"]["family"] + "-are" + suffix + ".json")).write_text(json.dumps(request, indent=2) + "\n") - request["accuracy"]["max_observed_mean"] = 0.01 - request["weights"]["retained_byte_seconds_weight"] = 0.01 - request["formal_minimums"] = [{"algorithm": "Cms", "params": {"Cms": {"width": 512, "depth": 5}}}] - (args.output / ("request-" + row["distribution"]["family"] + "-memory-weighted.json")).write_text(json.dumps(request, indent=2) + "\n") - for row in artifact["records"]: - context = {"distribution": row["distribution"], "environment": row["environment"], - "now_unix_seconds": max(r["measured_at_unix_seconds"] for r in artifact["records"])} - (args.output / ("context-" + row["distribution"]["family"] + ".json")).write_text(json.dumps(context, indent=2) + "\n") - print(f"Exported {len(artifact['records'])} sketch records and {len(baselines)} exact baselines.") - - -if __name__ == "__main__": - main() diff --git a/tools/empirical-bench/test_export.py b/tools/empirical-bench/test_export.py deleted file mode 100644 index ab5389b5..00000000 --- a/tools/empirical-bench/test_export.py +++ /dev/null @@ -1,51 +0,0 @@ -"""Unit checks for measurement normalization, independent of running the benchmark.""" -import unittest - -from run import cpu_per_op, export - - -class NormalizationTests(unittest.TestCase): - def test_complete_comparison_rejects_missing_live_phase_measurements(self): - """An old constructor-only probe cannot label consuming-wrapper CPU as disjoint.""" - row = {"schema_version": 5, "query_accuracy": {}, "sketch": "cms", "impl": "lib", - "sketch_config": {}, "workload": {}} - manifest = {"invocations": [{}]} - for resources in [[], [dict(row, build_cpu_ns_samples=[1.0])]]: - with self.subTest(resources=resources): - with self.assertRaisesRegex(ValueError, "disjoint live-state"): - export([row], manifest, [[]], [], resources) - - def test_partial_report_files_are_rejected_before_pairing(self): - """A truncated report file must fail rather than silently lose invocations.""" - for raw, invocations, operations in [([{}], [], []), ([], [{}], []), ([], [], [{}])]: - with self.subTest(lengths=(len(raw), len(invocations), len(operations))): - with self.assertRaisesRegex(ValueError, "misaligned invocation reports"): - export(raw, {"invocations": invocations}, operations, []) - - def test_cpu_normalizes_paired_work_not_wall_time(self): - """Different wall durations for identical work must not change the denominator.""" - row = {"query_throughput_items_per_sec": {"samples": [2000, 1000]}, - "query_wall_time_ms": {"samples": [100, 200]}, - "query_cpu_time_ms": {"user_ms": {"samples": [20, 20]}, - "sys_ms": {"samples": [10, 10]}}} - result = cpu_per_op(row, "query") - self.assertEqual(result["value"], 150000) - self.assertEqual(result["stddev"], 0) - self.assertEqual(result["samples"], 2) - - def test_unmeasured_cpu_is_not_zero(self): - """Missing CPU evidence stays absent instead of looking like free work.""" - self.assertIsNone(cpu_per_op({}, "query")) - - def test_misaligned_samples_rejected(self): - """CPU and rates from different run populations cannot be paired.""" - row = {"insert_throughput_items_per_sec": {"samples": [2000, 1000]}, - "insert_wall_time_ms": {"samples": [100]}, - "insert_cpu_time_ms": {"user_ms": {"samples": [20, 20]}, - "sys_ms": {"samples": [10, 10]}}} - with self.assertRaises(ValueError): - cpu_per_op(row, "insert") - - -if __name__ == "__main__": - unittest.main() From a6f23c214060eaca6ed7c023c6e62563383cc2d4 Mon Sep 17 00:00:00 2001 From: zz_y Date: Tue, 8 Sep 2026 12:38:11 -0600 Subject: [PATCH 3/3] docs: distinguish historical plan from backend execution prototype --- docs/design_docs/empirical-o11y-execution-plan.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/design_docs/empirical-o11y-execution-plan.md b/docs/design_docs/empirical-o11y-execution-plan.md index d7c55f03..7ffbdcac 100644 --- a/docs/design_docs/empirical-o11y-execution-plan.md +++ b/docs/design_docs/empirical-o11y-execution-plan.md @@ -1,5 +1,11 @@ # Offline evidence and o11y planning evaluation +This is the historical offline-evidence milestone plan. The current prototype +adds actual backend-only data-plane execution using the supplied OpenMetrics +dataset, as described in the [current evaluation guide](../user-guide/o11y-replay.md). +Benchmark production and execution tooling now belong to ASAPQuery-backend; +planner-only coverage and synthetic cached-result timing are not its acceptance criteria. + Audience: developers reproducing issue #322 and the planner/control-plane evaluation. ## Scope