From 14c224f44470eb298803d4c1288c036f31f9b5bd Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 19:46:40 -0600 Subject: [PATCH 1/2] fix(eval): measure CPU time in Top-K ERP resources --- Cargo.lock | 1 + data_plane/Cargo.toml | 1 + data_plane/examples/topk_dashboard/erp.rs | 81 +++++++++++++++++-- .../data/topk-dashboard-v2/report.md | 6 ++ .../reproduce_topk_erp.py | 4 +- .../topk-dashboard-results.md | 6 ++ 6 files changed, 90 insertions(+), 9 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 28d3dfb4..319c1783 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1211,6 +1211,7 @@ dependencies = [ "hex", "http-body-util", "lazy_static", + "libc", "memmap2", "moka", "prometheus", diff --git a/data_plane/Cargo.toml b/data_plane/Cargo.toml index ef0ea37a..c3872940 100644 --- a/data_plane/Cargo.toml +++ b/data_plane/Cargo.toml @@ -132,6 +132,7 @@ fs2 = "0.4" # none of them. [dev-dependencies] +libc = "0.2" asap-aware-mapping = { git = "https://github.com/ProjectASAP/ASAPPlanner", rev = "a9651cc" } tempfile = "3.20.0" criterion = { version = "0.5", features = ["html_reports"] } diff --git a/data_plane/examples/topk_dashboard/erp.rs b/data_plane/examples/topk_dashboard/erp.rs index c49437da..eb6cd800 100644 --- a/data_plane/examples/topk_dashboard/erp.rs +++ b/data_plane/examples/topk_dashboard/erp.rs @@ -6,6 +6,29 @@ use asap_aware_mapping::erp::{ }; use std::collections::BTreeMap; +// ERP CPU fields must contain scheduled process CPU time, not elapsed time +// distorted by descheduling or competing host work. +#[cfg(target_os = "linux")] +fn process_cpu_seconds() -> std::io::Result { + let mut timestamp = std::mem::MaybeUninit::::uninit(); + // SAFETY: clock_gettime initializes the writable timespec on success. + let status = + unsafe { libc::clock_gettime(libc::CLOCK_PROCESS_CPUTIME_ID, timestamp.as_mut_ptr()) }; + if status != 0 { + return Err(std::io::Error::last_os_error()); + } + let timestamp = unsafe { timestamp.assume_init() }; + Ok(timestamp.tv_sec as f64 + timestamp.tv_nsec as f64 * 1e-9) +} + +#[cfg(not(target_os = "linux"))] +fn process_cpu_seconds() -> std::io::Result { + Err(std::io::Error::new( + std::io::ErrorKind::Unsupported, + "ERP calibration requires a process CPU clock", + )) +} + pub fn bytes(c: Config) -> usize { c.rows * c.cols * 8 + c.heap * 32 } @@ -68,9 +91,9 @@ pub fn benchmark( data: &[Vec], config: Config, ) -> Result<(Vec, ErpResourceProfile), Box> { - let t = Instant::now(); + let t = process_cpu_seconds()?; let states: Vec<_> = data.iter().map(|p| build_pane(p, config)).collect(); - let update = t.elapsed().as_secs_f64(); + let update = (process_cpu_seconds()? - t).max(0.0); let mut loss = vec![0.0_f64; 4]; let mut merges = 0; let mut queries = 0; @@ -82,16 +105,16 @@ pub fn benchmark( .map(|i| window + (data.len() - window) * i / 4) .collect(); for end in ends { - let t = Instant::now(); + let t = process_cpu_seconds()?; let mut merged = states[end - window].clone(); for s in &states[end - window + 1..end] { merged.merge(s)?; merges += 1; } - merge_seconds += t.elapsed().as_secs_f64(); - let t = Instant::now(); + merge_seconds += (process_cpu_seconds()? - t).max(0.0); + let t = process_cpu_seconds()?; let predicted = merged.topk(); - query_seconds += t.elapsed().as_secs_f64(); + query_seconds += (process_cpu_seconds()? - t).max(0.0); queries += 1; loss[q] = loss[q].max(1. - recall(&predicted, &exact_topk(data, end, window))); } @@ -167,7 +190,7 @@ pub fn build( }, shapes, generation_seconds: began.elapsed().as_secs_f64(), - provenance: serde_json::json!({"args":a,"timing":"wall seconds per operation, not hardware CPU counters","scope":"window-conditioned extension of sketch-bench ERP v1; generated by backend adapter","held_out_used":false}), + provenance: serde_json::json!({"args":a,"resource_time_basis":"process_cpu_seconds","timing":"process CPU seconds per operation; generation_seconds is wall time","scope":"window-conditioned extension of sketch-bench ERP v1; generated by backend adapter","held_out_used":false}), }) } @@ -190,6 +213,17 @@ pub fn select( ) -> Result> { let began = Instant::now(); catalog.artifact.validate()?; + if catalog + .provenance + .get("resource_time_basis") + .and_then(serde_json::Value::as_str) + != Some("process_cpu_seconds") + { + return Err( + "ERP catalog CPU timing is missing or uses wall time; recalibrate before selection" + .into(), + ); + } let observed = Shape::observe(data); let mut matches: Vec<_> = catalog .shapes @@ -284,6 +318,31 @@ pub fn select( #[cfg(test)] mod tests { use super::*; + // Exercise the OS CPU clock through real update, merge, and readout work. + #[cfg(target_os = "linux")] + #[test] + fn calibration_records_process_cpu_for_each_operation() { + let data = vec![vec![1, 2]; 120]; + let (loss, resources) = benchmark( + &data, + Config { + family: Family::Cms, + rows: 3, + cols: 128, + heap: 16, + }, + ) + .unwrap(); + assert!(loss.iter().all(|loss| *loss == 0.0)); + for cpu in [ + resources.update_cpu_seconds, + resources.merge_cpu_seconds, + resources.query_cpu_seconds, + ] { + assert!(cpu.is_finite() && cpu > 0.0); + } + } + // Resource selection must react to measured evidence, not width thresholds. #[test] fn shape_detects_cardinality_and_rate_mismatch() { @@ -333,8 +392,14 @@ mod tests { }, shapes: BTreeMap::from([("test".into(), shape)]), generation_seconds: 0., - provenance: serde_json::json!({}), + provenance: serde_json::json!({"resource_time_basis": "process_cpu_seconds"}), }; + let provenance = catalog.provenance.take(); + assert!(select(&catalog, &data, &a, true) + .unwrap_err() + .to_string() + .contains("CPU timing")); + catalog.provenance = provenance; assert_eq!( select(&catalog, &data, &a, true).unwrap().configs, vec![small] diff --git a/tools/autosketch-comparison/data/topk-dashboard-v2/report.md b/tools/autosketch-comparison/data/topk-dashboard-v2/report.md index 2b6e1ee0..e3bb0eca 100644 --- a/tools/autosketch-comparison/data/topk-dashboard-v2/report.md +++ b/tools/autosketch-comparison/data/topk-dashboard-v2/report.md @@ -1,5 +1,11 @@ # Measured ERP Top-K dashboard comparison (v2) +The committed v2 artifacts are historical wall-time measurements. Their ERP +`*_cpu_seconds` fields were populated from elapsed wall time and must not be +used as CPU evidence. The corrected runner measures process CPU time and rejects +those old catalogs; recalibration into a fresh output directory is required. +No corrected performance numbers have been substituted into the historical report. + Supersedes the withdrawn v1 measurements. Values below are generated from release-mode raw data; they are not smoke-test results. Four TopK(10) frequency queries cover the latest 1, 5, 15, and 60 minutes. A new 30-second pane arrives before every dashboard refresh. Each trial evaluates 100 refreshes (400 panel queries), with exactly the same endpoints for all methods. Calibration uses the first 120 panes; held-out endpoints are 121–220. Update time includes warm-up and ingestion of all 220 panes. Events update sketches in their original order, one event per update. diff --git a/tools/autosketch-comparison/reproduce_topk_erp.py b/tools/autosketch-comparison/reproduce_topk_erp.py index ea35b14f..9f487a9d 100644 --- a/tools/autosketch-comparison/reproduce_topk_erp.py +++ b/tools/autosketch-comparison/reproduce_topk_erp.py @@ -25,6 +25,8 @@ def run(name, extra): if output.exists(): existing = json.loads(output.read_text()) recorded = existing.get("args", existing.get("provenance", {}).get("args", {})) + if "--build-erp" in extra and existing.get("provenance", {}).get("resource_time_basis") != "process_cpu_seconds": + raise ValueError(f"ERP CPU evidence must be recalibrated: {output}") if not recorded.get("backend_revision") or ("--build-erp" not in extra and recorded.get("backend_revision") != args.revision): raise ValueError(f"stale output: {output}") else: @@ -38,7 +40,7 @@ def run(name, extra): ("google", run("google-profile.json", ["--build-erp", "--input-tsv", str(args.google_replay)])), ] combined = {"artifact": {"schema_version": 1, "producer_version": args.revision, "records": []}, - "shapes": {}, "generation_seconds": 0, "provenance": {"sources": []}} + "shapes": {}, "generation_seconds": 0, "provenance": {"sources": [], "resource_time_basis": "process_cpu_seconds"}} for name, catalog in catalogs: for key, value in catalog["shapes"].items(): combined["shapes"][f"{name}/{key}"] = value diff --git a/tools/autosketch-comparison/topk-dashboard-results.md b/tools/autosketch-comparison/topk-dashboard-results.md index c8c334fc..b5549580 100644 --- a/tools/autosketch-comparison/topk-dashboard-results.md +++ b/tools/autosketch-comparison/topk-dashboard-results.md @@ -1,5 +1,11 @@ # Top-K dashboard results +The committed v2 artifacts are historical wall-time measurements. Their ERP +`*_cpu_seconds` fields were populated from elapsed wall time and must not be +used as CPU evidence. The corrected runner measures process CPU time and rejects +those old catalogs; recalibration into a fresh output directory is required. +No corrected performance numbers have been substituted into the historical report. + The v1 measurements are withdrawn. They used hardcoded ERP parameters and contained sampling, event-order, timestamp-alignment, and memory-accounting errors. Their raw files remain recoverable in git history at `c91d19e2` but must From 8c99cccecd91f2f5a1170ac0aeba027815480644 Mon Sep 17 00:00:00 2001 From: zz_y Date: Thu, 10 Sep 2026 19:59:57 -0600 Subject: [PATCH 2/2] docs(eval): preserve CPU provenance in regenerated reports --- tools/autosketch-comparison/summarize_topk_erp.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tools/autosketch-comparison/summarize_topk_erp.py b/tools/autosketch-comparison/summarize_topk_erp.py index 67cac8fb..21ce3a96 100644 --- a/tools/autosketch-comparison/summarize_topk_erp.py +++ b/tools/autosketch-comparison/summarize_topk_erp.py @@ -108,11 +108,15 @@ def main(): plt.close(fig) text += [f"![{name} measured results]({name}.svg)", ""] catalog=json.loads((root/"catalog.json").read_text()) + if catalog.get("provenance", {}).get("resource_time_basis") != "process_cpu_seconds": + text[2:2] = ["Historical timing limitation: this catalog wrote wall time into ERP CPU fields. These artifacts are not CPU evidence; recalibrate before using the corrected selector.", ""] + else: + text += ["ERP operation resources use process CPU time. Profile construction and dashboard latency use elapsed wall time.", ""] text += ["## Profile construction and scope", "", "| Catalog source | Measured construction seconds |", "|---|---:|"] for source in catalog["provenance"]["sources"]: text.append(f"| {source['name']} | {source['generation_seconds']:.6g} |") text += ["", "For a user dataset without an existing profile, cold-start cost includes its profile construction plus loading and selection. Google profiling replays the same calibration prefix three times; these are timing repetitions, not independent distribution samples. Synthetic profiles use three independent streams. No held-out events enter profile construction or shape observation.", "", - "This PR evaluates measured configuration selection through the real Planner ERP selector and shared/independent 30-second pane execution. It does not implement production online shape observation, arbitrary pane-width search, drift-triggered replanning, or a formal recall guarantee. The benchmark adapter emits ERP-compatible evidence; it is not an invocation of the sketch-bench executable. Memory is minimized first; empirical CPU costs are composed and recorded as estimates, not used as a competing optimization objective. Timings are sequential wall measurements on a shared host; consult manifest.json for revisions, commands and checksums.", ""] + "This PR evaluates measured configuration selection through the real Planner ERP selector and shared/independent 30-second pane execution. It does not implement production online shape observation, arbitrary pane-width search, drift-triggered replanning, or a formal recall guarantee. The benchmark adapter emits ERP-compatible evidence; it is not an invocation of the sketch-bench executable. Memory is minimized first; empirical CPU costs are composed and recorded as estimates, not used as a competing optimization objective. Dashboard timings are elapsed wall measurements; ERP CPU provenance is stated above; consult manifest.json for revisions, commands and checksums.", ""] (root/"report.md").write_text("\n".join(text))