From 76779da1c3c9b368f99539b23cc158a909168e79 Mon Sep 17 00:00:00 2001 From: adababys <8872412+adababys@users.noreply.github.com> Date: Tue, 27 Oct 2026 10:49:58 +0800 Subject: [PATCH] feat(orchestrator): checkpoint change-footprint metric (skip/fs-only opportunity) Add orchestrator.sandbox.checkpoint.change_footprint, a per-pause counter sliced by file_type (memfile|rootfs) and clean (dirty bytes <= one block), emitted from recordSnapshotDiff off data already computed every pause. The per-dimension clean rate is the filesystem-only-downgrade / skip opportunity measured on real traffic; both dimensions clean on one pause is the full-skip rate (join by sandbox/trace in the dashboard). This is the measurement a skip-unchanged feature needs BEFORE adding any mechanism: it answers how often a checkpoint is a no-op, from existing signals, with no new machinery, no in-guest agent, and no behavior change. --- .../pkg/sandbox/snapshot_metrics.go | 21 ++++++++++++++++ .../snapshot_metrics_footprint_test.go | 25 +++++++++++++++++++ packages/shared/pkg/telemetry/meters.go | 11 ++++++++ 3 files changed, 57 insertions(+) create mode 100644 packages/orchestrator/pkg/sandbox/snapshot_metrics_footprint_test.go diff --git a/packages/orchestrator/pkg/sandbox/snapshot_metrics.go b/packages/orchestrator/pkg/sandbox/snapshot_metrics.go index a82872d8ec..07c0c1f046 100644 --- a/packages/orchestrator/pkg/sandbox/snapshot_metrics.go +++ b/packages/orchestrator/pkg/sandbox/snapshot_metrics.go @@ -17,6 +17,7 @@ var ( snapshotDiffBytes = utils.Must(telemetry.GetHistogram(meter, telemetry.SnapshotDiffBytes)) snapshotDiffRatio = utils.Must(telemetry.GetFloatHistogram(meter, telemetry.SnapshotDiffRatio)) snapshotTotalBytes = utils.Must(telemetry.GetHistogram(meter, telemetry.SnapshotTotalBytes)) + checkpointChangeFootprint = utils.Must(telemetry.GetCounter(meter, telemetry.OrchestratorCheckpointChangeFootprintCounterName)) ) type SnapshotUseCase string @@ -26,6 +27,16 @@ const ( SnapshotUseCaseBuild SnapshotUseCase = "build" ) +// snapshotCleanThresholdBytes is the dirty-bytes ceiling (one typical block) +// at or below which a per-file snapshot diff counts as "clean" — nothing +// recovery-relevant changed in that dimension since the diff's baseline. +const snapshotCleanThresholdBytes = 4096 + +// snapshotDiffClean reports whether a dirty-byte count is clean (<= one block). +func snapshotDiffClean(dirtyBytes int64) bool { + return dirtyBytes <= snapshotCleanThresholdBytes +} + func recordSnapshotDiff( ctx context.Context, teamID string, @@ -51,6 +62,16 @@ func recordSnapshotDiff( if dm.Empty != nil { emptyBytes = int64(dm.Empty.GetCardinality()) * bs } + + // Change-footprint: one count per file_type per pause, sliced by whether + // this dimension was clean. The clean rate is the fs-only-downgrade / skip + // opportunity measured on real traffic (answers "how often is a checkpoint + // a no-op?"); both dimensions clean on one pause is the full-skip rate. + checkpointChangeFootprint.Add(ctx, 1, metric.WithAttributes( + ft, + attribute.Bool("clean", snapshotDiffClean(dirtyBytes)), + team, + )) for kind, b := range map[string]int64{ "dirty": dirtyBytes, "empty": emptyBytes, diff --git a/packages/orchestrator/pkg/sandbox/snapshot_metrics_footprint_test.go b/packages/orchestrator/pkg/sandbox/snapshot_metrics_footprint_test.go new file mode 100644 index 0000000000..69505f9692 --- /dev/null +++ b/packages/orchestrator/pkg/sandbox/snapshot_metrics_footprint_test.go @@ -0,0 +1,25 @@ +//go:build linux + +package sandbox + +import "testing" + +func TestSnapshotDiffClean(t *testing.T) { + cases := []struct { + name string + dirty int64 + want bool + }{ + {"zero dirty is clean", 0, true}, + {"one block is clean", snapshotCleanThresholdBytes, true}, + {"just over one block is dirty", snapshotCleanThresholdBytes + 1, false}, + {"large dirty is dirty", 119537664, false}, + } + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + if got := snapshotDiffClean(c.dirty); got != c.want { + t.Fatalf("snapshotDiffClean(%d) = %v, want %v", c.dirty, got, c.want) + } + }) + } +} diff --git a/packages/shared/pkg/telemetry/meters.go b/packages/shared/pkg/telemetry/meters.go index f710e513f1..2e3887cb45 100644 --- a/packages/shared/pkg/telemetry/meters.go +++ b/packages/shared/pkg/telemetry/meters.go @@ -119,6 +119,15 @@ const ( // A non-zero rate means lost snapshots. OrchestratorSnapshotUploadFailedCounterName CounterType = "orchestrator.snapshot.upload.failed" + // OrchestratorCheckpointChangeFootprintCounterName counts each per-pause + // snapshot diff by file_type (memfile|rootfs) and whether it was clean + // (dirty bytes at or below a small block-sized threshold). It is the + // measurement that answers "how often is a checkpoint a no-op?" — the + // per-dimension clean rate is the filesystem-only-downgrade / skip + // opportunity on real traffic; the full-skip rate is both dimensions clean + // on the same pause (join by sandbox/trace in the dashboard). + OrchestratorCheckpointChangeFootprintCounterName CounterType = "orchestrator.sandbox.checkpoint.change_footprint" + // OrchestratorTemplateCacheAncestorResolutionsCounterName counts each // ancestor build once per ancestor walk, by the path that resolved it // (verdict) and what landed in the child header's Builds map (resolution). @@ -820,6 +829,7 @@ var counterDesc = map[CounterType]string{ OrchestratorFPHFreedCounterName: "Bytes discarded by completed free-page-hinting drains and runs (FC free_page_hint_freed before the drain and after a metrics flush that follows it), labeled by phase", OrchestratorFPHLatchCounterName: "Guests the periodic hinter latched as unable to hint after consecutive sat-out runs, released by a completed run, or dropped with the loop that held the latch (state: latched, released, dropped)", OrchestratorSnapshotUploadFailedCounterName: "Number of pause-snapshot uploads that never landed durably", + OrchestratorCheckpointChangeFootprintCounterName: "Per-pause snapshot diffs by file_type (memfile|rootfs) and clean (dirty bytes <= one block); the clean rate is the fs-only-downgrade / skip opportunity", OrchestratorDeadStructureOutcomeCounterName: "Passes through a per-pause structure's drop site, by structure and outcome", OrchestratorDeadStructureBytesCounterName: "Bytes of per-pause structures dropped or kept at their drop site, by structure and outcome", SandboxPauseFsQuiescedCounterName: "Filesystem-only pauses by whether the rootfs was frozen (quiesced) vs sync fallback", @@ -911,6 +921,7 @@ var counterUnits = map[CounterType]string{ OrchestratorFPHFreedCounterName: "By", OrchestratorFPHLatchCounterName: "{transition}", OrchestratorSnapshotUploadFailedCounterName: "{snapshot}", + OrchestratorCheckpointChangeFootprintCounterName: "{pause}", OrchestratorDeadStructureOutcomeCounterName: "{pass}", OrchestratorDeadStructureBytesCounterName: "By", SandboxPauseFsQuiescedCounterName: "{snapshot}",