Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions packages/orchestrator/pkg/sandbox/snapshot_metrics.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@ var (
snapshotDiffBytes = utils.Must(telemetry.GetHistogram(meter, telemetry.SnapshotDiffBytes))
snapshotDiffRatio = utils.Must(telemetry.GetFloatHistogram(meter, telemetry.SnapshotDiffRatio))
snapshotTotalBytes = utils.Must(telemetry.GetHistogram(meter, telemetry.SnapshotTotalBytes))
checkpointChangeFootprint = utils.Must(telemetry.GetCounter(meter, telemetry.OrchestratorCheckpointChangeFootprintCounterName))
)

type SnapshotUseCase string
Expand All @@ -26,6 +27,16 @@ const (
SnapshotUseCaseBuild SnapshotUseCase = "build"
)

// snapshotCleanThresholdBytes is the dirty-bytes ceiling (one typical block)
// at or below which a per-file snapshot diff counts as "clean" — nothing
// recovery-relevant changed in that dimension since the diff's baseline.
const snapshotCleanThresholdBytes = 4096

// snapshotDiffClean reports whether a dirty-byte count is clean (<= one block).
func snapshotDiffClean(dirtyBytes int64) bool {
return dirtyBytes <= snapshotCleanThresholdBytes
}

func recordSnapshotDiff(
ctx context.Context,
teamID string,
Expand All @@ -51,6 +62,16 @@ func recordSnapshotDiff(
if dm.Empty != nil {
emptyBytes = int64(dm.Empty.GetCardinality()) * bs
}

// Change-footprint: one count per file_type per pause, sliced by whether
// this dimension was clean. The clean rate is the fs-only-downgrade / skip
// opportunity measured on real traffic (answers "how often is a checkpoint
// a no-op?"); both dimensions clean on one pause is the full-skip rate.
checkpointChangeFootprint.Add(ctx, 1, metric.WithAttributes(
ft,
attribute.Bool("clean", snapshotDiffClean(dirtyBytes)),
team,
))
for kind, b := range map[string]int64{
"dirty": dirtyBytes,
"empty": emptyBytes,
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
//go:build linux

package sandbox

import "testing"

func TestSnapshotDiffClean(t *testing.T) {
cases := []struct {
name string
dirty int64
want bool
}{
{"zero dirty is clean", 0, true},
{"one block is clean", snapshotCleanThresholdBytes, true},
{"just over one block is dirty", snapshotCleanThresholdBytes + 1, false},
{"large dirty is dirty", 119537664, false},
}
for _, c := range cases {
t.Run(c.name, func(t *testing.T) {
if got := snapshotDiffClean(c.dirty); got != c.want {
t.Fatalf("snapshotDiffClean(%d) = %v, want %v", c.dirty, got, c.want)
}
})
}
}
11 changes: 11 additions & 0 deletions packages/shared/pkg/telemetry/meters.go
Original file line number Diff line number Diff line change
Expand Up @@ -119,6 +119,15 @@ const (
// A non-zero rate means lost snapshots.
OrchestratorSnapshotUploadFailedCounterName CounterType = "orchestrator.snapshot.upload.failed"

// OrchestratorCheckpointChangeFootprintCounterName counts each per-pause
// snapshot diff by file_type (memfile|rootfs) and whether it was clean
// (dirty bytes at or below a small block-sized threshold). It is the
// measurement that answers "how often is a checkpoint a no-op?" — the
// per-dimension clean rate is the filesystem-only-downgrade / skip
// opportunity on real traffic; the full-skip rate is both dimensions clean
// on the same pause (join by sandbox/trace in the dashboard).
OrchestratorCheckpointChangeFootprintCounterName CounterType = "orchestrator.sandbox.checkpoint.change_footprint"

// OrchestratorTemplateCacheAncestorResolutionsCounterName counts each
// ancestor build once per ancestor walk, by the path that resolved it
// (verdict) and what landed in the child header's Builds map (resolution).
Expand Down Expand Up @@ -820,6 +829,7 @@ var counterDesc = map[CounterType]string{
OrchestratorFPHFreedCounterName: "Bytes discarded by completed free-page-hinting drains and runs (FC free_page_hint_freed before the drain and after a metrics flush that follows it), labeled by phase",
OrchestratorFPHLatchCounterName: "Guests the periodic hinter latched as unable to hint after consecutive sat-out runs, released by a completed run, or dropped with the loop that held the latch (state: latched, released, dropped)",
OrchestratorSnapshotUploadFailedCounterName: "Number of pause-snapshot uploads that never landed durably",
OrchestratorCheckpointChangeFootprintCounterName: "Per-pause snapshot diffs by file_type (memfile|rootfs) and clean (dirty bytes <= one block); the clean rate is the fs-only-downgrade / skip opportunity",
OrchestratorDeadStructureOutcomeCounterName: "Passes through a per-pause structure's drop site, by structure and outcome",
OrchestratorDeadStructureBytesCounterName: "Bytes of per-pause structures dropped or kept at their drop site, by structure and outcome",
SandboxPauseFsQuiescedCounterName: "Filesystem-only pauses by whether the rootfs was frozen (quiesced) vs sync fallback",
Expand Down Expand Up @@ -911,6 +921,7 @@ var counterUnits = map[CounterType]string{
OrchestratorFPHFreedCounterName: "By",
OrchestratorFPHLatchCounterName: "{transition}",
OrchestratorSnapshotUploadFailedCounterName: "{snapshot}",
OrchestratorCheckpointChangeFootprintCounterName: "{pause}",
OrchestratorDeadStructureOutcomeCounterName: "{pass}",
OrchestratorDeadStructureBytesCounterName: "By",
SandboxPauseFsQuiescedCounterName: "{snapshot}",
Expand Down