diff --git a/packages/orchestrator/pkg/factories/run.go b/packages/orchestrator/pkg/factories/run.go index a24ffc8aa0..56fe63f149 100644 --- a/packages/orchestrator/pkg/factories/run.go +++ b/packages/orchestrator/pkg/factories/run.go @@ -813,7 +813,7 @@ func run(config cfg.Config, opts Options) (success bool) { closers = append(closers, closer{"sandbox events deliveries (all)", eventsService.Close}) // sandbox observer - sandboxObserver, err := metrics.NewSandboxObserver(ctx, nodeID, serviceName, commitSHA, version, serviceInstanceID, sandboxes) + sandboxObserver, err := metrics.NewSandboxObserver(ctx, nodeID, serviceName, commitSHA, version, serviceInstanceID, sandboxes, featureFlags) if err != nil { logger.L().Fatal(ctx, "failed to create sandbox observer", zap.Error(err)) } diff --git a/packages/orchestrator/pkg/metrics/reclaimable_test.go b/packages/orchestrator/pkg/metrics/reclaimable_test.go new file mode 100644 index 0000000000..a4303b7177 --- /dev/null +++ b/packages/orchestrator/pkg/metrics/reclaimable_test.go @@ -0,0 +1,70 @@ +//go:build linux + +package metrics + +import "testing" + +// reclaimableBytes is the estimate the density-observability gauge exports; it +// must never over-report (it feeds a later reclaim decision), so these lock its +// floor-at-zero and free-page-reporting behavior. +func TestReclaimableBytes(t *testing.T) { + t.Parallel() + + const mib = int64(1) << 20 + + tests := []struct { + name string + faulted int64 + guestInUse int64 + freePageReporting bool + want int64 + }{ + { + name: "faulted above in-use yields the gap", + faulted: 2048 * mib, + guestInUse: 512 * mib, + want: 1536 * mib, + }, + { + name: "in-use above faulted floors at zero (never negative)", + faulted: 256 * mib, + guestInUse: 512 * mib, + want: 0, + }, + { + name: "equal footprint and use yields zero", + faulted: 512 * mib, + guestInUse: 512 * mib, + want: 0, + }, + { + name: "no guest report yields the whole footprint as upper bound", + faulted: 1024 * mib, + guestInUse: 0, + want: 1024 * mib, + }, + { + name: "free-page-reporting active reports zero (pool self-reclaims)", + faulted: 2048 * mib, + guestInUse: 128 * mib, + freePageReporting: true, + want: 0, + }, + { + name: "zero footprint yields zero", + faulted: 0, guestInUse: 0, want: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := reclaimableBytes(tt.faulted, tt.guestInUse, tt.freePageReporting) + if got != tt.want { + t.Errorf("reclaimableBytes(%d, %d, %v) = %d, want %d", + tt.faulted, tt.guestInUse, tt.freePageReporting, got, tt.want) + } + }) + } +} diff --git a/packages/orchestrator/pkg/metrics/sandboxes.go b/packages/orchestrator/pkg/metrics/sandboxes.go index f40d84dd37..64ec82bc83 100644 --- a/packages/orchestrator/pkg/metrics/sandboxes.go +++ b/packages/orchestrator/pkg/metrics/sandboxes.go @@ -19,6 +19,7 @@ import ( "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox" "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/envd" + "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" "github.com/e2b-dev/infra/packages/shared/pkg/logger" sbxlogger "github.com/e2b-dev/infra/packages/shared/pkg/logger/sandbox" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -61,21 +62,75 @@ type SandboxObserver struct { sandboxes *sandbox.Map - meter metric.Meter - cpuTotal metric.Int64ObservableGauge - cpuUsed metric.Float64ObservableGauge - cpuPossible metric.Int64ObservableGauge - cpuTarget metric.Int64ObservableGauge - cpuAttempts metric.Int64ObservableGauge - cpuPending metric.Int64ObservableGauge - memoryTotal metric.Int64ObservableGauge - memoryUsed metric.Int64ObservableGauge - memoryCache metric.Int64ObservableGauge - diskTotal metric.Int64ObservableGauge - diskUsed metric.Int64ObservableGauge + meter metric.Meter + cpuTotal metric.Int64ObservableGauge + cpuUsed metric.Float64ObservableGauge + cpuPossible metric.Int64ObservableGauge + cpuTarget metric.Int64ObservableGauge + cpuAttempts metric.Int64ObservableGauge + cpuPending metric.Int64ObservableGauge + memoryTotal metric.Int64ObservableGauge + memoryUsed metric.Int64ObservableGauge + memoryCache metric.Int64ObservableGauge + memWorkingSet metric.Int64ObservableGauge + memReclaimable metric.Int64ObservableGauge + memShareableBase metric.Int64ObservableGauge + diskTotal metric.Int64ObservableGauge + diskUsed metric.Int64ObservableGauge + + featureFlags *featureflags.Client } -func NewSandboxObserver(ctx context.Context, nodeID, serviceName, serviceCommit, serviceVersion, serviceInstanceID string, sandboxes *sandbox.Map) (*SandboxObserver, error) { +// reclaimableBytes estimates pool memory the node holds for a sandbox that the +// guest is not actively using — the opportunity a later live reclaim (balloon +// inflate + host-side punch) could return to the hugepage pool. +// +// - faultedBytes: host bytes faulted into the pool since resume +// (MemoryWorkingSetBytes); an upper bound on current pool residency. +// - guestInUseBytes: the guest's actively-used RAM (envd-reported, excluding +// reclaimable page cache). 0/absent yields the whole footprint as the +// upper-bound estimate. +// - freePageReporting: whether the sandbox's balloon returns freed pages to +// the host itself. When true the pool is largely self-reclaiming, so the +// standing reclaim opportunity is reported as 0 rather than double-counting +// pages the device already discards. +// +// shareableBaseBytes is a per-template lower bound on the pool memory a +// same-template page-sharing scheme could reclaim on this node: with N +// sandboxes of one template each holding a private copy of the base image, one +// shared copy would keep one and free the other N-1. Using the SMALLEST +// base-image footprint among them keeps it a lower bound — it never claims more +// than every sandbox demonstrably faulted. Fewer than two sandboxes share +// nothing, so it returns 0. +func shareableBaseBytes(sourceBytes []int64) int64 { + if len(sourceBytes) < 2 { + return 0 + } + + minBytes := sourceBytes[0] + for _, b := range sourceBytes[1:] { + if b < minBytes { + minBytes = b + } + } + + return int64(len(sourceBytes)-1) * minBytes +} + +func reclaimableBytes(faultedBytes, guestInUseBytes int64, freePageReporting bool) int64 { + if freePageReporting { + return 0 + } + + r := faultedBytes - guestInUseBytes + if r < 0 { + return 0 + } + + return r +} + +func NewSandboxObserver(ctx context.Context, nodeID, serviceName, serviceCommit, serviceVersion, serviceInstanceID string, sandboxes *sandbox.Map, featureFlags *featureflags.Client) (*SandboxObserver, error) { deltaTemporality := otlpmetricgrpc.WithTemporalitySelector(func(kind sdkmetric.InstrumentKind) metricdata.Temporality { // Use delta temporality for gauges and cumulative for all other instrument kinds. // This is used to prevent reporting sandbox metrics indefinitely. @@ -163,22 +218,41 @@ func NewSandboxObserver(ctx context.Context, nodeID, serviceName, serviceCommit, return nil, fmt.Errorf("failed to create disk used gauge: %w", err) } + memWorkingSet, err := telemetry.GetGaugeInt(meter, telemetry.SandboxMemWorkingSetGaugeName) + if err != nil { + return nil, fmt.Errorf("failed to create memory working-set gauge: %w", err) + } + + memReclaimable, err := telemetry.GetGaugeInt(meter, telemetry.SandboxMemReclaimableGaugeName) + if err != nil { + return nil, fmt.Errorf("failed to create memory reclaimable gauge: %w", err) + } + + memShareableBase, err := telemetry.GetGaugeInt(meter, telemetry.SandboxMemShareableBaseGaugeName) + if err != nil { + return nil, fmt.Errorf("failed to create memory shareable-base gauge: %w", err) + } + so := &SandboxObserver{ - exportInterval: sandboxMetricExportPeriod, - meterExporter: externalMeterExporter, - sandboxes: sandboxes, - meter: meter, - cpuTotal: cpuTotal, - cpuUsed: cpuUsed, - cpuPossible: cpuPossible, - cpuTarget: cpuTarget, - cpuAttempts: cpuAttempts, - cpuPending: cpuPending, - memoryTotal: memoryTotal, - memoryUsed: memoryUsed, - memoryCache: memoryCache, - diskTotal: diskTotal, - diskUsed: diskUsed, + exportInterval: sandboxMetricExportPeriod, + meterExporter: externalMeterExporter, + sandboxes: sandboxes, + meter: meter, + cpuTotal: cpuTotal, + cpuUsed: cpuUsed, + cpuPossible: cpuPossible, + cpuTarget: cpuTarget, + cpuAttempts: cpuAttempts, + cpuPending: cpuPending, + memoryTotal: memoryTotal, + memoryUsed: memoryUsed, + memoryCache: memoryCache, + diskTotal: diskTotal, + diskUsed: diskUsed, + memWorkingSet: memWorkingSet, + memReclaimable: memReclaimable, + memShareableBase: memShareableBase, + featureFlags: featureFlags, } registration, err := so.startObserving() @@ -199,6 +273,24 @@ func (so *SandboxObserver) startObserving() (metric.Registration, error) { // first Go forever, and with it this callback and the SDK pipeline. sbxs := so.sandboxes.Items() + // Density observability (off by default): per-template lower bound on + // pool memory a same-template page-sharing scheme could reclaim. In + // memory only (no RPC): group the resident sandboxes by build and emit + // (N-1) x min(base footprint) per build with >= 2 sandboxes. + if so.featureFlags != nil && so.featureFlags.BoolFlag(ctx, featureflags.SandboxDensityMetricsFlag) { + byBuild := make(map[string][]int64) + for _, sbx := range sbxs { + byBuild[sbx.Runtime.BuildID] = append(byBuild[sbx.Runtime.BuildID], sbx.MemorySourceBytes()) + } + for buildID, srcBytes := range byBuild { + est := shareableBaseBytes(srcBytes) + if est == 0 { + continue + } + o.ObserveInt64(so.memShareableBase, est, metric.WithAttributes(attribute.String("build_id", buildID))) + } + } + wg := errgroup.Group{} // Run concurrently to prevent blocking if there are many sandboxes other callbacks limit := math.Ceil(float64(len(sbxs)) / metricsParallelismFactor) @@ -285,6 +377,22 @@ func (so *SandboxObserver) startObserving() (metric.Registration, error) { o.ObserveInt64(so.memoryUsed, memoryUsed, attributes) } + // Density observability (off by default): host-pool footprint + // and reclaim estimate. Flag-gated so no new series are produced + // until explicitly enabled. + if so.featureFlags != nil && so.featureFlags.BoolFlag(ctx, featureflags.SandboxDensityMetricsFlag) { + workingSet := sbx.MemoryWorkingSetBytes() + o.ObserveInt64(so.memWorkingSet, workingSet, attributes) + + guestInUse := int64(0) + if memoryReported { + guestInUse = memoryUsed + } + o.ObserveInt64(so.memReclaimable, + reclaimableBytes(workingSet, guestInUse, sbx.FreePageReportingActive()), + attributes) + } + ok, err := utils.IsGTEVersion(sbx.Config.Envd.Version, minEnvdVersionForCacheMetrics) if err != nil { logger.L().Error(ctx, "Failed to check envd version for cache metrics", zap.Error(err), logger.WithSandboxID(sbx.Runtime.SandboxID)) @@ -343,7 +451,8 @@ func (so *SandboxObserver) startObserving() (metric.Registration, error) { return nil }, so.cpuTotal, so.cpuUsed, so.cpuPossible, so.cpuTarget, so.cpuAttempts, so.cpuPending, - so.memoryTotal, so.memoryUsed, so.memoryCache, so.diskTotal, so.diskUsed) + so.memoryTotal, so.memoryUsed, so.memoryCache, so.diskTotal, so.diskUsed, + so.memWorkingSet, so.memReclaimable, so.memShareableBase) if err != nil { return nil, err } diff --git a/packages/orchestrator/pkg/metrics/sandboxes_test.go b/packages/orchestrator/pkg/metrics/sandboxes_test.go index d55c065e26..834731b219 100644 --- a/packages/orchestrator/pkg/metrics/sandboxes_test.go +++ b/packages/orchestrator/pkg/metrics/sandboxes_test.go @@ -110,6 +110,9 @@ func newTestSandboxObserver(t *testing.T, meter metric.Meter, sandboxes *sandbox {&so.memoryCache, telemetry.SandboxRamCacheGaugeName}, {&so.diskTotal, telemetry.SandboxDiskTotalGaugeName}, {&so.diskUsed, telemetry.SandboxDiskUsedGaugeName}, + {&so.memWorkingSet, telemetry.SandboxMemWorkingSetGaugeName}, + {&so.memReclaimable, telemetry.SandboxMemReclaimableGaugeName}, + {&so.memShareableBase, telemetry.SandboxMemShareableBaseGaugeName}, } { gauge, err := telemetry.GetGaugeInt(meter, g.name) require.NoError(t, err) diff --git a/packages/orchestrator/pkg/metrics/shareable_test.go b/packages/orchestrator/pkg/metrics/shareable_test.go new file mode 100644 index 0000000000..96d180da43 --- /dev/null +++ b/packages/orchestrator/pkg/metrics/shareable_test.go @@ -0,0 +1,34 @@ +//go:build linux + +package metrics + +import "testing" + +// shareableBaseBytes is the per-template lower bound driving the density +// go/no-go metric; it must never over-report, so these lock the floor and the +// (N-1)*min shape. +func TestShareableBaseBytes(t *testing.T) { + t.Parallel() + + const mib = int64(1) << 20 + tests := []struct { + name string + src []int64 + want int64 + }{ + {"no sandboxes shares nothing", nil, 0}, + {"one sandbox shares nothing", []int64{512 * mib}, 0}, + {"two equal: one copy freeable", []int64{512 * mib, 512 * mib}, 512 * mib}, + {"two unequal: uses the smaller (lower bound)", []int64{512 * mib, 300 * mib}, 300 * mib}, + {"four sandboxes: (4-1)*min", []int64{400 * mib, 500 * mib, 420 * mib, 410 * mib}, 3 * 400 * mib}, + {"a zero-footprint member drags the lower bound to 0", []int64{512 * mib, 0, 512 * mib}, 0}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := shareableBaseBytes(tt.src); got != tt.want { + t.Errorf("shareableBaseBytes(%v) = %d, want %d", tt.src, got, tt.want) + } + }) + } +} diff --git a/packages/orchestrator/pkg/sandbox/sandbox.go b/packages/orchestrator/pkg/sandbox/sandbox.go index 3837dbc6ea..40c81dbc5b 100644 --- a/packages/orchestrator/pkg/sandbox/sandbox.go +++ b/packages/orchestrator/pkg/sandbox/sandbox.go @@ -2176,6 +2176,37 @@ func (s *Sandbox) FirecrackerExit() *fc.ExitInfo { // MemoryHandlerErr reports why the memory handler exited, and nil if it has // not exited or exited cleanly. Only meaningful once the execution has ended. +// MemoryWorkingSetBytes is the host memory faulted into the hugepage pool for +// this sandbox since resume — its pool footprint and an upper bound on current +// residency (UFFD serves are cumulative and hugetlbfs pages are not returned +// without explicit reclaim). 0 when no memory backend is attached. +func (s *Sandbox) MemoryWorkingSetBytes() int64 { + if s.Resources == nil || s.memory == nil { + return 0 + } + + return s.memory.ServeStats().Bytes +} + +// MemorySourceBytes is the base-image content this sandbox faulted in from the +// shared template source (SourcePages in bytes) — the portion of its pool +// footprint that a same-template page-sharing scheme could collapse to one +// shared physical copy. 0 when no memory backend is attached. +func (s *Sandbox) MemorySourceBytes() int64 { + if s.Resources == nil || s.memory == nil { + return 0 + } + + return s.memory.ServeStats().SourceBytes +} + +// FreePageReportingActive reports whether this sandbox's balloon returns freed +// guest pages to the host on its own, so the pool is largely self-reclaiming. +// Read from the cached balloon mode; no device round-trip. +func (s *Sandbox) FreePageReportingActive() bool { + return s.BalloonModeValue() == userfaultfd.BalloonModeReporting +} + func (s *Sandbox) MemoryHandlerErr() error { if s.Resources == nil || s.memory == nil { return nil diff --git a/packages/orchestrator/pkg/sandbox/uffd/userfaultfd/metrics.go b/packages/orchestrator/pkg/sandbox/uffd/userfaultfd/metrics.go index d89f29ea38..b47dc3078a 100644 --- a/packages/orchestrator/pkg/sandbox/uffd/userfaultfd/metrics.go +++ b/packages/orchestrator/pkg/sandbox/uffd/userfaultfd/metrics.go @@ -386,6 +386,12 @@ type ServeSnapshot struct { // times the page size, but it is tracked directly so it stays correct // across mixed page sizes. Bytes int64 + // SourceBytes is SourcePages in bytes: the base-image content this sandbox + // faulted in from the shared template source. These are the pages a future + // same-template page-sharing scheme could serve from one shared physical + // copy instead of a private per-sandbox copy; private (written) pages are + // excluded because they cannot be shared. + SourceBytes int64 // WPFaults is the number of synchronous write-protect faults resolved // (guest writes to protected pages). Zero under WP_ASYNC guests. Sampled // per interval it approximates the dirty-set growth at 2 MiB granularity. @@ -415,6 +421,7 @@ func (s ServeSnapshot) Sub(earlier ServeSnapshot) ServeSnapshot { Pages: s.Pages - earlier.Pages, SourcePages: s.SourcePages - earlier.SourcePages, Bytes: s.Bytes - earlier.Bytes, + SourceBytes: s.SourceBytes - earlier.SourceBytes, WPFaults: s.WPFaults - earlier.WPFaults, Deferred: s.Deferred - earlier.Deferred, WPDeferred: s.WPDeferred - earlier.WPDeferred, @@ -430,6 +437,7 @@ func (u *Userfaultfd) ServeStats() ServeSnapshot { Pages: u.servedPages.Load(), SourcePages: u.servedSourcePages.Load(), Bytes: u.servedBytes.Load(), + SourceBytes: u.servedSourcePages.Load() * int64(u.pageSize), WPFaults: u.wpFaultsResolved.Load(), Deferred: u.servedDeferred.Load(), Deduped: u.servedDeduped.Load(), diff --git a/packages/shared/pkg/featureflags/flags.go b/packages/shared/pkg/featureflags/flags.go index 592bd2e195..c3a8893767 100644 --- a/packages/shared/pkg/featureflags/flags.go +++ b/packages/shared/pkg/featureflags/flags.go @@ -186,9 +186,13 @@ func OverrideJSONFlag(flag JSONFlag, value ldvalue.Value) { } var ( - SnapshotFeatureFlag = NewBoolFlag("use-nfs-for-snapshots", env.IsDevelopment()) - TemplateFeatureFlag = NewBoolFlag("use-nfs-for-templates", env.IsDevelopment()) - EnableWriteThroughCacheFlag = NewBoolFlag("write-to-cache-on-writes", false) + SnapshotFeatureFlag = NewBoolFlag("use-nfs-for-snapshots", env.IsDevelopment()) + TemplateFeatureFlag = NewBoolFlag("use-nfs-for-templates", env.IsDevelopment()) + EnableWriteThroughCacheFlag = NewBoolFlag("write-to-cache-on-writes", false) + // SandboxDensityMetricsFlag exports per-sandbox host-pool working-set and + // reclaimable-estimate gauges (density observability). Off by default: no new + // metric series are produced until it is enabled. + SandboxDensityMetricsFlag = NewBoolFlag("sandbox-density-metrics", false) UseNFSCacheForBuildingTemplatesFlag = NewBoolFlag("use-nfs-for-building-templates", env.IsDevelopment()) CreateStorageCacheSpansFlag = NewBoolFlag("create-storage-cache-spans", env.IsDevelopment()) OrchAcceptsCombinedHostFlag = NewBoolFlag("orch-accepts-combined-host", false) diff --git a/packages/shared/pkg/telemetry/meters.go b/packages/shared/pkg/telemetry/meters.go index a9b6cf0497..231357aecc 100644 --- a/packages/shared/pkg/telemetry/meters.go +++ b/packages/shared/pkg/telemetry/meters.go @@ -753,7 +753,13 @@ const ( SandboxRamUsedGaugeName GaugeIntType = "e2b.sandbox.ram.used" SandboxRamTotalGaugeName GaugeIntType = "e2b.sandbox.ram.total" SandboxRamCacheGaugeName GaugeIntType = "e2b.sandbox.ram.cache" - SandboxCpuTotalGaugeName GaugeIntType = "e2b.sandbox.cpu.total" + // Host-side pool footprint and reclaim estimate (density observability). + SandboxMemWorkingSetGaugeName GaugeIntType = "orchestrator.sandbox.mem.working_set" + SandboxMemReclaimableGaugeName GaugeIntType = "orchestrator.sandbox.mem.reclaimable_estimate" + // Per-template lower bound on pool memory a same-template page-sharing scheme + // could reclaim on this node (density observability). + SandboxMemShareableBaseGaugeName GaugeIntType = "orchestrator.sandbox.mem.shareable_base_estimate" + SandboxCpuTotalGaugeName GaugeIntType = "e2b.sandbox.cpu.total" // CPU hotplug, reported by envd: how many CPUs the guest can online, how many the orchestrator // asked for, how many tries that took, and how long a sysfs online/offline write has been stuck. SandboxCpuPossibleGaugeName GaugeIntType = "e2b.sandbox.cpu.possible" @@ -1045,6 +1051,9 @@ var gaugeIntDesc = map[GaugeIntType]string{ SandboxRamUsedGaugeName: "Amount of RAM used by the sandbox.", SandboxRamTotalGaugeName: "Amount of RAM available to the sandbox.", SandboxRamCacheGaugeName: "Amount of RAM used by the page cache in the sandbox.", + SandboxMemWorkingSetGaugeName: "Host memory faulted into the hugepage pool for this sandbox since resume (its pool footprint, an upper bound on current residency).", + SandboxMemReclaimableGaugeName: "Estimate of pool memory the node holds for this sandbox that the guest is not actively using; 0 when the sandbox's balloon returns freed pages itself (free-page-reporting).", + SandboxMemShareableBaseGaugeName: "Per build_id lower bound on pool memory a same-template page-sharing scheme could reclaim on this node: (sandboxes-1) x the smallest base-image footprint among them.", SandboxCpuTotalGaugeName: "Amount of CPU available to the sandbox.", SandboxCpuPossibleGaugeName: "Number of CPUs the sandbox guest can bring online.", SandboxCpuTargetGaugeName: "Number of online CPUs last requested from the sandbox guest, 0 when none was.", @@ -1086,6 +1095,9 @@ var gaugeIntUnits = map[GaugeIntType]string{ SandboxRamUsedGaugeName: "{By}", SandboxRamTotalGaugeName: "{By}", SandboxRamCacheGaugeName: "{By}", + SandboxMemWorkingSetGaugeName: "{By}", + SandboxMemReclaimableGaugeName: "{By}", + SandboxMemShareableBaseGaugeName: "{By}", SandboxCpuTotalGaugeName: "{count}", SandboxCpuPossibleGaugeName: "{count}", SandboxCpuTargetGaugeName: "{count}",