diff --git a/packages/orchestrator/pkg/factories/run.go b/packages/orchestrator/pkg/factories/run.go index a24ffc8aa0..56fe63f149 100644 --- a/packages/orchestrator/pkg/factories/run.go +++ b/packages/orchestrator/pkg/factories/run.go @@ -813,7 +813,7 @@ func run(config cfg.Config, opts Options) (success bool) { closers = append(closers, closer{"sandbox events deliveries (all)", eventsService.Close}) // sandbox observer - sandboxObserver, err := metrics.NewSandboxObserver(ctx, nodeID, serviceName, commitSHA, version, serviceInstanceID, sandboxes) + sandboxObserver, err := metrics.NewSandboxObserver(ctx, nodeID, serviceName, commitSHA, version, serviceInstanceID, sandboxes, featureFlags) if err != nil { logger.L().Fatal(ctx, "failed to create sandbox observer", zap.Error(err)) } diff --git a/packages/orchestrator/pkg/metrics/reclaimable_test.go b/packages/orchestrator/pkg/metrics/reclaimable_test.go new file mode 100644 index 0000000000..a4303b7177 --- /dev/null +++ b/packages/orchestrator/pkg/metrics/reclaimable_test.go @@ -0,0 +1,70 @@ +//go:build linux + +package metrics + +import "testing" + +// reclaimableBytes is the estimate the density-observability gauge exports; it +// must never over-report (it feeds a later reclaim decision), so these lock its +// floor-at-zero and free-page-reporting behavior. +func TestReclaimableBytes(t *testing.T) { + t.Parallel() + + const mib = int64(1) << 20 + + tests := []struct { + name string + faulted int64 + guestInUse int64 + freePageReporting bool + want int64 + }{ + { + name: "faulted above in-use yields the gap", + faulted: 2048 * mib, + guestInUse: 512 * mib, + want: 1536 * mib, + }, + { + name: "in-use above faulted floors at zero (never negative)", + faulted: 256 * mib, + guestInUse: 512 * mib, + want: 0, + }, + { + name: "equal footprint and use yields zero", + faulted: 512 * mib, + guestInUse: 512 * mib, + want: 0, + }, + { + name: "no guest report yields the whole footprint as upper bound", + faulted: 1024 * mib, + guestInUse: 0, + want: 1024 * mib, + }, + { + name: "free-page-reporting active reports zero (pool self-reclaims)", + faulted: 2048 * mib, + guestInUse: 128 * mib, + freePageReporting: true, + want: 0, + }, + { + name: "zero footprint yields zero", + faulted: 0, guestInUse: 0, want: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + got := reclaimableBytes(tt.faulted, tt.guestInUse, tt.freePageReporting) + if got != tt.want { + t.Errorf("reclaimableBytes(%d, %d, %v) = %d, want %d", + tt.faulted, tt.guestInUse, tt.freePageReporting, got, tt.want) + } + }) + } +} diff --git a/packages/orchestrator/pkg/metrics/sandboxes.go b/packages/orchestrator/pkg/metrics/sandboxes.go index f40d84dd37..4802fae013 100644 --- a/packages/orchestrator/pkg/metrics/sandboxes.go +++ b/packages/orchestrator/pkg/metrics/sandboxes.go @@ -19,6 +19,7 @@ import ( "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox" "github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/envd" + "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" "github.com/e2b-dev/infra/packages/shared/pkg/logger" sbxlogger "github.com/e2b-dev/infra/packages/shared/pkg/logger/sandbox" "github.com/e2b-dev/infra/packages/shared/pkg/telemetry" @@ -61,21 +62,51 @@ type SandboxObserver struct { sandboxes *sandbox.Map - meter metric.Meter - cpuTotal metric.Int64ObservableGauge - cpuUsed metric.Float64ObservableGauge - cpuPossible metric.Int64ObservableGauge - cpuTarget metric.Int64ObservableGauge - cpuAttempts metric.Int64ObservableGauge - cpuPending metric.Int64ObservableGauge - memoryTotal metric.Int64ObservableGauge - memoryUsed metric.Int64ObservableGauge - memoryCache metric.Int64ObservableGauge - diskTotal metric.Int64ObservableGauge - diskUsed metric.Int64ObservableGauge + meter metric.Meter + cpuTotal metric.Int64ObservableGauge + cpuUsed metric.Float64ObservableGauge + cpuPossible metric.Int64ObservableGauge + cpuTarget metric.Int64ObservableGauge + cpuAttempts metric.Int64ObservableGauge + cpuPending metric.Int64ObservableGauge + memoryTotal metric.Int64ObservableGauge + memoryUsed metric.Int64ObservableGauge + memoryCache metric.Int64ObservableGauge + memWorkingSet metric.Int64ObservableGauge + memReclaimable metric.Int64ObservableGauge + diskTotal metric.Int64ObservableGauge + diskUsed metric.Int64ObservableGauge + + featureFlags *featureflags.Client } -func NewSandboxObserver(ctx context.Context, nodeID, serviceName, serviceCommit, serviceVersion, serviceInstanceID string, sandboxes *sandbox.Map) (*SandboxObserver, error) { +// reclaimableBytes estimates pool memory the node holds for a sandbox that the +// guest is not actively using — the opportunity a later live reclaim (balloon +// inflate + host-side punch) could return to the hugepage pool. +// +// - faultedBytes: host bytes faulted into the pool since resume +// (MemoryWorkingSetBytes); an upper bound on current pool residency. +// - guestInUseBytes: the guest's actively-used RAM (envd-reported, excluding +// reclaimable page cache). 0/absent yields the whole footprint as the +// upper-bound estimate. +// - freePageReporting: whether the sandbox's balloon returns freed pages to +// the host itself. When true the pool is largely self-reclaiming, so the +// standing reclaim opportunity is reported as 0 rather than double-counting +// pages the device already discards. +func reclaimableBytes(faultedBytes, guestInUseBytes int64, freePageReporting bool) int64 { + if freePageReporting { + return 0 + } + + r := faultedBytes - guestInUseBytes + if r < 0 { + return 0 + } + + return r +} + +func NewSandboxObserver(ctx context.Context, nodeID, serviceName, serviceCommit, serviceVersion, serviceInstanceID string, sandboxes *sandbox.Map, featureFlags *featureflags.Client) (*SandboxObserver, error) { deltaTemporality := otlpmetricgrpc.WithTemporalitySelector(func(kind sdkmetric.InstrumentKind) metricdata.Temporality { // Use delta temporality for gauges and cumulative for all other instrument kinds. // This is used to prevent reporting sandbox metrics indefinitely. @@ -163,6 +194,16 @@ func NewSandboxObserver(ctx context.Context, nodeID, serviceName, serviceCommit, return nil, fmt.Errorf("failed to create disk used gauge: %w", err) } + memWorkingSet, err := telemetry.GetGaugeInt(meter, telemetry.SandboxMemWorkingSetGaugeName) + if err != nil { + return nil, fmt.Errorf("failed to create memory working-set gauge: %w", err) + } + + memReclaimable, err := telemetry.GetGaugeInt(meter, telemetry.SandboxMemReclaimableGaugeName) + if err != nil { + return nil, fmt.Errorf("failed to create memory reclaimable gauge: %w", err) + } + so := &SandboxObserver{ exportInterval: sandboxMetricExportPeriod, meterExporter: externalMeterExporter, @@ -179,6 +220,9 @@ func NewSandboxObserver(ctx context.Context, nodeID, serviceName, serviceCommit, memoryCache: memoryCache, diskTotal: diskTotal, diskUsed: diskUsed, + memWorkingSet: memWorkingSet, + memReclaimable: memReclaimable, + featureFlags: featureFlags, } registration, err := so.startObserving() @@ -285,6 +329,22 @@ func (so *SandboxObserver) startObserving() (metric.Registration, error) { o.ObserveInt64(so.memoryUsed, memoryUsed, attributes) } + // Density observability (off by default): host-pool footprint + // and reclaim estimate. Flag-gated so no new series are produced + // until explicitly enabled. + if so.featureFlags != nil && so.featureFlags.BoolFlag(ctx, featureflags.SandboxDensityMetricsFlag) { + workingSet := sbx.MemoryWorkingSetBytes() + o.ObserveInt64(so.memWorkingSet, workingSet, attributes) + + guestInUse := int64(0) + if memoryReported { + guestInUse = memoryUsed + } + o.ObserveInt64(so.memReclaimable, + reclaimableBytes(workingSet, guestInUse, sbx.FreePageReportingActive()), + attributes) + } + ok, err := utils.IsGTEVersion(sbx.Config.Envd.Version, minEnvdVersionForCacheMetrics) if err != nil { logger.L().Error(ctx, "Failed to check envd version for cache metrics", zap.Error(err), logger.WithSandboxID(sbx.Runtime.SandboxID)) @@ -343,7 +403,8 @@ func (so *SandboxObserver) startObserving() (metric.Registration, error) { return nil }, so.cpuTotal, so.cpuUsed, so.cpuPossible, so.cpuTarget, so.cpuAttempts, so.cpuPending, - so.memoryTotal, so.memoryUsed, so.memoryCache, so.diskTotal, so.diskUsed) + so.memoryTotal, so.memoryUsed, so.memoryCache, so.diskTotal, so.diskUsed, + so.memWorkingSet, so.memReclaimable) if err != nil { return nil, err } diff --git a/packages/orchestrator/pkg/metrics/sandboxes_test.go b/packages/orchestrator/pkg/metrics/sandboxes_test.go index d55c065e26..2011234198 100644 --- a/packages/orchestrator/pkg/metrics/sandboxes_test.go +++ b/packages/orchestrator/pkg/metrics/sandboxes_test.go @@ -110,6 +110,8 @@ func newTestSandboxObserver(t *testing.T, meter metric.Meter, sandboxes *sandbox {&so.memoryCache, telemetry.SandboxRamCacheGaugeName}, {&so.diskTotal, telemetry.SandboxDiskTotalGaugeName}, {&so.diskUsed, telemetry.SandboxDiskUsedGaugeName}, + {&so.memWorkingSet, telemetry.SandboxMemWorkingSetGaugeName}, + {&so.memReclaimable, telemetry.SandboxMemReclaimableGaugeName}, } { gauge, err := telemetry.GetGaugeInt(meter, g.name) require.NoError(t, err) diff --git a/packages/orchestrator/pkg/sandbox/sandbox.go b/packages/orchestrator/pkg/sandbox/sandbox.go index 3837dbc6ea..a58a174d60 100644 --- a/packages/orchestrator/pkg/sandbox/sandbox.go +++ b/packages/orchestrator/pkg/sandbox/sandbox.go @@ -2176,6 +2176,25 @@ func (s *Sandbox) FirecrackerExit() *fc.ExitInfo { // MemoryHandlerErr reports why the memory handler exited, and nil if it has // not exited or exited cleanly. Only meaningful once the execution has ended. +// MemoryWorkingSetBytes is the host memory faulted into the hugepage pool for +// this sandbox since resume — its pool footprint and an upper bound on current +// residency (UFFD serves are cumulative and hugetlbfs pages are not returned +// without explicit reclaim). 0 when no memory backend is attached. +func (s *Sandbox) MemoryWorkingSetBytes() int64 { + if s.Resources == nil || s.memory == nil { + return 0 + } + + return s.memory.ServeStats().Bytes +} + +// FreePageReportingActive reports whether this sandbox's balloon returns freed +// guest pages to the host on its own, so the pool is largely self-reclaiming. +// Read from the cached balloon mode; no device round-trip. +func (s *Sandbox) FreePageReportingActive() bool { + return s.BalloonModeValue() == userfaultfd.BalloonModeReporting +} + func (s *Sandbox) MemoryHandlerErr() error { if s.Resources == nil || s.memory == nil { return nil diff --git a/packages/shared/pkg/featureflags/flags.go b/packages/shared/pkg/featureflags/flags.go index 592bd2e195..c3a8893767 100644 --- a/packages/shared/pkg/featureflags/flags.go +++ b/packages/shared/pkg/featureflags/flags.go @@ -186,9 +186,13 @@ func OverrideJSONFlag(flag JSONFlag, value ldvalue.Value) { } var ( - SnapshotFeatureFlag = NewBoolFlag("use-nfs-for-snapshots", env.IsDevelopment()) - TemplateFeatureFlag = NewBoolFlag("use-nfs-for-templates", env.IsDevelopment()) - EnableWriteThroughCacheFlag = NewBoolFlag("write-to-cache-on-writes", false) + SnapshotFeatureFlag = NewBoolFlag("use-nfs-for-snapshots", env.IsDevelopment()) + TemplateFeatureFlag = NewBoolFlag("use-nfs-for-templates", env.IsDevelopment()) + EnableWriteThroughCacheFlag = NewBoolFlag("write-to-cache-on-writes", false) + // SandboxDensityMetricsFlag exports per-sandbox host-pool working-set and + // reclaimable-estimate gauges (density observability). Off by default: no new + // metric series are produced until it is enabled. + SandboxDensityMetricsFlag = NewBoolFlag("sandbox-density-metrics", false) UseNFSCacheForBuildingTemplatesFlag = NewBoolFlag("use-nfs-for-building-templates", env.IsDevelopment()) CreateStorageCacheSpansFlag = NewBoolFlag("create-storage-cache-spans", env.IsDevelopment()) OrchAcceptsCombinedHostFlag = NewBoolFlag("orch-accepts-combined-host", false) diff --git a/packages/shared/pkg/telemetry/meters.go b/packages/shared/pkg/telemetry/meters.go index a9b6cf0497..90fb5c125a 100644 --- a/packages/shared/pkg/telemetry/meters.go +++ b/packages/shared/pkg/telemetry/meters.go @@ -753,7 +753,10 @@ const ( SandboxRamUsedGaugeName GaugeIntType = "e2b.sandbox.ram.used" SandboxRamTotalGaugeName GaugeIntType = "e2b.sandbox.ram.total" SandboxRamCacheGaugeName GaugeIntType = "e2b.sandbox.ram.cache" - SandboxCpuTotalGaugeName GaugeIntType = "e2b.sandbox.cpu.total" + // Host-side pool footprint and reclaim estimate (density observability). + SandboxMemWorkingSetGaugeName GaugeIntType = "orchestrator.sandbox.mem.working_set" + SandboxMemReclaimableGaugeName GaugeIntType = "orchestrator.sandbox.mem.reclaimable_estimate" + SandboxCpuTotalGaugeName GaugeIntType = "e2b.sandbox.cpu.total" // CPU hotplug, reported by envd: how many CPUs the guest can online, how many the orchestrator // asked for, how many tries that took, and how long a sysfs online/offline write has been stuck. SandboxCpuPossibleGaugeName GaugeIntType = "e2b.sandbox.cpu.possible" @@ -1045,6 +1048,8 @@ var gaugeIntDesc = map[GaugeIntType]string{ SandboxRamUsedGaugeName: "Amount of RAM used by the sandbox.", SandboxRamTotalGaugeName: "Amount of RAM available to the sandbox.", SandboxRamCacheGaugeName: "Amount of RAM used by the page cache in the sandbox.", + SandboxMemWorkingSetGaugeName: "Host memory faulted into the hugepage pool for this sandbox since resume (its pool footprint, an upper bound on current residency).", + SandboxMemReclaimableGaugeName: "Estimate of pool memory the node holds for this sandbox that the guest is not actively using; 0 when the sandbox's balloon returns freed pages itself (free-page-reporting).", SandboxCpuTotalGaugeName: "Amount of CPU available to the sandbox.", SandboxCpuPossibleGaugeName: "Number of CPUs the sandbox guest can bring online.", SandboxCpuTargetGaugeName: "Number of online CPUs last requested from the sandbox guest, 0 when none was.", @@ -1086,6 +1091,8 @@ var gaugeIntUnits = map[GaugeIntType]string{ SandboxRamUsedGaugeName: "{By}", SandboxRamTotalGaugeName: "{By}", SandboxRamCacheGaugeName: "{By}", + SandboxMemWorkingSetGaugeName: "{By}", + SandboxMemReclaimableGaugeName: "{By}", SandboxCpuTotalGaugeName: "{count}", SandboxCpuPossibleGaugeName: "{count}", SandboxCpuTargetGaugeName: "{count}",