feat(orchestrator): per-sandbox host-pool working set and reclaim estimate (density observability, off by default)

packages/orchestrator/pkg/factories/run.go

@@ -813,7 +813,7 @@ func run(config cfg.Config, opts Options) (success bool) {
 	closers = append(closers, closer{"sandbox events deliveries (all)", eventsService.Close})
 
 	// sandbox observer
-	sandboxObserver, err := metrics.NewSandboxObserver(ctx, nodeID, serviceName, commitSHA, version, serviceInstanceID, sandboxes)
+	sandboxObserver, err := metrics.NewSandboxObserver(ctx, nodeID, serviceName, commitSHA, version, serviceInstanceID, sandboxes, featureFlags)
 	if err != nil {
 		logger.L().Fatal(ctx, "failed to create sandbox observer", zap.Error(err))
 	}

packages/orchestrator/pkg/metrics/reclaimable_test.go

@@ -0,0 +1,70 @@
+//go:build linux
+
+package metrics
+
+import "testing"
+
+// reclaimableBytes is the estimate the density-observability gauge exports; it
+// must never over-report (it feeds a later reclaim decision), so these lock its
+// floor-at-zero and free-page-reporting behavior.
+func TestReclaimableBytes(t *testing.T) {
+	t.Parallel()
+
+	const mib = int64(1) << 20
+
+	tests := []struct {
+		name              string
+		faulted           int64
+		guestInUse        int64
+		freePageReporting bool
+		want              int64
+	}{
+		{
+			name:       "faulted above in-use yields the gap",
+			faulted:    2048 * mib,
+			guestInUse: 512 * mib,
+			want:       1536 * mib,
+		},
+		{
+			name:       "in-use above faulted floors at zero (never negative)",
+			faulted:    256 * mib,
+			guestInUse: 512 * mib,
+			want:       0,
+		},
+		{
+			name:       "equal footprint and use yields zero",
+			faulted:    512 * mib,
+			guestInUse: 512 * mib,
+			want:       0,
+		},
+		{
+			name:       "no guest report yields the whole footprint as upper bound",
+			faulted:    1024 * mib,
+			guestInUse: 0,
+			want:       1024 * mib,
+		},
+		{
+			name:              "free-page-reporting active reports zero (pool self-reclaims)",
+			faulted:           2048 * mib,
+			guestInUse:        128 * mib,
+			freePageReporting: true,
+			want:              0,
+		},
+		{
+			name:    "zero footprint yields zero",
+			faulted: 0, guestInUse: 0, want: 0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+
+			got := reclaimableBytes(tt.faulted, tt.guestInUse, tt.freePageReporting)
+			if got != tt.want {
+				t.Errorf("reclaimableBytes(%d, %d, %v) = %d, want %d",
+					tt.faulted, tt.guestInUse, tt.freePageReporting, got, tt.want)
+			}
+		})
+	}
+}

packages/orchestrator/pkg/metrics/sandboxes.go

@@ -19,6 +19,7 @@ import (
 
 	"github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox"
 	"github.com/e2b-dev/infra/packages/orchestrator/pkg/sandbox/envd"
+	"github.com/e2b-dev/infra/packages/shared/pkg/featureflags"
 	"github.com/e2b-dev/infra/packages/shared/pkg/logger"
 	sbxlogger "github.com/e2b-dev/infra/packages/shared/pkg/logger/sandbox"
 	"github.com/e2b-dev/infra/packages/shared/pkg/telemetry"
@@ -61,21 +62,51 @@ type SandboxObserver struct {
 
 	sandboxes *sandbox.Map
 
-	meter       metric.Meter
-	cpuTotal    metric.Int64ObservableGauge
-	cpuUsed     metric.Float64ObservableGauge
-	cpuPossible metric.Int64ObservableGauge
-	cpuTarget   metric.Int64ObservableGauge
-	cpuAttempts metric.Int64ObservableGauge
-	cpuPending  metric.Int64ObservableGauge
-	memoryTotal metric.Int64ObservableGauge
-	memoryUsed  metric.Int64ObservableGauge
-	memoryCache metric.Int64ObservableGauge
-	diskTotal   metric.Int64ObservableGauge
-	diskUsed    metric.Int64ObservableGauge
+	meter          metric.Meter
+	cpuTotal       metric.Int64ObservableGauge
+	cpuUsed        metric.Float64ObservableGauge
+	cpuPossible    metric.Int64ObservableGauge
+	cpuTarget      metric.Int64ObservableGauge
+	cpuAttempts    metric.Int64ObservableGauge
+	cpuPending     metric.Int64ObservableGauge
+	memoryTotal    metric.Int64ObservableGauge
+	memoryUsed     metric.Int64ObservableGauge
+	memoryCache    metric.Int64ObservableGauge
+	memWorkingSet  metric.Int64ObservableGauge
+	memReclaimable metric.Int64ObservableGauge
+	diskTotal      metric.Int64ObservableGauge
+	diskUsed       metric.Int64ObservableGauge
+
+	featureFlags *featureflags.Client
 }
 
-func NewSandboxObserver(ctx context.Context, nodeID, serviceName, serviceCommit, serviceVersion, serviceInstanceID string, sandboxes *sandbox.Map) (*SandboxObserver, error) {
+// reclaimableBytes estimates pool memory the node holds for a sandbox that the
+// guest is not actively using — the opportunity a later live reclaim (balloon
+// inflate + host-side punch) could return to the hugepage pool.
+//
+//   - faultedBytes:    host bytes faulted into the pool since resume
+//     (MemoryWorkingSetBytes); an upper bound on current pool residency.
+//   - guestInUseBytes:  the guest's actively-used RAM (envd-reported, excluding
+//     reclaimable page cache). 0/absent yields the whole footprint as the
+//     upper-bound estimate.
+//   - freePageReporting: whether the sandbox's balloon returns freed pages to
+//     the host itself. When true the pool is largely self-reclaiming, so the
+//     standing reclaim opportunity is reported as 0 rather than double-counting
+//     pages the device already discards.
+func reclaimableBytes(faultedBytes, guestInUseBytes int64, freePageReporting bool) int64 {
+	if freePageReporting {
+		return 0
+	}
+
+	r := faultedBytes - guestInUseBytes
+	if r < 0 {
+		return 0
+	}
+
+	return r
+}
+
+func NewSandboxObserver(ctx context.Context, nodeID, serviceName, serviceCommit, serviceVersion, serviceInstanceID string, sandboxes *sandbox.Map, featureFlags *featureflags.Client) (*SandboxObserver, error) {
 	deltaTemporality := otlpmetricgrpc.WithTemporalitySelector(func(kind sdkmetric.InstrumentKind) metricdata.Temporality {
 		// Use delta temporality for gauges and cumulative for all other instrument kinds.
 		// This is used to prevent reporting sandbox metrics indefinitely.
@@ -163,6 +194,16 @@ func NewSandboxObserver(ctx context.Context, nodeID, serviceName, serviceCommit,
 		return nil, fmt.Errorf("failed to create disk used gauge: %w", err)
 	}
 
+	memWorkingSet, err := telemetry.GetGaugeInt(meter, telemetry.SandboxMemWorkingSetGaugeName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create memory working-set gauge: %w", err)
+	}
+
+	memReclaimable, err := telemetry.GetGaugeInt(meter, telemetry.SandboxMemReclaimableGaugeName)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create memory reclaimable gauge: %w", err)
+	}
+
 	so := &SandboxObserver{
 		exportInterval: sandboxMetricExportPeriod,
 		meterExporter:  externalMeterExporter,
@@ -179,6 +220,9 @@ func NewSandboxObserver(ctx context.Context, nodeID, serviceName, serviceCommit,
 		memoryCache:    memoryCache,
 		diskTotal:      diskTotal,
 		diskUsed:       diskUsed,
+		memWorkingSet:  memWorkingSet,
+		memReclaimable: memReclaimable,
+		featureFlags:   featureFlags,
 	}
 
 	registration, err := so.startObserving()
@@ -285,6 +329,22 @@ func (so *SandboxObserver) startObserving() (metric.Registration, error) {
 						o.ObserveInt64(so.memoryUsed, memoryUsed, attributes)
 					}
 
+					// Density observability (off by default): host-pool footprint
+					// and reclaim estimate. Flag-gated so no new series are produced
+					// until explicitly enabled.
+					if so.featureFlags != nil && so.featureFlags.BoolFlag(ctx, featureflags.SandboxDensityMetricsFlag) {
+						workingSet := sbx.MemoryWorkingSetBytes()
+						o.ObserveInt64(so.memWorkingSet, workingSet, attributes)
+
+						guestInUse := int64(0)
+						if memoryReported {
+							guestInUse = memoryUsed
+						}
+						o.ObserveInt64(so.memReclaimable,
+							reclaimableBytes(workingSet, guestInUse, sbx.FreePageReportingActive()),
+							attributes)
+					}
+
 					ok, err := utils.IsGTEVersion(sbx.Config.Envd.Version, minEnvdVersionForCacheMetrics)
 					if err != nil {
 						logger.L().Error(ctx, "Failed to check envd version for cache metrics", zap.Error(err), logger.WithSandboxID(sbx.Runtime.SandboxID))
@@ -343,7 +403,8 @@ func (so *SandboxObserver) startObserving() (metric.Registration, error) {
 
 			return nil
 		}, so.cpuTotal, so.cpuUsed, so.cpuPossible, so.cpuTarget, so.cpuAttempts, so.cpuPending,
-		so.memoryTotal, so.memoryUsed, so.memoryCache, so.diskTotal, so.diskUsed)
+		so.memoryTotal, so.memoryUsed, so.memoryCache, so.diskTotal, so.diskUsed,
+		so.memWorkingSet, so.memReclaimable)
 	if err != nil {
 		return nil, err
 	}

packages/orchestrator/pkg/metrics/sandboxes_test.go

@@ -110,6 +110,8 @@ func newTestSandboxObserver(t *testing.T, meter metric.Meter, sandboxes *sandbox
 		{&so.memoryCache, telemetry.SandboxRamCacheGaugeName},
 		{&so.diskTotal, telemetry.SandboxDiskTotalGaugeName},
 		{&so.diskUsed, telemetry.SandboxDiskUsedGaugeName},
+		{&so.memWorkingSet, telemetry.SandboxMemWorkingSetGaugeName},
+		{&so.memReclaimable, telemetry.SandboxMemReclaimableGaugeName},
 	} {
 		gauge, err := telemetry.GetGaugeInt(meter, g.name)
 		require.NoError(t, err)

packages/orchestrator/pkg/sandbox/sandbox.go

@@ -2176,6 +2176,25 @@ func (s *Sandbox) FirecrackerExit() *fc.ExitInfo {
 
 // MemoryHandlerErr reports why the memory handler exited, and nil if it has
 // not exited or exited cleanly. Only meaningful once the execution has ended.
+// MemoryWorkingSetBytes is the host memory faulted into the hugepage pool for
+// this sandbox since resume — its pool footprint and an upper bound on current
+// residency (UFFD serves are cumulative and hugetlbfs pages are not returned
+// without explicit reclaim). 0 when no memory backend is attached.
+func (s *Sandbox) MemoryWorkingSetBytes() int64 {
+	if s.Resources == nil || s.memory == nil {
+		return 0
+	}
+
+	return s.memory.ServeStats().Bytes
+}
+
+// FreePageReportingActive reports whether this sandbox's balloon returns freed
+// guest pages to the host on its own, so the pool is largely self-reclaiming.
+// Read from the cached balloon mode; no device round-trip.
+func (s *Sandbox) FreePageReportingActive() bool {
+	return s.BalloonModeValue() == userfaultfd.BalloonModeReporting
+}
+
 func (s *Sandbox) MemoryHandlerErr() error {
 	if s.Resources == nil || s.memory == nil {
 		return nil

packages/shared/pkg/featureflags/flags.go

@@ -186,9 +186,13 @@ func OverrideJSONFlag(flag JSONFlag, value ldvalue.Value) {
 }
 
 var (
-	SnapshotFeatureFlag                 = NewBoolFlag("use-nfs-for-snapshots", env.IsDevelopment())
-	TemplateFeatureFlag                 = NewBoolFlag("use-nfs-for-templates", env.IsDevelopment())
-	EnableWriteThroughCacheFlag         = NewBoolFlag("write-to-cache-on-writes", false)
+	SnapshotFeatureFlag         = NewBoolFlag("use-nfs-for-snapshots", env.IsDevelopment())
+	TemplateFeatureFlag         = NewBoolFlag("use-nfs-for-templates", env.IsDevelopment())
+	EnableWriteThroughCacheFlag = NewBoolFlag("write-to-cache-on-writes", false)
+	// SandboxDensityMetricsFlag exports per-sandbox host-pool working-set and
+	// reclaimable-estimate gauges (density observability). Off by default: no new
+	// metric series are produced until it is enabled.
+	SandboxDensityMetricsFlag           = NewBoolFlag("sandbox-density-metrics", false)
 	UseNFSCacheForBuildingTemplatesFlag = NewBoolFlag("use-nfs-for-building-templates", env.IsDevelopment())
 	CreateStorageCacheSpansFlag         = NewBoolFlag("create-storage-cache-spans", env.IsDevelopment())
 	OrchAcceptsCombinedHostFlag         = NewBoolFlag("orch-accepts-combined-host", false)

packages/shared/pkg/telemetry/meters.go

@@ -753,7 +753,10 @@ const (
 	SandboxRamUsedGaugeName  GaugeIntType = "e2b.sandbox.ram.used"
 	SandboxRamTotalGaugeName GaugeIntType = "e2b.sandbox.ram.total"
 	SandboxRamCacheGaugeName GaugeIntType = "e2b.sandbox.ram.cache"
-	SandboxCpuTotalGaugeName GaugeIntType = "e2b.sandbox.cpu.total"
+	// Host-side pool footprint and reclaim estimate (density observability).
+	SandboxMemWorkingSetGaugeName  GaugeIntType = "orchestrator.sandbox.mem.working_set"
+	SandboxMemReclaimableGaugeName GaugeIntType = "orchestrator.sandbox.mem.reclaimable_estimate"
+	SandboxCpuTotalGaugeName       GaugeIntType = "e2b.sandbox.cpu.total"
 	// CPU hotplug, reported by envd: how many CPUs the guest can online, how many the orchestrator
 	// asked for, how many tries that took, and how long a sysfs online/offline write has been stuck.
 	SandboxCpuPossibleGaugeName       GaugeIntType = "e2b.sandbox.cpu.possible"
@@ -1045,6 +1048,8 @@ var gaugeIntDesc = map[GaugeIntType]string{
 	SandboxRamUsedGaugeName:              "Amount of RAM used by the sandbox.",
 	SandboxRamTotalGaugeName:             "Amount of RAM available to the sandbox.",
 	SandboxRamCacheGaugeName:             "Amount of RAM used by the page cache in the sandbox.",
+	SandboxMemWorkingSetGaugeName:        "Host memory faulted into the hugepage pool for this sandbox since resume (its pool footprint, an upper bound on current residency).",
+	SandboxMemReclaimableGaugeName:       "Estimate of pool memory the node holds for this sandbox that the guest is not actively using; 0 when the sandbox's balloon returns freed pages itself (free-page-reporting).",
 	SandboxCpuTotalGaugeName:             "Amount of CPU available to the sandbox.",
 	SandboxCpuPossibleGaugeName:          "Number of CPUs the sandbox guest can bring online.",
 	SandboxCpuTargetGaugeName:            "Number of online CPUs last requested from the sandbox guest, 0 when none was.",
@@ -1086,6 +1091,8 @@ var gaugeIntUnits = map[GaugeIntType]string{
 	SandboxRamUsedGaugeName:              "{By}",
 	SandboxRamTotalGaugeName:             "{By}",
 	SandboxRamCacheGaugeName:             "{By}",
+	SandboxMemWorkingSetGaugeName:        "{By}",
+	SandboxMemReclaimableGaugeName:       "{By}",
 	SandboxCpuTotalGaugeName:             "{count}",
 	SandboxCpuPossibleGaugeName:          "{count}",
 	SandboxCpuTargetGaugeName:            "{count}",