feat(orchestrator): pool-pressure-adaptive free-page hinting interval

packages/orchestrator/pkg/sandbox/fph_periodic.go

@@ -353,7 +353,7 @@ func (s *Sandbox) hintTicks(ctx context.Context, cfg featureflags.PeriodicHintin
 			return true
 		case <-timer.C:
 		}
-		timer.Reset(cfg.Interval)
+		timer.Reset(effectiveHintInterval(cfg, nodeHugePagePressurePct()))
 
 		if cfg = cfgFn(); !cfg.Enabled() {
 			return false

packages/orchestrator/pkg/sandbox/fph_pressure.go

@@ -0,0 +1,133 @@
+//go:build linux
+
+package sandbox
+
+import (
+	"bufio"
+	"os"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/e2b-dev/infra/packages/shared/pkg/featureflags"
+)
+
+// effectiveHintInterval is the interval the periodic free-page-hinting loop
+// waits before its next tick. It is the configured Interval unless the node's
+// hugepage pool is under pressure AND an adaptation interval is configured, in
+// which case the shorter of the two is used so just-freed guest pages return to
+// the pool sooner (shrinking the reclaim-lag window). Pure for testing.
+//
+// poolUsagePct is the node hugepage pool utilization (0-100); a negative value
+// means "unknown" (meminfo unreadable or no pool), which never triggers the
+// adaptation — the fixed Interval is used, failing safe toward current
+// behaviour.
+func effectiveHintInterval(cfg featureflags.PeriodicHintingConfig, poolUsagePct int) time.Duration {
+	if cfg.PressureInterval <= 0 || cfg.PressureThresholdPct <= 0 {
+		return cfg.Interval
+	}
+	if poolUsagePct < cfg.PressureThresholdPct {
+		return cfg.Interval
+	}
+	// Under pressure: hint on the shorter cadence. Guard against a misconfigured
+	// PressureInterval that is somehow longer than Interval — never slow down
+	// under pressure.
+	if cfg.PressureInterval < cfg.Interval {
+		return cfg.PressureInterval
+	}
+
+	return cfg.Interval
+}
+
+// hugePagePressure caches the node hugepage pool utilization so the per-sandbox
+// hinting loops do not each read /proc/meminfo every tick. One read per TTL
+// serves every sandbox on the node.
+type hugePagePressure struct {
+	mu       sync.Mutex
+	at       time.Time
+	pct      int
+	readFile func() (total, free uint64, ok bool)
+}
+
+const hugePagePressureTTL = time.Second
+
+var nodeHugePagePressure = &hugePagePressure{readFile: readMeminfoHugePages}
+
+// nodeHugePagePressurePct returns the node hugepage pool utilization in percent
+// (0-100), or -1 when it cannot be determined (no pool, or meminfo unreadable).
+func nodeHugePagePressurePct() int { return nodeHugePagePressure.usagePct(time.Now()) }
+
+func (h *hugePagePressure) usagePct(now time.Time) int {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+
+	if !h.at.IsZero() && now.Sub(h.at) < hugePagePressureTTL {
+		return h.pct
+	}
+
+	h.at = now
+	total, free, ok := h.readFile()
+	h.pct = hugePageUsagePct(total, free, ok)
+
+	return h.pct
+}
+
+// hugePageUsagePct turns a (total, free) pool reading into a 0-100 utilization,
+// or -1 when unknown. Pure for testing.
+func hugePageUsagePct(total, free uint64, ok bool) int {
+	if !ok || total == 0 || free > total {
+		return -1
+	}
+
+	used := total - free
+
+	return int(used * 100 / total)
+}
+
+// readMeminfoHugePages reads HugePages_Total and HugePages_Free from
+// /proc/meminfo. ok is false when either line is missing or the file cannot be
+// read, so the caller treats pool pressure as unknown.
+func readMeminfoHugePages() (total, free uint64, ok bool) {
+	f, err := os.Open("/proc/meminfo")
+	if err != nil {
+		return 0, 0, false
+	}
+	defer f.Close()
+
+	var haveTotal, haveFree bool
+	sc := bufio.NewScanner(f)
+	for sc.Scan() {
+		line := sc.Text()
+		switch {
+		case strings.HasPrefix(line, "HugePages_Total:"):
+			if v, parsed := parseMeminfoCount(line); parsed {
+				total, haveTotal = v, true
+			}
+		case strings.HasPrefix(line, "HugePages_Free:"):
+			if v, parsed := parseMeminfoCount(line); parsed {
+				free, haveFree = v, true
+			}
+		}
+		if haveTotal && haveFree {
+			break
+		}
+	}
+
+	return total, free, haveTotal && haveFree
+}
+
+// parseMeminfoCount pulls the integer out of a "Key: <n>" meminfo line (the
+// HugePages_ counts are bare counts, no unit).
+func parseMeminfoCount(line string) (uint64, bool) {
+	fields := strings.Fields(line)
+	if len(fields) < 2 {
+		return 0, false
+	}
+	v, err := strconv.ParseUint(fields[1], 10, 64)
+	if err != nil {
+		return 0, false
+	}
+
+	return v, true
+}

packages/orchestrator/pkg/sandbox/fph_pressure_test.go

@@ -0,0 +1,99 @@
+//go:build linux
+
+package sandbox
+
+import (
+	"testing"
+	"time"
+
+	"github.com/e2b-dev/infra/packages/shared/pkg/featureflags"
+)
+
+func TestEffectiveHintInterval(t *testing.T) {
+	t.Parallel()
+
+	base := 30 * time.Second
+	fast := 5 * time.Second
+	cfg := func(pressure time.Duration, thr int) featureflags.PeriodicHintingConfig {
+		return featureflags.PeriodicHintingConfig{Interval: base, PressureInterval: pressure, PressureThresholdPct: thr}
+	}
+
+	tests := []struct {
+		name    string
+		cfg     featureflags.PeriodicHintingConfig
+		poolPct int
+		want    time.Duration
+	}{
+		{"adaptation disabled (no pressure interval) keeps base", cfg(0, 80), 95, base},
+		{"adaptation disabled (no threshold) keeps base", cfg(fast, 0), 95, base},
+		{"below threshold keeps base", cfg(fast, 80), 79, base},
+		{"at threshold uses fast interval", cfg(fast, 80), 80, fast},
+		{"above threshold uses fast interval", cfg(fast, 80), 99, fast},
+		{"unknown pool pressure (-1) keeps base, fails safe", cfg(fast, 80), -1, base},
+		{"pressure interval not shorter than base never slows down", cfg(60*time.Second, 80), 95, base},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			if got := effectiveHintInterval(tt.cfg, tt.poolPct); got != tt.want {
+				t.Errorf("effectiveHintInterval(%v, %d) = %v, want %v", tt.cfg, tt.poolPct, got, tt.want)
+			}
+		})
+	}
+}
+
+func TestHugePageUsagePct(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name        string
+		total, free uint64
+		ok          bool
+		want        int
+	}{
+		{"half used", 64, 32, true, 50},
+		{"full", 64, 0, true, 100},
+		{"empty", 64, 64, true, 0},
+		{"no pool (total 0) is unknown", 0, 0, true, -1},
+		{"unreadable is unknown", 64, 32, false, -1},
+		{"free>total is unknown (inconsistent)", 64, 65, true, -1},
+		{"69 of 100", 100, 31, true, 69},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			if got := hugePageUsagePct(tt.total, tt.free, tt.ok); got != tt.want {
+				t.Errorf("hugePageUsagePct(%d,%d,%v) = %d, want %d", tt.total, tt.free, tt.ok, got, tt.want)
+			}
+		})
+	}
+}
+
+// the cached reader serves a stable value within the TTL from one underlying read.
+func TestHugePagePressureCachesWithinTTL(t *testing.T) {
+	t.Parallel()
+
+	reads := 0
+	h := &hugePagePressure{readFile: func() (uint64, uint64, bool) {
+		reads++
+		return 100, uint64(100 - reads*10), true // free drops each read
+	}}
+
+	t0 := time.Now()
+	first := h.usagePct(t0)
+	again := h.usagePct(t0.Add(hugePagePressureTTL / 2))
+	if first != again {
+		t.Errorf("within TTL expected cached %d, got %d", first, again)
+	}
+	if reads != 1 {
+		t.Errorf("expected 1 underlying read within TTL, got %d", reads)
+	}
+
+	later := h.usagePct(t0.Add(hugePagePressureTTL * 2))
+	if reads != 2 {
+		t.Errorf("expected a fresh read after TTL, got %d reads", reads)
+	}
+	if later == first {
+		t.Errorf("expected a refreshed value after TTL")
+	}
+}

packages/shared/pkg/featureflags/flags.go

@@ -977,6 +977,15 @@ type PeriodicHintingConfig struct {
 	UnresponsiveRetry time.Duration
 	HostSlots         int
 	Stop              HintStopConfig
+	// PressureInterval, when > 0, is the hinting interval used while the node's
+	// hugepage pool is at or above PressureThresholdPct utilization. It is meant
+	// to be shorter than Interval: under pool pressure, free-page hinting runs
+	// more often so a guest's just-freed pages return to the pool sooner,
+	// shrinking the reclaim-lag window between a guest freeing a page and the
+	// pool getting it back. 0 disables the adaptation — the fixed Interval is
+	// always used (the historical behaviour).
+	PressureInterval     time.Duration
+	PressureThresholdPct int
 }
 
 func (c PeriodicHintingConfig) Enabled() bool { return c.Interval > 0 }
@@ -1026,14 +1035,27 @@ func GetPeriodicHintingConfig(ctx context.Context, ff *Client, contexts ...ldcon
 		return false
 	}
 	cfg := PeriodicHintingConfig{
-		Interval:          max(ms("interval", 0), 0),
-		Timeout:           ms("timeout", defaultPeriodicHintTimeout.Milliseconds()),
-		QuietAfterStart:   max(ms("quiet_after_start", 10000), 0),
-		SilentRuns:        int(max(num("silent_runs", 3), 0)),
-		UnresponsiveRetry: max(ms("unresponsive_retry", 300000), 0),
-		HostSlots:         int(max(num("host_slots", 16), 0)),
-		Stop:              hintStopConfig(top),
-		ObserveOnly:       boolean("observe_only"),
+		Interval:             max(ms("interval", 0), 0),
+		Timeout:              ms("timeout", defaultPeriodicHintTimeout.Milliseconds()),
+		QuietAfterStart:      max(ms("quiet_after_start", 10000), 0),
+		SilentRuns:           int(max(num("silent_runs", 3), 0)),
+		UnresponsiveRetry:    max(ms("unresponsive_retry", 300000), 0),
+		HostSlots:            int(max(num("host_slots", 16), 0)),
+		Stop:                 hintStopConfig(top),
+		ObserveOnly:          boolean("observe_only"),
+		PressureInterval:     max(ms("pressure_interval", 0), 0),
+		PressureThresholdPct: int(num("pressure_threshold_pct", 0)),
+	}
+	// A pressure interval below the floor (but non-zero) would hint too hot;
+	// clamp it off rather than silently flooring, same policy as Interval.
+	if cfg.PressureInterval > 0 && cfg.PressureInterval < minPeriodicHintingInterval {
+		complain("pressure_interval", "below the one-second floor; pressure adaptation off", v.GetByKey("pressure_interval"))
+		cfg.PressureInterval = 0
+	}
+	// An out-of-range threshold disables the adaptation rather than triggering
+	// on every tick (0) or never (>100).
+	if cfg.PressureThresholdPct <= 0 || cfg.PressureThresholdPct > 100 {
+		cfg.PressureInterval = 0
 	}
 	if cfg.Interval > 0 && cfg.Interval < minPeriodicHintingInterval {
 		complain("interval", "below the one-second floor; periodic hinting off", v.GetByKey("interval"))