diff --git a/packages/orchestrator/pkg/sandbox/fph_periodic.go b/packages/orchestrator/pkg/sandbox/fph_periodic.go index 93af3dae48..64d246f0f9 100644 --- a/packages/orchestrator/pkg/sandbox/fph_periodic.go +++ b/packages/orchestrator/pkg/sandbox/fph_periodic.go @@ -353,7 +353,7 @@ func (s *Sandbox) hintTicks(ctx context.Context, cfg featureflags.PeriodicHintin return true case <-timer.C: } - timer.Reset(cfg.Interval) + timer.Reset(effectiveHintInterval(cfg, nodeHugePagePressurePct())) if cfg = cfgFn(); !cfg.Enabled() { return false diff --git a/packages/orchestrator/pkg/sandbox/fph_pressure.go b/packages/orchestrator/pkg/sandbox/fph_pressure.go new file mode 100644 index 0000000000..4a26f094da --- /dev/null +++ b/packages/orchestrator/pkg/sandbox/fph_pressure.go @@ -0,0 +1,133 @@ +//go:build linux + +package sandbox + +import ( + "bufio" + "os" + "strconv" + "strings" + "sync" + "time" + + "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" +) + +// effectiveHintInterval is the interval the periodic free-page-hinting loop +// waits before its next tick. It is the configured Interval unless the node's +// hugepage pool is under pressure AND an adaptation interval is configured, in +// which case the shorter of the two is used so just-freed guest pages return to +// the pool sooner (shrinking the reclaim-lag window). Pure for testing. +// +// poolUsagePct is the node hugepage pool utilization (0-100); a negative value +// means "unknown" (meminfo unreadable or no pool), which never triggers the +// adaptation — the fixed Interval is used, failing safe toward current +// behaviour. +func effectiveHintInterval(cfg featureflags.PeriodicHintingConfig, poolUsagePct int) time.Duration { + if cfg.PressureInterval <= 0 || cfg.PressureThresholdPct <= 0 { + return cfg.Interval + } + if poolUsagePct < cfg.PressureThresholdPct { + return cfg.Interval + } + // Under pressure: hint on the shorter cadence. Guard against a misconfigured + // PressureInterval that is somehow longer than Interval — never slow down + // under pressure. + if cfg.PressureInterval < cfg.Interval { + return cfg.PressureInterval + } + + return cfg.Interval +} + +// hugePagePressure caches the node hugepage pool utilization so the per-sandbox +// hinting loops do not each read /proc/meminfo every tick. One read per TTL +// serves every sandbox on the node. +type hugePagePressure struct { + mu sync.Mutex + at time.Time + pct int + readFile func() (total, free uint64, ok bool) +} + +const hugePagePressureTTL = time.Second + +var nodeHugePagePressure = &hugePagePressure{readFile: readMeminfoHugePages} + +// nodeHugePagePressurePct returns the node hugepage pool utilization in percent +// (0-100), or -1 when it cannot be determined (no pool, or meminfo unreadable). +func nodeHugePagePressurePct() int { return nodeHugePagePressure.usagePct(time.Now()) } + +func (h *hugePagePressure) usagePct(now time.Time) int { + h.mu.Lock() + defer h.mu.Unlock() + + if !h.at.IsZero() && now.Sub(h.at) < hugePagePressureTTL { + return h.pct + } + + h.at = now + total, free, ok := h.readFile() + h.pct = hugePageUsagePct(total, free, ok) + + return h.pct +} + +// hugePageUsagePct turns a (total, free) pool reading into a 0-100 utilization, +// or -1 when unknown. Pure for testing. +func hugePageUsagePct(total, free uint64, ok bool) int { + if !ok || total == 0 || free > total { + return -1 + } + + used := total - free + + return int(used * 100 / total) +} + +// readMeminfoHugePages reads HugePages_Total and HugePages_Free from +// /proc/meminfo. ok is false when either line is missing or the file cannot be +// read, so the caller treats pool pressure as unknown. +func readMeminfoHugePages() (total, free uint64, ok bool) { + f, err := os.Open("/proc/meminfo") + if err != nil { + return 0, 0, false + } + defer f.Close() + + var haveTotal, haveFree bool + sc := bufio.NewScanner(f) + for sc.Scan() { + line := sc.Text() + switch { + case strings.HasPrefix(line, "HugePages_Total:"): + if v, parsed := parseMeminfoCount(line); parsed { + total, haveTotal = v, true + } + case strings.HasPrefix(line, "HugePages_Free:"): + if v, parsed := parseMeminfoCount(line); parsed { + free, haveFree = v, true + } + } + if haveTotal && haveFree { + break + } + } + + return total, free, haveTotal && haveFree +} + +// parseMeminfoCount pulls the integer out of a "Key: " meminfo line (the +// HugePages_ counts are bare counts, no unit). +func parseMeminfoCount(line string) (uint64, bool) { + fields := strings.Fields(line) + if len(fields) < 2 { + return 0, false + } + v, err := strconv.ParseUint(fields[1], 10, 64) + if err != nil { + return 0, false + } + + return v, true +} diff --git a/packages/orchestrator/pkg/sandbox/fph_pressure_test.go b/packages/orchestrator/pkg/sandbox/fph_pressure_test.go new file mode 100644 index 0000000000..ba9372844f --- /dev/null +++ b/packages/orchestrator/pkg/sandbox/fph_pressure_test.go @@ -0,0 +1,99 @@ +//go:build linux + +package sandbox + +import ( + "testing" + "time" + + "github.com/e2b-dev/infra/packages/shared/pkg/featureflags" +) + +func TestEffectiveHintInterval(t *testing.T) { + t.Parallel() + + base := 30 * time.Second + fast := 5 * time.Second + cfg := func(pressure time.Duration, thr int) featureflags.PeriodicHintingConfig { + return featureflags.PeriodicHintingConfig{Interval: base, PressureInterval: pressure, PressureThresholdPct: thr} + } + + tests := []struct { + name string + cfg featureflags.PeriodicHintingConfig + poolPct int + want time.Duration + }{ + {"adaptation disabled (no pressure interval) keeps base", cfg(0, 80), 95, base}, + {"adaptation disabled (no threshold) keeps base", cfg(fast, 0), 95, base}, + {"below threshold keeps base", cfg(fast, 80), 79, base}, + {"at threshold uses fast interval", cfg(fast, 80), 80, fast}, + {"above threshold uses fast interval", cfg(fast, 80), 99, fast}, + {"unknown pool pressure (-1) keeps base, fails safe", cfg(fast, 80), -1, base}, + {"pressure interval not shorter than base never slows down", cfg(60*time.Second, 80), 95, base}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := effectiveHintInterval(tt.cfg, tt.poolPct); got != tt.want { + t.Errorf("effectiveHintInterval(%v, %d) = %v, want %v", tt.cfg, tt.poolPct, got, tt.want) + } + }) + } +} + +func TestHugePageUsagePct(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + total, free uint64 + ok bool + want int + }{ + {"half used", 64, 32, true, 50}, + {"full", 64, 0, true, 100}, + {"empty", 64, 64, true, 0}, + {"no pool (total 0) is unknown", 0, 0, true, -1}, + {"unreadable is unknown", 64, 32, false, -1}, + {"free>total is unknown (inconsistent)", 64, 65, true, -1}, + {"69 of 100", 100, 31, true, 69}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + if got := hugePageUsagePct(tt.total, tt.free, tt.ok); got != tt.want { + t.Errorf("hugePageUsagePct(%d,%d,%v) = %d, want %d", tt.total, tt.free, tt.ok, got, tt.want) + } + }) + } +} + +// the cached reader serves a stable value within the TTL from one underlying read. +func TestHugePagePressureCachesWithinTTL(t *testing.T) { + t.Parallel() + + reads := 0 + h := &hugePagePressure{readFile: func() (uint64, uint64, bool) { + reads++ + return 100, uint64(100 - reads*10), true // free drops each read + }} + + t0 := time.Now() + first := h.usagePct(t0) + again := h.usagePct(t0.Add(hugePagePressureTTL / 2)) + if first != again { + t.Errorf("within TTL expected cached %d, got %d", first, again) + } + if reads != 1 { + t.Errorf("expected 1 underlying read within TTL, got %d", reads) + } + + later := h.usagePct(t0.Add(hugePagePressureTTL * 2)) + if reads != 2 { + t.Errorf("expected a fresh read after TTL, got %d reads", reads) + } + if later == first { + t.Errorf("expected a refreshed value after TTL") + } +} diff --git a/packages/shared/pkg/featureflags/flags.go b/packages/shared/pkg/featureflags/flags.go index 592bd2e195..64a844e328 100644 --- a/packages/shared/pkg/featureflags/flags.go +++ b/packages/shared/pkg/featureflags/flags.go @@ -977,6 +977,15 @@ type PeriodicHintingConfig struct { UnresponsiveRetry time.Duration HostSlots int Stop HintStopConfig + // PressureInterval, when > 0, is the hinting interval used while the node's + // hugepage pool is at or above PressureThresholdPct utilization. It is meant + // to be shorter than Interval: under pool pressure, free-page hinting runs + // more often so a guest's just-freed pages return to the pool sooner, + // shrinking the reclaim-lag window between a guest freeing a page and the + // pool getting it back. 0 disables the adaptation — the fixed Interval is + // always used (the historical behaviour). + PressureInterval time.Duration + PressureThresholdPct int } func (c PeriodicHintingConfig) Enabled() bool { return c.Interval > 0 } @@ -1026,14 +1035,27 @@ func GetPeriodicHintingConfig(ctx context.Context, ff *Client, contexts ...ldcon return false } cfg := PeriodicHintingConfig{ - Interval: max(ms("interval", 0), 0), - Timeout: ms("timeout", defaultPeriodicHintTimeout.Milliseconds()), - QuietAfterStart: max(ms("quiet_after_start", 10000), 0), - SilentRuns: int(max(num("silent_runs", 3), 0)), - UnresponsiveRetry: max(ms("unresponsive_retry", 300000), 0), - HostSlots: int(max(num("host_slots", 16), 0)), - Stop: hintStopConfig(top), - ObserveOnly: boolean("observe_only"), + Interval: max(ms("interval", 0), 0), + Timeout: ms("timeout", defaultPeriodicHintTimeout.Milliseconds()), + QuietAfterStart: max(ms("quiet_after_start", 10000), 0), + SilentRuns: int(max(num("silent_runs", 3), 0)), + UnresponsiveRetry: max(ms("unresponsive_retry", 300000), 0), + HostSlots: int(max(num("host_slots", 16), 0)), + Stop: hintStopConfig(top), + ObserveOnly: boolean("observe_only"), + PressureInterval: max(ms("pressure_interval", 0), 0), + PressureThresholdPct: int(num("pressure_threshold_pct", 0)), + } + // A pressure interval below the floor (but non-zero) would hint too hot; + // clamp it off rather than silently flooring, same policy as Interval. + if cfg.PressureInterval > 0 && cfg.PressureInterval < minPeriodicHintingInterval { + complain("pressure_interval", "below the one-second floor; pressure adaptation off", v.GetByKey("pressure_interval")) + cfg.PressureInterval = 0 + } + // An out-of-range threshold disables the adaptation rather than triggering + // on every tick (0) or never (>100). + if cfg.PressureThresholdPct <= 0 || cfg.PressureThresholdPct > 100 { + cfg.PressureInterval = 0 } if cfg.Interval > 0 && cfg.Interval < minPeriodicHintingInterval { complain("interval", "below the one-second floor; periodic hinting off", v.GetByKey("interval"))