From 2df2a3d8d1054cc0c1b5d1cbe8ff0e78244ea48e Mon Sep 17 00:00:00 2001 From: ylm Date: Wed, 9 Sep 2026 23:09:42 -0400 Subject: [PATCH 1/2] Clarify the search in-flight ceiling's dual role MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The admission gate admits before the body is parsed, so it can't tell a cheap request (a live-search-cache coalesced join or TTL hit) from an expensive fresh compute. That makes SERVER_MAX_IN_FLIGHT_SEARCHES two bounds in one: it caps request concurrency, and it is still the heap bound for distinct searches — the worst case is that many concurrent multi-MB result documents. The live-search cache collapses identical bursts to ~one document but does nothing for distinct concurrent queries, so raising this ceiling to be friendlier to identical bursts would raise the distinct-query worst case and re-expose the heap exhaustion the bound exists to prevent. Document that, so the number isn't mistaken for a pure heap figure and raised. Comment-only; the default stays 30. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_014GsYGsuCqHJz9GAti4jheG --- packages/runtime-common/search-bounds.ts | 33 ++++++++++++++++++------ 1 file changed, 25 insertions(+), 8 deletions(-) diff --git a/packages/runtime-common/search-bounds.ts b/packages/runtime-common/search-bounds.ts index dd2ebf07305..930b671fd32 100644 --- a/packages/runtime-common/search-bounds.ts +++ b/packages/runtime-common/search-bounds.ts @@ -39,11 +39,12 @@ const log = logger('search-bounds'); // - In-flight ceiling (SERVER_MAX_IN_FLIGHT_SEARCHES, with // SEARCH_ADMISSION_WAIT_MS) — server-side only, and unlike the others a // bound on the process rather than on a request: how many searches it runs -// at once, across every caller. Each in-flight search holds tens of MB of -// heap while its result set is assembled, so this is the number that -// decides whether a burst exhausts the heap. Enforced at admission in the -// realm-server's request middleware; arrivals above the ceiling wait -// briefly for a slot and are then shed with 429 + Retry-After. +// at once, across every caller. Each distinct in-flight search holds tens +// of MB of heap while its result set is assembled, so this is the number +// that decides whether a burst of distinct searches exhausts the heap +// (identical ones share one document via the live-search cache). Enforced +// at admission in the realm-server's request middleware; arrivals above the +// ceiling wait briefly for a slot and are then shed with 429 + Retry-After. // // All bounds are exported consts, overridable via env for ops tuning. // --------------------------------------------------------------------------- @@ -154,9 +155,25 @@ export const SEARCH_CONCURRENCY_CAP = parsePositiveInt( // Max searches the realm-server process runs at once, across every caller. // Enforced server-side at admission (see the realm-server's -// `search-inflight.ts`). Sized against the per-search heap cost: a few dozen -// concurrent federated searches exhaust a 2 GB heap, so the default keeps a -// process on the default heap alive and leaves headroom on a larger one. +// `search-inflight.ts`). +// +// This ceiling plays two roles at once, and the second is why it can't simply +// be raised. It bounds request concurrency; and because the gate admits before +// the body is parsed — so a shed costs nothing — it cannot tell a cheap request +// from an expensive one, so it is also the heap bound for *distinct* searches: +// the worst case is this many concurrent multi-MB result documents, which is +// what exhausts a 2 GB heap. The default holds a process on the default heap +// alive and leaves headroom on a larger one. +// +// The live-search cache (coalescing + short-TTL body cache) makes a burst of +// byte-identical searches cost ~one document rather than one per request, but +// it does nothing for distinct concurrent queries, and the gate can't tell the +// two apart at admission time. So raising this to be friendlier to identical +// bursts would also raise the distinct-query worst case and re-expose the heap +// exhaustion this bound exists to prevent — identical-burst overflow is instead +// shed and safely retried into a cache hit. Tune per environment against the +// distinct-query heap cost, never against identical-burst volume. +// // Indexing traffic is admitted regardless of this ceiling (it is bounded // upstream by the prerender pool), so the effective room for interactive // searches is whatever indexing isn't using. From 046a7a3b00a9f4ea1a85f834ac1afacd59fd991e Mon Sep 17 00:00:00 2001 From: ylm Date: Thu, 10 Sep 2026 19:47:04 -0400 Subject: [PATCH 2/2] Address review: scope live-search-cache claims to /_federated-search The in-flight ceiling's doc comments implied the live-search cache benefits every gated search and that a shed request always retries into a cache hit. Neither holds: the gate admits both `/_search` and `/_federated-search`, but `LiveSearchCache` is wired only into the federated handler, so per-realm `/_search` calls assemble independently; and a retry is a hit only while the first response is still retained (non-zero LIVE_SEARCH_CACHE_TTL_MS, body within LIVE_SEARCH_CACHE_MAX_BYTES), otherwise it re-assembles the document. Qualify both comments accordingly. Co-Authored-By: Claude Opus 4.8 (1M context) Claude-Session: https://claude.ai/code/session_014GsYGsuCqHJz9GAti4jheG --- packages/runtime-common/search-bounds.ts | 26 +++++++++++++++--------- 1 file changed, 16 insertions(+), 10 deletions(-) diff --git a/packages/runtime-common/search-bounds.ts b/packages/runtime-common/search-bounds.ts index 930b671fd32..5e0124c324b 100644 --- a/packages/runtime-common/search-bounds.ts +++ b/packages/runtime-common/search-bounds.ts @@ -42,8 +42,10 @@ const log = logger('search-bounds'); // at once, across every caller. Each distinct in-flight search holds tens // of MB of heap while its result set is assembled, so this is the number // that decides whether a burst of distinct searches exhausts the heap -// (identical ones share one document via the live-search cache). Enforced -// at admission in the realm-server's request middleware; arrivals above the +// (identical live `/_federated-search` requests can share one document via +// the live-search cache, which is wired into that handler only; per-realm +// `/_search` calls, also gated here, assemble independently). Enforced at +// admission in the realm-server's request middleware; arrivals above the // ceiling wait briefly for a slot and are then shed with 429 + Retry-After. // // All bounds are exported consts, overridable via env for ops tuning. @@ -165,14 +167,18 @@ export const SEARCH_CONCURRENCY_CAP = parsePositiveInt( // what exhausts a 2 GB heap. The default holds a process on the default heap // alive and leaves headroom on a larger one. // -// The live-search cache (coalescing + short-TTL body cache) makes a burst of -// byte-identical searches cost ~one document rather than one per request, but -// it does nothing for distinct concurrent queries, and the gate can't tell the -// two apart at admission time. So raising this to be friendlier to identical -// bursts would also raise the distinct-query worst case and re-expose the heap -// exhaustion this bound exists to prevent — identical-burst overflow is instead -// shed and safely retried into a cache hit. Tune per environment against the -// distinct-query heap cost, never against identical-burst volume. +// The live-search cache (coalescing + short-TTL body cache, on the +// `/_federated-search` handler only) makes a burst of byte-identical federated +// searches cost ~one document rather than one per request, but it does nothing +// for distinct concurrent queries — nor for per-realm `/_search`, which the +// gate admits too — and the gate can't tell any of these apart at admission +// time. So raising this to be friendlier to identical bursts would also raise +// the distinct-query worst case and re-expose the heap exhaustion this bound +// exists to prevent — identical-burst overflow is instead shed and retried, +// which lands as a cache hit only while the first response is still retained +// (a non-zero LIVE_SEARCH_CACHE_TTL_MS, body within LIVE_SEARCH_CACHE_MAX_BYTES); +// otherwise the retry re-assembles the document. Tune per environment against +// the distinct-query heap cost, never against identical-burst volume. // // Indexing traffic is admitted regardless of this ceiling (it is bounded // upstream by the prerender pool), so the effective room for interactive