-
Notifications
You must be signed in to change notification settings - Fork 0
[WRONG BRANCH] fix(server): bound remote catalog serialization #383
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -11,6 +11,7 @@ | |
| * copy is the one nobody looks at in the dashboard. | ||
| */ | ||
| import { createHash } from "node:crypto"; | ||
| import { statSync } from "node:fs"; | ||
|
|
||
| /** | ||
| * Upper bound for the REMOTE route only. | ||
|
|
@@ -35,6 +36,29 @@ export interface SerializedCatalog { | |
| etag?: string; | ||
| /** Byte length of `body`, present only when `body` is. */ | ||
| bytes?: number; | ||
| /** Remote-only preflight failure; management serialization never sets this. */ | ||
| error?: "too_large"; | ||
| } | ||
|
|
||
| interface CatalogFileIdentity { | ||
| key: string; | ||
| size: number; | ||
| } | ||
|
|
||
| let remoteCache: { path: string; identity: string; serialized: SerializedCatalog } | undefined; | ||
| let remoteSerialization: { path: string; identity: string; result: Promise<SerializedCatalog> } | undefined; | ||
|
|
||
| function catalogFileIdentity(path: string): CatalogFileIdentity | null { | ||
| try { | ||
| const stat = statSync(path, { bigint: true }); | ||
| if (!stat.isFile()) return null; | ||
| return { | ||
| key: `${stat.dev}:${stat.ino}:${stat.size}:${stat.mtimeNs}:${stat.ctimeNs}`, | ||
| size: Number(stat.size), | ||
| }; | ||
| } catch { | ||
| return null; | ||
| } | ||
| } | ||
|
|
||
| export function catalogEtag(body: string): string { | ||
|
|
@@ -61,6 +85,46 @@ export async function serializePersistedCatalog(): Promise<SerializedCatalog> { | |
| return { body, etag: catalogEtag(body), bytes }; | ||
| } | ||
|
|
||
| /** | ||
| * Serialize the catalog for the remotely reachable route. | ||
| * | ||
| * File size is checked before the synchronous reader can materialize it. A | ||
| * validated file identity caches the expensive parse/stringify/hash result, | ||
| * and the shared promise limits cache misses to one serialization at a time. | ||
| */ | ||
| export async function serializeRemotePersistedCatalog(): Promise<SerializedCatalog> { | ||
| const { readCodexCatalogPath } = await import("../codex/catalog"); | ||
| const path = readCodexCatalogPath(); | ||
| const identity = catalogFileIdentity(path); | ||
| if (!identity) return { body: null }; | ||
| if (identity.size > MAX_REMOTE_CATALOG_BYTES) return { body: null, error: "too_large" }; | ||
| if (remoteCache?.path === path && remoteCache.identity === identity.key) return remoteCache.serialized; | ||
| if (remoteSerialization) { | ||
| if (remoteSerialization.path === path && remoteSerialization.identity === identity.key) { | ||
| return remoteSerialization.result; | ||
| } | ||
| await remoteSerialization.result; | ||
| return serializeRemotePersistedCatalog(); | ||
| } | ||
|
|
||
| const result = (async () => { | ||
| const serialized = await serializePersistedCatalog(); | ||
| if (serialized.bytes !== undefined && serialized.bytes > MAX_REMOTE_CATALOG_BYTES) { | ||
| return { body: null, error: "too_large" } as SerializedCatalog; | ||
| } | ||
| const after = catalogFileIdentity(path); | ||
| if (after?.key !== identity.key) return { body: null }; | ||
|
Comment on lines
+115
to
+116
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
When a catalog sync atomically replaces this file after the initial identity check but before this post-serialization check, the new file is valid and present, yet this branch returns Useful? React with 👍 / 👎. |
||
| if (serialized.body !== null) remoteCache = { path, identity: identity.key, serialized }; | ||
| return serialized; | ||
| })(); | ||
| remoteSerialization = { path, identity: identity.key, result }; | ||
| try { | ||
| return await result; | ||
| } finally { | ||
| if (remoteSerialization?.result === result) remoteSerialization = undefined; | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * The authoritative Codex version for a catalog response, or undefined. | ||
| * | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
When a previously cached catalog is later deleted, becomes unreadable, or is replaced by an oversized file, these early returns never clear
remoteCache, so the old serialized body remains strongly referenced for the rest of the process unless another valid catalog is successfully serialized. Because one cached entry may be nearlyMAX_REMOTE_CATALOG_BYTES(256 MiB), this can retain substantial heap even after the operator removes the problematic catalog; clear the stale cache whenever the current path or identity is missing or rejected.Useful? React with 👍 / 👎.