From 88b4385914b5e8e9cfcffa725a1861182e3753c4 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Tue, 20 Oct 2026 10:47:03 -0400 Subject: [PATCH 1/3] Record layout F: records in packs per collection and set, a layout per collection Layout F keeps a collection's records out of leaf bodies, in packs written once and found through a sorted index keyed by (record id, record hash); rewritten leaf nodes are stored as deltas against an earlier full node. Each collection's public and private records are kept apart (runs///: packs, index runs, a manifest as objects, its own maintain jobs), so a public-only mirror copies files that hold nothing private. Version hashes are the same as today's layout's. Chosen with the scale bench (record-storage-layout.md, scale-testing.md): at 12M records a daily version commits in 68-93 s against today's 294-371 s and writes 165-414 MB against 16 GB; 20,000 collections load flat. - A layout per collection: collections.record_layout (migration 0025; existing collections are `today`), new collections get PlatformStorage.newCollections (default `today` for now); forks take their parent's layout and reference its files; stores open each collection's repository in its layout. - F's jobs: layout.maintain and layout.merge per set, within a Worker's storage-call budget; reclaim per set (not started yet: needs the write fence). - Commits: units hold their runs, the assembly names each tree's as one pending object; a type changing privacy commits serially and copies its records to the other set. - Mirrors copy the sets the placement holds; sync packs carry bodies built from F's records and full nodes, so receivers see today's format; fsck reads F. - Cleanup drops a deleted F collection's files after its grace, keeping what a live fork still reads. - Compaction writes a joined run index once (the last two pieces can both join: a write R2 refuses). - The local file store lists from the prefix's folder and reads small objects whole in get. The server suite runs once a layout (vitest projects `today` and `f`), and a test runs today's and F side by side: same versions, same records, both pass fsck. --- packages/protocol/src/index.ts | 22 + packages/protocol/src/repo/build.ts | 71 +- packages/protocol/src/repo/file-sets.ts | 7 +- packages/protocol/src/repo/fsck.ts | 31 +- packages/protocol/src/repo/gzip.ts | 12 +- packages/protocol/src/repo/layouts/index.ts | 40 + packages/protocol/src/repo/layouts/keyed.ts | 1196 ++++++ packages/protocol/src/repo/layouts/kit.ts | 1168 ++++++ packages/protocol/src/repo/layouts/reclaim.ts | 120 + packages/protocol/src/repo/layouts/types.ts | 75 + packages/protocol/src/repo/repo.ts | 301 +- packages/protocol/src/repo/sync.ts | 44 +- packages/protocol/src/stores/fs.ts | 60 +- packages/protocol/src/tree/assemble.ts | 10 +- packages/protocol/src/tree/builder.ts | 2 + packages/protocol/src/tree/merge.ts | 57 +- packages/protocol/src/tree/read.ts | 65 +- packages/protocol/src/tree/source.ts | 11 + packages/protocol/test/repo/layouts.test.ts | 630 +++ .../protocol/test/stores/contract.test.ts | 12 + packages/protocol/test/stores/fs-get.test.ts | 30 + packages/protocol/test/stores/fs-list.test.ts | 50 + .../server/drizzle/0025_record_layout.sql | 3 + .../server/drizzle/meta/0025_snapshot.json | 3583 +++++++++++++++++ packages/server/drizzle/meta/_journal.json | 7 + packages/server/src/api/manage.ts | 3 + packages/server/src/api/versions.ts | 3 +- packages/server/src/cleanup/layout-reclaim.ts | 98 + packages/server/src/cleanup/marks.ts | 6 + packages/server/src/cleanup/sweep.ts | 59 +- packages/server/src/db/schema.ts | 15 + packages/server/src/handlers.ts | 54 +- packages/server/src/jobs.ts | 5 + packages/server/src/locations/mirror.ts | 76 +- packages/server/src/ports.ts | 6 +- packages/server/src/push/compact.ts | 4 +- packages/server/src/push/parallel.ts | 63 +- packages/server/src/push/runs.ts | 9 +- packages/server/src/stores.ts | 55 +- packages/server/src/versions/fork.ts | 15 +- packages/server/src/versions/view.ts | 35 +- packages/server/test/cleanup.test.ts | 9 +- packages/server/test/commit.test.ts | 3 +- packages/server/test/fsck.test.ts | 8 +- packages/server/test/harness.ts | 33 +- packages/server/test/layouts.test.ts | 166 + packages/server/test/manage.test.ts | 6 +- packages/server/test/mirrors.test.ts | 125 +- packages/server/test/parallel.test.ts | 50 +- packages/server/test/recover.test.ts | 47 + packages/server/test/schema-parallel.test.ts | 30 +- packages/server/vitest.config.ts | 17 +- 52 files changed, 8495 insertions(+), 112 deletions(-) create mode 100644 packages/protocol/src/repo/layouts/index.ts create mode 100644 packages/protocol/src/repo/layouts/keyed.ts create mode 100644 packages/protocol/src/repo/layouts/kit.ts create mode 100644 packages/protocol/src/repo/layouts/reclaim.ts create mode 100644 packages/protocol/src/repo/layouts/types.ts create mode 100644 packages/protocol/test/repo/layouts.test.ts create mode 100644 packages/protocol/test/stores/fs-get.test.ts create mode 100644 packages/protocol/test/stores/fs-list.test.ts create mode 100644 packages/server/drizzle/0025_record_layout.sql create mode 100644 packages/server/drizzle/meta/0025_snapshot.json create mode 100644 packages/server/src/cleanup/layout-reclaim.ts create mode 100644 packages/server/test/layouts.test.ts diff --git a/packages/protocol/src/index.ts b/packages/protocol/src/index.ts index a8fbd827..9b78817f 100644 --- a/packages/protocol/src/index.ts +++ b/packages/protocol/src/index.ts @@ -6,6 +6,28 @@ export * from './format.js' export * from './repo/types.js' export * from './repo/repo.js' +// Record layouts: today's (leaf bodies) and F (packs per collection and set), record-storage-layout.md. +export { + headConfig, + HomeStore, + type Inherited, + type HeldRun, + KeyedLayout, + LAYOUTS, + type LayerInfo, + type LayoutName, + layoutConfig, + type LiveSet, + type MergePlan, + type PackInfo, + MissingRecordError, + type ReclaimPlan, + recordKey, + type RecordSet, + type RecordSets, + setHome, +} from './repo/layouts/index.js' +export { LiveHashes, markRange, recordRoots } from './repo/layouts/reclaim.js' export { fsck, type FsckCursor, diff --git a/packages/protocol/src/repo/build.ts b/packages/protocol/src/repo/build.ts index 3ddef862..83384a1d 100644 --- a/packages/protocol/src/repo/build.ts +++ b/packages/protocol/src/repo/build.ts @@ -28,6 +28,7 @@ import { getEntry, isEmptySet, iterate, + leaves, makeRoot, mergeTree, type PrivateSetObject, @@ -176,6 +177,38 @@ async function* treeAsUpserts( } } +/** Records one copy writes as a pending run of the set's index before starting another. */ +const COPY_BATCH = 100_000 + +/** + * Layout F: a type's tree moving to the other set takes its records along, copied + * as stored into that set's packs, since a set's files hold only its own records + * (a public mirror copies them as they are). O(type size), as a rebuild is; the + * old set's copies go when its reclaim finds them unreferenced. + */ +async function copyRecords(repo: Repo, root: string | null, from: SetName, to: SetName) { + if (!repo.records || root === null) return + const source = new RepoSource(recordTree, repo, { set: from }) + let writer = repo.records[to].writer() + let n = 0 + for await (const leaf of leaves(source, root)) { + const node = await source.node(leaf.hash) + const entries = (node.kind === 'leaf' ? node.entries : []) as RecordEntry[] + for (let i = 0; i < entries.length; i += 256) { + const at = Array.from({ length: Math.min(256, entries.length - i) }, (_, j) => i + j) + const lines = await repo.records[from].linesAt(leaf.hash, entries, at) + at.forEach((k, j) => writer.add(entries[k]!, lines[j]!)) + n += at.length + } + if (n >= COPY_BATCH) { + await writer.finish() + writer = repo.records[to].writer() + n = 0 + } + } + await writer.finish() +} + const summary = (t: TreeSummary | undefined): TreeSummary => t ? { root: t.root, count: t.count, bytes: t.bytes } : { root: null, count: 0, bytes: 0 } @@ -187,8 +220,15 @@ export async function buildVersion(repo: Repo, input: BuildInput): Promise(repo, { bodyOf: bodyOfRecord }) + // A source and a sink a set: layout F reads and writes each set's records apart. + const sources = { + public: new RepoSource(recordTree, repo, { set: 'public' }), + private: new RepoSource(recordTree, repo, { set: 'private' }), + } + const sinks = { + public: new RepoSink(repo, { bodyOf: bodyOfRecord, set: 'public' }), + private: new RepoSink(repo, { bodyOf: bodyOfRecord, set: 'private' }), + } const builderOpts = { payloadBytes: recordPayloadBytes, dropPayload: dropRecordBody, @@ -208,8 +248,9 @@ export async function buildVersion(repo: Repo, input: BuildInput): Promise>, ) => { - const result = await mergeTree(source, sink, baseTree, changes, { + const result = await mergeTree(sources[set], sinks[set], baseTree, changes, { ...builderOpts, + oldPayloads: !refs.noOldRefs.has(set), onChange: (before, after) => { refs.record(set, before, after, type) recordsChanged = true @@ -229,6 +270,9 @@ export async function buildVersion(repo: Repo, input: BuildInput): Promise = { public: await tracksTypes(repo, refsRoots.public), private: await tracksTypes(repo, refsRoots.private), @@ -256,7 +300,7 @@ export async function buildVersion(repo: Repo, input: BuildInput): Promise `schemas/${t.schemaHash}.json`)) + sinks.public.written.push(...input.types.map((t) => `schemas/${t.schemaHash}.json`)) for (const t of input.types) { const pubBase = basePublic.types[t.slug] @@ -303,6 +347,7 @@ export async function buildVersion(repo: Repo, input: BuildInput): Promise 0) recordsChanged = true } } - await sink.flush() + await sinks.public.flush() + await sinks.private.flush() // Record ids are unique per type across both sets (docs/protocol-v2.md §4). The // base holds that, so only ids entering a set can break it: probe the other set @@ -376,9 +423,10 @@ export async function buildVersion(repo: Repo, input: BuildInput): Promise `${type}/${hash}` * `resolve`, which must run before `refs` is used. */ export class FileRefDelta { + /** + * Sets whose base holds no file reference at all: a changed record's old version + * can't take one away, so it isn't read (packed layouts skip fetching it). + */ + readonly noOldRefs = new Set() /** Per set: file hash → change in references. */ readonly refs: Record> = { public: new Map(), private: new Map() } /** Per set: "/" → change in that type's references. */ @@ -98,7 +103,7 @@ export class FileRefDelta { /** Feed one record change of a type in a set (mergeTree's onChange). */ record(set: SetName, before: RecordEntry | null, after: RecordEntry | null, type: string): void { - if (before) this.#body(set, type, before.body, -1) + if (before && !this.noOldRefs.has(set)) this.#body(set, type, before.body, -1) if (after) this.#body(set, type, after.body, +1) } diff --git a/packages/protocol/src/repo/fsck.ts b/packages/protocol/src/repo/fsck.ts index abba1e8f..08a48a30 100644 --- a/packages/protocol/src/repo/fsck.ts +++ b/packages/protocol/src/repo/fsck.ts @@ -23,6 +23,7 @@ import { mergeTree } from '../tree/merge.js' import { fileTree, recordTree, type RecordEntry } from '../tree/node.js' import { diffTrees, leaves, newNodes } from '../tree/read.js' import { verifyTree } from '../tree/verify.js' +import type { RecordSet } from './layouts/index.js' import { type PublicKeyInfo, readCollectionInfo, @@ -31,7 +32,7 @@ import { verifyLog, verifyLogEntries, } from './log.js' -import { keys, openRepo, type Repo, RepoSource } from './repo.js' +import { keys, type Repo, RepoSource } from './repo.js' export interface FsckOptions { /** Version hashes to check; with a collection id and none given, the log's. */ @@ -63,7 +64,8 @@ export interface FsckReport { export async function fsck(given: Repo, opts: FsckOptions = {}): Promise { // Storage itself: no shared cache or memory of earlier reads, and every object // hash-checked as if from a location we don't operate. - const repo = openRepo(given.blobs, { trusted: false }) + // Untrusted, so every object is checked against its hash; in the layout it has. + const repo = given.reopen({ trusted: false }) const max = opts.maxErrors ?? 100 const report: FsckReport = { ok: true, @@ -121,7 +123,7 @@ export async function fsck(given: Repo, opts: FsckOptions = {}): Promise() const seenSchemas = new Set() - const checkSet = async (where: string, set: SetObject) => { + const checkSet = async (where: string, set: SetObject, rs: RecordSet) => { for (const [slug, t] of Object.entries(set.types)) { if (!seenSchemas.has(t.schema)) { seenSchemas.add(t.schema) @@ -147,7 +149,7 @@ export async function fsck(given: Repo, opts: FsckOptions = {}): Promise records.node(leaf.hash)) if (!node || node.kind !== 'leaf') continue await attempt(`body of ${leaf.hash}`, () => - repo.bodyLines({ hash: leaf.hash, entries: node.entries as RecordEntry[] }), + repo.bodyLines({ hash: leaf.hash, entries: node.entries as RecordEntry[] }, rs), ) } } @@ -204,7 +206,7 @@ export async function fsck(given: Repo, opts: FsckOptions = {}): Promise { - const repo = openRepo(given.blobs, { trusted: false }) + // Untrusted, so every object is checked against its hash; in the layout it has. + const repo = given.reopen({ trusted: false }) const max = opts.maxErrors ?? 100 const budget = opts.changeBudget ?? 1_000_000 const report: FsckReport = { @@ -347,7 +350,7 @@ export async function fsckStep( if (n !== size || hasher.hex() !== hash) fail(`file ${hash} fails its hash or size`) } - const checkSet = async (where: string, set: SetObject, base: SetObject | null) => { + const checkSet = async (where: string, set: SetObject, base: SetObject | null, rs: RecordSet) => { let changes = 0 for (const [slug, t] of Object.entries(set.types)) { const was = base?.types[slug] @@ -369,7 +372,7 @@ export async function fsckStep( const node = await attempt(`leaf ${n.hash}`, () => records.node(n.hash)) if (!node || node.kind !== 'leaf') continue await attempt(`body of ${n.hash}`, () => - repo.bodyLines({ hash: n.hash, entries: node.entries as RecordEntry[] }), + repo.bodyLines({ hash: n.hash, entries: node.entries as RecordEntry[] }, rs), ) } } @@ -418,11 +421,17 @@ export async function fsckStep( if (now) { const pubErr = checkSetObject(now.root.public) if (pubErr) fail(`${where} public ${pubErr}`) - else spent += await checkSet(`${where} public`, now.root.public, before?.root.public ?? null) + else + spent += await checkSet( + `${where} public`, + now.root.public, + before?.root.public ?? null, + 'public', + ) if (now.priv) { const privErr = checkSetObject(now.priv, true) if (privErr) fail(`${where} private ${privErr}`) - else spent += await checkSet(`${where} private`, now.priv, before?.priv ?? null) + else spent += await checkSet(`${where} private`, now.priv, before?.priv ?? null, 'private') } } cursor = { seq, log, versionHash: entry.versionHash } diff --git a/packages/protocol/src/repo/gzip.ts b/packages/protocol/src/repo/gzip.ts index 6aaa084a..bee49b3d 100644 --- a/packages/protocol/src/repo/gzip.ts +++ b/packages/protocol/src/repo/gzip.ts @@ -35,6 +35,7 @@ interface Gunzip extends AsyncIterable { end(chunk?: Uint8Array): void destroy(err?: Error): void once(event: 'drain' | 'close', listener: () => void): void + off(event: 'drain' | 'close', listener: () => void): void readonly destroyed: boolean } const zlib = ( @@ -129,9 +130,16 @@ export async function* gunzipStream(input: AsyncIterable): AsyncGene for await (const c of input) { if (stream.destroyed) return if (!stream.write(c)) + // Whichever comes first; the other listener is removed, or one is left + // behind every time the stream pushes back (a leak across a long body). await new Promise((resolve) => { - stream.once('drain', resolve) - stream.once('close', resolve) + const done = () => { + stream.off('drain', done) + stream.off('close', done) + resolve() + } + stream.once('drain', done) + stream.once('close', done) }) } stream.end() diff --git a/packages/protocol/src/repo/layouts/index.ts b/packages/protocol/src/repo/layouts/index.ts new file mode 100644 index 00000000..d6e153db --- /dev/null +++ b/packages/protocol/src/repo/layouts/index.ts @@ -0,0 +1,40 @@ +/** The record layouts, by name (record-storage-layout.md). */ +import type { Lru } from '../lru.js' +import type { Store } from '../types.js' +import { HomeStore, KeyedLayout } from './keyed.js' +import type { LayoutName, RecordSet, RecordSets } from './types.js' + +export { + HomeStore, + type Inherited, + KeyedLayout, + layoutConfig, + type MergePlan, + MissingRecordError, + type ReclaimPlan, +} from './keyed.js' +export { headConfig, type LayerInfo, type PackInfo, pendingConfig } from './kit.js' +export * from './types.js' + +/** A set's home under the collection's: `runs///` holds its files. */ +export const setHome = (collection: string, set: RecordSet) => `${collection}/${set}` + +/** The record stores for `name`, a set each (null for today's leaf bodies). */ +export function recordStore( + name: LayoutName, + lru: Lru, + scope: string, + /** The location's store and the collection whose prefix (`runs//`) holds its files. */ + own?: { root: Store; home: string }, +): RecordSets | null { + switch (name) { + case 'today': + return null + case 'f': { + if (!own) throw new Error('Layout f keeps records per collection: it needs the collection') + const set = (s: RecordSet) => + new KeyedLayout(new HomeStore(own.root, setHome(own.home, s)), lru, `${scope}/${s}`) + return { public: set('public'), private: set('private') } + } + } +} diff --git a/packages/protocol/src/repo/layouts/keyed.ts b/packages/protocol/src/repo/layouts/keyed.ts new file mode 100644 index 00000000..9e5305e6 --- /dev/null +++ b/packages/protocol/src/repo/layouts/keyed.ts @@ -0,0 +1,1196 @@ +import type { RecordEntry } from '../../tree/node.js' +/** + * Layout F (record-storage-layout.md): records in packs, found through an index + * keyed by (id, record hash), with no map per leaf. Everything is per collection: + * its packs, index and manifest live under its own prefix (`runs//`), + * the manifest as objects (no database), so a collection is maintained on its own + * and a mirror copies its files as written. A fork reads its parent's files where + * they are (HomeStore); the parent keeps whatever a live fork lists. + * + * A commit unit appends its new records to packs in key order and its index + * entries as one layer. A maintain job then merges what's due: adjacent layers + * into one, and small packs of a tier into one, re-pointing their records in a + * new layer. Reclaiming keeps only what a mark found live. + * + * The record type isn't part of the key (readers of a leaf don't know it), so a + * leaf's index range also holds other types' records with ids in that range. + */ +import { compareUtf8 } from '../../utf8.js' +import type { Lru } from '../lru.js' +import { PrefixedStore, type Store } from '../types.js' +import { + FANOUT, + LAYER_FANOUT, + type LayerInfo, + type PendingRun, + LayerReader, + LayeredIndex, + levelFor, + RunWriter, + runEntries, + runs, + type Manifest, + ManifestStore, + mergeSorted, + PackRegistry, + newId, + type PackInfo, + PackReader, + PackWriter, + packDirKey, + packKey, + parseLoc, + topSeq, + layerDirKey, + layerFilterKey, + layerKey, + filterKey, +} from './kit.js' +import { type KeyRange, LiveHashes } from './reclaim.js' +import { type LiveSet, recordKey, type RecordStore, type RecordWriter } from './types.js' + +/** Index entries a reclaim range job works on: its live set is 8 bytes an entry. */ +export const RECLAIM_RANGE_ENTRIES = 250_000 + +/** A reclaim's snapshot of the index and packs, and the key ranges its jobs work in. */ +export interface ReclaimPlan { + id: string + layers: LayerInfo[] + packs: { id: string; records: number }[] + ranges: KeyRange[] + /** The registry objects read (the swap replaces them). */ + registry: string[] + /** Where the new run goes: the plan's lowest level and newest sequence (newer commits still win). */ + level: number + seq: number +} + +interface RangeResult { + files: LayerInfo[] + perPack: Record + kept: number + dropped: number +} + +const reclaimKey = (plan: string, name: string) => `reclaim/${plan}/${name}` + +/** + * A merge of more than layoutConfig.mergeJobEntries index entries (a large + * collection's biggest runs: past 5 minutes of CPU at 12M records) is split + * into key ranges, a job each, and swapped in whole when the last is done. + * A split merge not swapped in by then is abandoned (a range job that never ran), and its output deleted. */ +const MERGE_ABANDON_MS = 60 * 60_000 + +/** A merge split into key ranges: its inputs, the ranges, and where its run goes. */ +export interface MergePlan { + id: string + layers: LayerInfo[] + ranges: KeyRange[] + level: number + seq: number + at: number +} + +const mergeKey = (plan: string, name: string) => `merges/${plan}/${name}` + +export interface MaintainOptions { + /** + * Run a merge too large for one job as range jobs (the server queues + * `layout.merge` for each). Without it every merge runs here. + */ + split?: (plan: MergePlan) => Promise +} + +/** A maintain lease older than this (a queue job's 15-minute wall clock) belonged to a job that died. */ +const MAINTAIN_LEASE_MS = 16 * 60_000 + +/** Stored bytes of small packs one pack merge takes (up to `layoutConfig.mergeInputs` packs). */ +const MERGE_PACKS_BYTES = 16 * 1024 * 1024 + +/** Index runs past which the fewest-entry adjacent ones merge regardless of size. */ +const MAX_RUNS = 10 + +/** Packs under this many stored bytes, times 8 per tier, are merged eight at a time. */ +const SMALL_PACK = 4 * 1024 * 1024 +const MAX_PACK_TIER = 3 +/** How long a replaced object stays readable (readers holding an older manifest). */ +export const layoutConfig = { + graceMs: 60_000, + /** Index entries one merge job takes before it's split into key-range jobs (tests set it low). */ + mergeJobEntries: 1_000_000, + /** How long one maintain job merges (one pass at least) before leaving the rest to the next. */ + maintainBudgetMs: 10 * 60_000, + /** + * Storage calls one maintain job makes before leaving the rest to the next: Workers + * allow 10,000 subrequests an invocation, and the pass that crosses this, the sweep + * and settling the manifest fit in the rest (round 19: a 12M load's 354 units left + * one job ~10,460). + */ + maintainCalls: 6_000, + /** + * Index runs, or packs, one merge takes at most: each is two reads, so a pass's + * calls stay well inside what `maintainCalls` leaves (tests set it low). + */ + mergeInputs: 128, +} + +/** + * `store`, counting its storage calls into `n` (a Worker's subrequests). A proxy keeps + * `instanceof` and a store class's own methods, which run on the store itself (their + * calls aren't counted: a few at most a job). + */ +function counted(store: S, n: { calls: number }): S { + const io = new Set(['get', 'head', 'put', 'list', 'delete', 'putParts', 'putPieces', 'copy']) + return new Proxy(store, { + get(t, k) { + const v = Reflect.get(t, k, t) as unknown + if (typeof v !== 'function') return v + const f = v as (...a: unknown[]) => unknown + return io.has(String(k)) + ? (...a: unknown[]) => { + n.calls++ + return f.apply(t, a) + } + : f.bind(t) + }, + }) +} + +/** A leaf's record that the set's index doesn't hold (its other set may). */ +export class MissingRecordError extends Error {} + +export class KeyedLayout implements RecordStore { + readonly manifests: ManifestStore + /** Packs no merge will take again (the manifest lists only the ones merges still may). */ + readonly registry: PackRegistry + readonly packs: PackReader + readonly index: LayeredIndex + readonly layers: LayerReader + + /** Every storage call this layout makes goes through `store`, counted here. */ + readonly #io = { calls: 0 } + readonly store: Store + + readonly name = 'f' + + constructor( + store: Store, + readonly lru: Lru, + readonly scope: string, + ) { + this.store = store = counted(store, this.#io) + this.manifests = new ManifestStore(store, this.name) + this.registry = new PackRegistry(store, this.name) + this.packs = new PackReader(store, lru, scope) + this.index = new LayeredIndex(store, lru, scope, this.manifests) + this.layers = this.index.reader + } + + /** Whether no pack merge will take `p` again: it then lives in the registry, not the manifest. */ + static final(p: PackInfo): boolean { + return p.tier >= MAX_PACK_TIER || p.bytes >= SMALL_PACK * 8 ** p.tier + } + + /** Every pack: the manifest's and the registry's (reclaim, forks). */ + async allPacks(): Promise { + const m = await this.manifests.latest() + return [...m.packs, ...(await this.registry.read()).packs] + } + + async #find(m: Manifest, entries: readonly RecordEntry[]): Promise<(string | undefined)[]> { + // A few records: each looked up through the layers' filters, so only files + // that may hold it are read. A run of a leaf: one range over the layers. + const found = + entries.length <= 64 + ? await this.index.lookup(m, [...new Set(entries.map((e) => e.key))]) + : await this.index.range( + m, + `${entries[0]!.key}\u0000`, + `${entries.at(-1)!.key}\u0000\uffff`, + ) + return entries.map((e) => found.get(recordKey(e.key, e.hash))) + } + + async leafLines(leafHash: string, entries: readonly RecordEntry[]): Promise { + if (!entries.length) return [] + try { + return await this.#leafLines(leafHash, entries, await this.manifests.current()) + } catch (err) { + // A head behind the newest manifest (F moves it at most once a second) can name + // files a merge has since swept: read again from the newest. + if (!(err instanceof Error) || !/^Missing (pack|layer)/.test(err.message)) throw err + return this.#leafLines(leafHash, entries, await this.manifests.latest()) + } + } + + async #leafLines( + leafHash: string, + entries: readonly RecordEntry[], + m: Manifest, + ): Promise { + let found = await this.#find(m, entries) + // Another isolate may have committed since this one read the manifest. + if (found.some((x) => x === undefined)) + found = await this.#find(await this.manifests.latest(), entries) + const missing = found.findIndex((x) => x === undefined) + if (missing >= 0) { + const e = entries[missing]! + throw new MissingRecordError( + `Layout ${this.name}: leaf ${leafHash} has no stored record ${e.key} (${e.hash})`, + ) + } + return this.packs.lines(found as string[]) + } + + linesAt( + leafHash: string, + entries: readonly RecordEntry[], + at: readonly number[], + ): Promise { + return this.leafLines( + leafHash, + at.map((i) => entries[i]!), + ) + } + + async entryLine(leafHash: string, entries: readonly RecordEntry[], i: number): Promise { + return (await this.leafLines(leafHash, [entries[i]!]))[0]! + } + + writer(): RecordWriter { + return new KeyedWriter(this) + } + + /** A commit's runs its units held back (RecordWriter.finish), as one pending object. */ + appendRuns(runs: PendingRun[]): Promise { + return this.manifests.appendRuns(runs) + } + + /** + * Merges and sweeps due on this store. One at a time per store: every commit + * queues a maintain job, and with thousands of collections committing into one + * location's index the jobs piled up, each working out the same merge for one of + * them to swap in (20,000 collections: the bulk queue full of them, loads ~10× + * slower). A job that finds another holding the lease leaves at once: the holder + * reads the manifest again each pass, so it takes in what that commit added + * (and the next commit's job takes in anything after). + * + * True when a budget ended it with merges perhaps still due: the caller queues + * another job, as a load's last commit has no next commit to. + */ + async maintain(opts: MaintainOptions = {}): Promise { + const lease = await this.#lease() + if (!lease) return false + const start = this.#io.calls + const spent = () => this.#io.calls - start + let more = false + try { + // A layer merge and a pack merge each pass, until neither is due: with commits + // landing throughout, layers alone could keep a job busy and packs pile up. A + // job stops well inside its lease and a Worker's subrequests; the next carries on. + const until = Date.now() + layoutConfig.maintainBudgetMs + for (;;) { + const layers = await this.#mergeLayers(opts.split) + const packs = await this.#mergePacks() + if (!layers && !packs) break + if (Date.now() >= until || spent() >= layoutConfig.maintainCalls) { + more = true + break + } + } + // What's left of the budget, or a few at least: the rest waits for the next job. + if (!(await this.#sweepRetired(Math.max(16, layoutConfig.maintainCalls - spent())))) + more = true + await this.manifests.settle() + } finally { + await lease() + } + return more + } + + /** The store's maintain lease, or null if another job holds it; the result releases it. */ + async #lease(): Promise<(() => Promise) | null> { + const key = 'maintain/lease' + for (let attempt = 0; attempt < 2; attempt++) { + const token = newId() + await this.store.put(key, JSON.stringify({ token, at: Date.now() }), { ifAbsent: true }) + // The holder can release between the get and the read; that reads as not held. + const v = await this.store + .get(key) + .then(async (held) => + held ? (JSON.parse(await held.text()) as { token: string; at: number }) : null, + ) + .catch(() => null) + if (v?.token === token) return () => this.store.delete(key) + // A lease older than a job can run is a job that died holding it. + if (!v || Date.now() - v.at < MAINTAIN_LEASE_MS) return null + await this.store.delete(key) + } + return null + } + + /** + * Merge the newest runs, while each older one is no larger than all the newer + * ones together, when there are LAYER_FANOUT of them or more. A merged run + * takes its newest input's sequence number, which is only right when no other + * run's number falls between its inputs': so merges only ever take runs next + * to each other in sequence, newest first (as RocksDB's universal compaction). + * Small commits merge with each other; a large run (an initial load's) joins + * once the newer ones add up to its size; each older run is at least as large + * as all newer ones, so there are about log2 of the entries' runs at most. + */ + async #mergeLayers(split?: MaintainOptions['split']): Promise { + // A split merge's inputs stay one block until it swaps in; runs between such + // blocks merge as usual, adjacent among themselves, so every block stays one. + // At 20,000 collections (round 12) commits added ~200 runs a minute, and + // stopping every merge while a split merge's ranges waited in the queue left + // hundreds of runs. + const blocked = new Set( + (split ? await this.#mergesInFlight() : []).flatMap((p) => runs(p.layers).map((r) => r.id)), + ) + const m = await this.manifests.latest() + const all = runs(m.layers).sort((a, b) => b.seq - a.seq) + const size = (r: (typeof all)[number]) => r.files.reduce((n, f) => n + f.entries, 0) + // Stretches of adjacent runs no split merge holds, newest first. + const free: (typeof all)[] = [[]] + for (const r of all) + if (!blocked.has(r.id)) free.at(-1)!.push(r) + else if (free.at(-1)!.length) free.push([]) + let inputs: typeof all | null = null + for (const rs of free) { + let take = 1 + let newer = rs.length ? size(rs[0]!) : 0 + while (take < rs.length && size(rs[take]!) <= newer) newer += size(rs[take++]!) + if (take >= LAYER_FANOUT) { + inputs = rs.slice(0, take) + break + } + } + // Too many runs (small ones stranded behind a larger one, which the rule above + // never reaches; a pack merge's run, the newest when written, strands all before + // it): merge the longest stretch of adjacent runs no larger than the smallest + // that must go, so one merge brings the count back near MAX_RUNS. Four at a time + // fell behind ~3 commits a second (round 13 at 20,000 collections: 114 runs). + // A lookup checks every run, so their number is a read cost on every commit. + if (!inputs && all.length > MAX_RUNS) { + const cap = all.map(size).sort((a, b) => a - b)[all.length - MAX_RUNS]! + for (const rs of free) + for (let i = 0; i < rs.length;) { + let j = i + while (j < rs.length && size(rs[j]!) <= cap) j++ + if (j - i >= 2 && j - i > (inputs?.length ?? 0)) inputs = rs.slice(i, j) + i = j + 1 + } + // None adjacent: the LAYER_FANOUT adjacent runs with the fewest entries. + if (!inputs) { + let least = Infinity + for (const rs of free) + for (let i = 0; i + LAYER_FANOUT <= rs.length; i++) { + const n = rs.slice(i, i + LAYER_FANOUT).reduce((t, r) => t + size(r), 0) + if (n < least) { + least = n + inputs = rs.slice(i, i + LAYER_FANOUT) + } + } + } + } + if (!inputs) return false + // The newest of them: still adjacent; the next pass takes the rest. + inputs = inputs.slice(0, layoutConfig.mergeInputs) + if (split && inputs.reduce((n, r) => n + size(r), 0) > layoutConfig.mergeJobEntries) { + await split(await this.mergePlan(inputs.flatMap((r) => r.files))) + // Its inputs are a block now; other runs may still merge. + return true + } + const out = new RunWriter(this.store, Math.max(...inputs.map((r) => r.level))) + let n = 0 + // Entries of dropped packs never reach a merge: reclaim rewrites the index as it drops them. + for await (const [k, v] of mergeSorted(inputs.map((r) => runEntries(this.layers, r.files)))) { + out.add(k, v) + if (++n % 1024 === 0) await out.ready() + } + const files = await out.finish(Math.max(...inputs.map((r) => r.seq))) + return this.#swap({ + oldLayers: inputs.flatMap((r) => r.files.map((f) => f.id)), + newLayers: files, + keepSeq: true, + }) + } + + /** Merge eight small packs of a tier into one, re-pointing their records in a new layer. */ + async #mergePacks(): Promise { + const m = await this.manifests.latest() + const tiers = new Map() + for (const p of m.packs) { + if (KeyedLayout.final(p)) continue + tiers.set(p.tier, [...(tiers.get(p.tier) ?? []), p]) + } + // The tier with the most packs: the lowest that has enough never runs out while + // commits land, so higher tiers never merged (round 14 at 20,000 collections: + // 718 tier-1 packs listed and rising). + const [tier, most] = [...tiers].reduce<[number, number]>( + (best, [t, ps]) => (ps.length > best[1] ? [t, ps.length] : best), + [-1, 0], + ) + if (most < FANOUT) return false + // As many of the tier's packs as fit MERGE_PACKS_BYTES stored (at least FANOUT): many + // collections' tiny commits each write a tiny pack, and eight at a time fell behind + // (6,639 waiting at 15,000 collections, every commit reading them all). + const inputs: PackInfo[] = [] + let stored = 0 + for (const p of tiers.get(tier)!) { + if ( + inputs.length >= FANOUT && + (stored + p.bytes > MERGE_PACKS_BYTES || inputs.length >= layoutConfig.mergeInputs) + ) + break + inputs.push(p) + stored += p.bytes + } + const reader = this.packs + const streams = inputs.map((p) => + (async function* () { + for await (const { key, line } of reader.all(p.id)) yield [key, line] as [string, string] + })(), + ) + const pw = new PackWriter(this.store, { tier: tier + 1 }) + const lw = new RunWriter(this.store, 0) + let n = 0 + for await (const [key, line] of mergeSorted(streams)) { + lw.add(key, pw.append(key, line)) + if (++n % 1024 === 0) await Promise.all([pw.ready(), lw.ready()]) + } + const packs = await pw.finish() + const files = await lw.finish() + return this.#swap({ oldPacks: inputs.map((p) => p.id), newLayers: files, newPacks: packs }) + } + + /** + * Replace layers and packs in one manifest change: only if every input the + * manifest lists is still there (another job may have merged them first); + * otherwise drop what was written. New packs no merge will take are listed in + * the registry, the rest in the manifest; `unlisted` are old packs already in + * neither (reclaim took them out of the registry), retired with the rest. + */ + async #swap(o: { + oldLayers?: string[] + oldPacks?: string[] + unlisted?: string[] + newLayers?: LayerInfo[] + newPacks?: PackInfo[] + /** New layers keep their sequence (a merged run); else they're the newest (re-pointed records). */ + keepSeq?: boolean + }): Promise { + const oldLayers = new Set(o.oldLayers ?? []) + const oldPacks = new Set(o.oldPacks ?? []) + const newLayers = o.newLayers ?? [] + const newPacks = o.newPacks ?? [] + const listed = newPacks.filter((p) => !KeyedLayout.final(p)) + const registered = await this.registry.add(newPacks.filter(KeyedLayout.final)) + let lost = false + await this.manifests.update((m) => { + const haveL = new Set(m.layers.map((l) => l.id)) + const haveP = new Set(m.packs.map((p) => p.id)) + if ( + ![...oldLayers].every((id) => haveL.has(id)) || + ![...oldPacks].every((id) => haveP.has(id)) + ) { + lost = true + return null + } + const now = Date.now() + m.layers = m.layers.filter((l) => !oldLayers.has(l.id)) + m.packs = m.packs.filter((p) => !oldPacks.has(p.id)) + // A pack merge's layer is the newest: its places win over the old ones. + const seq = topSeq(m) + 1 + for (const l of newLayers) m.layers.push(o.keepSeq ? l : { ...l, seq }) + m.packs.push(...listed) + for (const id of oldLayers) + m.retired.push( + { key: layerKey(id), at: now }, + { key: layerDirKey(id), at: now }, + { key: layerFilterKey(id), at: now }, + ) + for (const id of [...oldPacks, ...(o.unlisted ?? [])]) + m.retired.push({ key: packKey(id), at: now }, { key: packDirKey(id), at: now }) + return m + }) + if (lost) { + for (const l of newLayers) + await Promise.all([ + this.store.delete(layerKey(l.id)), + this.store.delete(layerDirKey(l.id)), + this.store.delete(layerFilterKey(l.id)), + ]) + for (const p of newPacks) + await Promise.all([this.store.delete(packKey(p.id)), this.store.delete(packDirKey(p.id))]) + if (registered) await this.store.delete(registered) + return false + } + return true + } + + /** Delete retired objects past their grace, `limit` at most; false if some are left for later. */ + async #sweepRetired(limit = Infinity): Promise { + const m = await this.manifests.current(0) + const cutoff = Date.now() - layoutConfig.graceMs + let due = m.retired.filter((r) => r.at < cutoff) + if (!due.length) return true + // F: a live fork may still read what this collection merged away; those stay + // retired (swept once the fork goes) rather than deleted. + if (this.store instanceof HomeStore) { + const kept = await this.store.forkHeld() + due = due.filter((r) => !kept.has(fileId(r.key) ?? '')) + if (!due.length) return true + } + const all = due.length + due = due.slice(0, limit) + for (let i = 0; i < due.length; i += 16) + await Promise.all(due.slice(i, i + 16).map((r) => this.store.delete(r.key))) + const gone = new Set(due.map((r) => r.key)) + await this.manifests.update((n) => { + n.retired = n.retired.filter((r) => !gone.has(r.key)) + return n + }) + return due.length === all + } + + /** The whole reclaim in this process, from a mark of every live record (small stores, tests). */ + async reclaim(live: LiveSet): Promise> { + const h = new LiveHashes() + for (const k of live.records) { + const z = k.indexOf('\u0000') + h.add(k.slice(0, z), k.slice(z + 1)) + } + const plan = await this.reclaimPlan() + for (let i = 0; i < plan.ranges.length; i++) await this.reclaimRange(plan, i, h) + const swapped = await this.reclaimSwap(plan) + if (!swapped) return {} + for (let i = 0; i < swapped.rewrite.length; i += 4) + await this.reclaimRewrite(swapped.rewrite.slice(i, i + 4)) + return swapped.stats + } + + // --- Reclaim a key range at a time (the server's bounded jobs: cleanup/layout-reclaim.ts) --- + + /** + * Step 1: the index and packs as they are now, and the key ranges to work in, + * each of about `rangeEntries` index entries. Saved for the range jobs. + */ + /** Key ranges over `layers` of about `rangeEntries` entries each, cut at record ids. */ + async #cutRanges(layers: readonly LayerInfo[], rangeEntries: number): Promise { + // Cut points from the layers' page keys, sampled to keep a plan small. + const cuts: [string, number][] = [] + const total = layers.reduce((n, l) => n + l.entries, 0) + for (const l of layers) { + const { keys, perPage } = await this.layers.pageKeys(l) + const every = Math.max(1, Math.floor(keys.length / Math.max(1, (l.entries / total) * 50_000))) + for (let i = 0; i < keys.length; i += every) cuts.push([filterKey(keys[i]!), perPage * every]) + } + cuts.sort((a, b) => compareUtf8(a[0], b[0])) + const ranges: KeyRange[] = [] + let lo: string | null = null + let n = 0 + for (const [k, w] of cuts) { + if (n >= rangeEntries && (lo === null || compareUtf8(k, lo) > 0)) { + ranges.push({ lo, hi: k }) + lo = k + n = 0 + } + n += w + } + ranges.push({ lo, hi: null }) + return ranges + } + + // --- A merge split into key ranges (the server's `layout.merge` jobs) --- + + /** Plan a merge of `layers` (adjacent runs' files) as key-range jobs. */ + async mergePlan( + layers: LayerInfo[], + rangeEntries = layoutConfig.mergeJobEntries, + ): Promise { + const plan: MergePlan = { + id: newId(), + layers, + ranges: await this.#cutRanges(layers, rangeEntries), + level: Math.max(0, ...layers.map((l) => l.level)), + seq: Math.max(...layers.map((l) => l.seq)), + at: Date.now(), + } + await this.store.put(mergeKey(plan.id, 'plan'), JSON.stringify(plan)) + return plan + } + + async mergePlanById(id: string): Promise { + const o = await this.store.get(mergeKey(id, 'plan')) + return o ? (JSON.parse(await o.text()) as MergePlan) : null + } + + /** One range of a split merge: its part of the merged run. Whether every range is now done. */ + async mergeRange(plan: MergePlan, i: number): Promise { + // Its inputs merged elsewhere (another maintain, which only a broken lease allows): + // nothing to swap in, and waiting for it to be abandoned would hold up merges. + const have = new Set((await this.manifests.latest()).layers.map((l) => l.id)) + if (!plan.layers.every((l) => have.has(l.id))) { + await this.#dropMerge(plan, true) + return false + } + const r = plan.ranges[i]! + const out = new RunWriter(this.store, plan.level) + const inputs = runs(plan.layers).sort((a, b) => b.seq - a.seq) + let n = 0 + for await (const [k, v] of mergeSorted( + inputs.map((x) => runEntries(this.layers, x.files, r.lo ?? undefined, r.hi ?? undefined)), + )) { + out.add(k, v) + if (++n % 1024 === 0) await out.ready() + } + const files = await out.finish(plan.seq) + await this.store.put( + mergeKey(plan.id, `range-${String(i).padStart(6, '0')}`), + JSON.stringify(files), + ) + return (await this.#listed(mergeKey(plan.id, 'range-'))).length >= plan.ranges.length + } + + /** + * Swap a split merge's run in for its inputs, once (the first caller claims it). + * False when another caller has it, or its inputs went meanwhile (its output is deleted). + */ + async mergeSwap(plan: MergePlan): Promise { + const token = newId() + await this.store.put(mergeKey(plan.id, 'swap'), token, { ifAbsent: true }) + if ((await (await this.store.get(mergeKey(plan.id, 'swap')))?.text()) !== token) return false + const files: LayerInfo[] = [] + for (let i = 0; i < plan.ranges.length; i++) { + const o = await this.store.get(mergeKey(plan.id, `range-${String(i).padStart(6, '0')}`)) + if (!o) { + await this.#dropMerge(plan, false) + return false + } + files.push(...(JSON.parse(await o.text()) as LayerInfo[])) + } + const level = Math.max(plan.level, levelFor(files.reduce((n, f) => n + f.entries, 0))) + const swapped = await this.#swap({ + oldLayers: plan.layers.map((l) => l.id), + newLayers: files.map((f) => ({ ...f, level, seq: plan.seq, run: plan.id })), + keepSeq: true, + }) + await this.#dropMerge(plan, false) + await this.manifests.settle() + return swapped + } + + async #listed(prefix: string): Promise { + const keys: string[] = [] + let cursor: string | undefined + do { + const page = await this.store.list(prefix, cursor) + keys.push(...page.keys) + cursor = page.cursor + } while (cursor) + return keys + } + + /** The split merges under way; one past MERGE_ABANDON_MS is abandoned and cleared. */ + async #mergesInFlight(): Promise { + const out: MergePlan[] = [] + for (const key of await this.#listed('merges/')) { + if (!key.endsWith('/plan')) continue + const plan = await this.mergePlanById(key.slice('merges/'.length, -'/plan'.length)) + if (!plan) continue + if (Date.now() - plan.at > MERGE_ABANDON_MS) await this.#dropMerge(plan, true) + else out.push(plan) + } + return out + } + + /** A split merge's objects gone; with `outputs`, the layer files its ranges wrote too. */ + async #dropMerge(plan: MergePlan, outputs: boolean) { + const keys = [mergeKey(plan.id, 'plan')] + for (let i = 0; i < plan.ranges.length; i++) { + const key = mergeKey(plan.id, `range-${String(i).padStart(6, '0')}`) + keys.push(key) + if (!outputs) continue + const o = await this.store.get(key) + if (o) + for (const f of JSON.parse(await o.text()) as LayerInfo[]) + keys.push(layerKey(f.id), layerDirKey(f.id), layerFilterKey(f.id)) + } + for (let i = 0; i < keys.length; i += 16) + await Promise.all(keys.slice(i, i + 16).map((k) => this.store.delete(k))) + await this.store.delete(mergeKey(plan.id, 'swap')) + } + + async reclaimPlan(rangeEntries = RECLAIM_RANGE_ENTRIES): Promise { + const m = await this.manifests.latest() + const ranges = await this.#cutRanges(m.layers, rangeEntries) + const registry = await this.registry.read() + const plan: ReclaimPlan = { + id: newId(), + layers: m.layers, + packs: [...m.packs, ...registry.packs].map((p) => ({ id: p.id, records: p.records })), + registry: registry.keys, + ranges, + level: Math.max(0, ...m.layers.map((l) => l.level)), + // Inherited layers (a fork's) number below zero. + seq: m.layers.length ? Math.max(...m.layers.map((l) => l.seq)) : 0, + } + await this.store.put(reclaimKey(plan.id, 'plan'), JSON.stringify(plan)) + return plan + } + + async reclaimPlanById(id: string): Promise { + const o = await this.store.get(reclaimKey(id, 'plan')) + if (!o) throw new Error(`No reclaim plan ${id}`) + return JSON.parse(await o.text()) as ReclaimPlan + } + + /** + * Step 2, one job a range: the plan's index entries in the range that `live` + * holds, written as files of the range's part of one new run, with live + * records counted per pack. Returns whether every range is now done. + */ + async reclaimRange(plan: ReclaimPlan, i: number, live: LiveHashes): Promise { + const r = plan.ranges[i]! + const from = r.lo ?? undefined + const to = r.hi ?? undefined + const packs = new Set(plan.packs.map((p) => p.id)) + const perPack: Record = {} + let kept = 0 + let dropped = 0 + const out = new RunWriter(this.store, plan.level) + const inputs = runs(plan.layers).sort((a, b) => b.seq - a.seq) + for await (const [k, v] of mergeSorted( + inputs.map((x) => runEntries(this.layers, x.files, from, to)), + )) { + const pack = parseLoc(v).pack + const z = k.lastIndexOf('\u0000') + if (!packs.has(pack) || !live.has(k.slice(0, z), k.slice(z + 1))) { + dropped++ + continue + } + out.add(k, v) + perPack[pack] = (perPack[pack] ?? 0) + 1 + if (++kept % 1024 === 0) await out.ready() + } + const files = await out.finish(plan.seq) + const result: RangeResult = { files, perPack, kept, dropped } + await this.store.put( + reclaimKey(plan.id, `range-${String(i).padStart(6, '0')}`), + JSON.stringify(result), + ) + let done = 0 + let cursor: string | undefined + do { + const page = await this.store.list(reclaimKey(plan.id, 'range-'), cursor) + done += page.keys.length + cursor = page.cursor + } while (cursor) + return done >= plan.ranges.length + } + + /** + * Step 3, once every range is done (and only once: the first caller claims it): + * swap the ranges' run in for the plan's layers, drop packs with nothing live, + * and return the packs under half live, for rewrite jobs. Null when another + * caller has it, or a merge replaced the plan's layers meanwhile (nothing changes). + */ + async reclaimSwap( + plan: ReclaimPlan, + ): Promise<{ rewrite: string[]; stats: Record } | null> { + const token = newId() + await this.store.put(reclaimKey(plan.id, 'swap'), token, { ifAbsent: true }) + if ((await (await this.store.get(reclaimKey(plan.id, 'swap')))?.text()) !== token) return null + const files: LayerInfo[] = [] + const live = new Map() + const stats = { + ranges: plan.ranges.length, + entriesKept: 0, + entriesDropped: 0, + packs: plan.packs.length, + packsDropped: 0, + packsRewritten: 0, + recordsDropped: 0, + } + for (let i = 0; i < plan.ranges.length; i++) { + const o = await this.store.get(reclaimKey(plan.id, `range-${String(i).padStart(6, '0')}`)) + // Gone: another job swapped this plan in and dropped it (and the claim this one just made again). + if (!o) { + await this.store.delete(reclaimKey(plan.id, 'swap')) + return null + } + const r = JSON.parse(await o.text()) as RangeResult + files.push(...r.files.map((f) => ({ ...f, seq: plan.seq, run: plan.id }))) + for (const [p, n] of Object.entries(r.perPack)) live.set(p, (live.get(p) ?? 0) + n) + stats.entriesKept += r.kept + stats.entriesDropped += r.dropped + } + const drop: string[] = [] + const rewrite: string[] = [] + for (const p of plan.packs) { + const n = live.get(p.id) ?? 0 + if (n === 0) drop.push(p.id) + else if (n < p.records / 2) rewrite.push(p.id) + stats.recordsDropped += p.records - n + } + stats.packsDropped = drop.length + stats.packsRewritten = rewrite.length + // One run across the ranges, at the level its whole size puts it. + const level = Math.max(plan.level, levelFor(stats.entriesKept)) + for (const f of files) f.level = level + const m = await this.manifests.latest() + const listed = new Set(m.packs.map((p) => p.id)) + const swapped = await this.#swap({ + oldLayers: plan.layers.map((l) => l.id), + oldPacks: drop.filter((id) => listed.has(id)), + unlisted: drop.filter((id) => !listed.has(id)), + newLayers: files, + keepSeq: true, + }) + if (swapped) { + // The registry's objects as one, without the dropped packs or the ones being rewritten + // (their rewrite lists what it writes). + const gone = new Set([...drop, ...rewrite]) + const kept: PackInfo[] = [] + for (const key of plan.registry) { + const o = await this.store.get(key) + if (o) + kept.push(...(JSON.parse(await o.text()) as PackInfo[]).filter((p) => !gone.has(p.id))) + } + await this.registry.replace(plan.registry, kept) + } + await this.#dropPlan(plan) + await this.manifests.settle() + return swapped ? { rewrite, stats } : null + } + + async #dropPlan(plan: ReclaimPlan) { + const keys = [reclaimKey(plan.id, 'plan')] + for (let i = 0; i < plan.ranges.length; i++) + keys.push(reclaimKey(plan.id, `range-${String(i).padStart(6, '0')}`)) + for (let i = 0; i < keys.length; i += 16) + await Promise.all(keys.slice(i, i + 16).map((k) => this.store.delete(k))) + // The claim last: a late caller finds it, or finds the range results gone. + await this.store.delete(reclaimKey(plan.id, 'swap')) + } + + /** + * Step 4, a few packs a job: copy their live records into new packs and point + * the index at them with a new, newest layer. A record is live when the index + * still points at this copy of it. + */ + async reclaimRewrite(packIds: readonly string[]): Promise> { + const m = await this.manifests.latest() + // Packs the manifest lists must still be there; the others reclaim took out of the registry. + const listed = new Set(m.packs.map((p) => p.id)) + const retired = new Set(m.retired.map((r) => r.key)) + const ids = packIds.filter((id) => !retired.has(packKey(id))) + if (!ids.length) return {} + const pw = new PackWriter(this.store, { tier: 0 }) + const moved: [string, string][] = [] + let kept = 0 + let dropped = 0 + for (const id of ids) { + let batch: { key: string; line: string; place: string }[] = [] + const flush = async () => { + const found = await this.index.lookup(m, [...new Set(batch.map((b) => filterKey(b.key)))]) + for (const b of batch) { + const v = found.get(b.key) + if (v === undefined || v !== b.place) { + dropped++ + continue + } + moved.push([b.key, pw.append(b.key, b.line)]) + kept++ + } + batch = [] + await pw.ready() + } + for await (const x of this.packs.all(id)) { + batch.push(x) + if (batch.length === 1024) await flush() + } + await flush() + } + const packs = await pw.finish() + moved.sort((a, b) => compareUtf8(a[0], b[0])) + const lw = new RunWriter(this.store, 0) + for (const [k, v] of moved) lw.add(k, v) + const files = await lw.finish() + await this.#swap({ + oldPacks: ids.filter((id) => listed.has(id)), + unlisted: ids.filter((id) => !listed.has(id)), + newLayers: files, + newPacks: packs, + }) + await this.manifests.settle() + return { packsRead: ids.length, recordsKept: kept, recordsDropped: dropped } + } + + /** + * A fork lists the parent's runs and packs as they are, and reads them where they + * live; nothing is copied. + */ + async forkInto(target: RecordStore): Promise> { + if (!(target instanceof KeyedLayout)) throw new Error('forkInto: not a keyed layout') + return this.#forkByReference(target) + } + + async #forkByReference(target: KeyedLayout): Promise> { + const from = this.store + const to = target.store + if (!(from instanceof HomeStore) || !(to instanceof HomeStore)) + throw new Error('forkInto: F needs both collections in their own prefixes') + const m = await this.manifests.latest() + const registered = (await this.registry.read()).packs + // Each file's home: where the parent read it from (its own parent's), else the parent. + const mine = await from.inherited() + const inherited: Inherited = { packs: {}, layers: {} } + for (const p of [...m.packs, ...registered]) + inherited.packs[p.id] = mine.packs[p.id] ?? from.home + for (const l of m.layers) inherited.layers[l.id] = mine.layers[l.id] ?? from.home + const homes = new Set([...Object.values(inherited.packs), ...Object.values(inherited.layers)]) + // Every home first learns of the fork: from then on it keeps what the fork lists. + for (const h of homes) + await from.root.put(forkMarkerKey(h, to.home), JSON.stringify({ at: Date.now() }), { + ifAbsent: true, + }) + const body = JSON.stringify(inherited) + await to.put(INHERITED_KEY, body) + to.forget() + await target.registry.add(registered) + // The parent's layers keep their order but number below anything the fork writes. + const top = Math.max(0, ...m.layers.map((l) => l.seq)) + await target.manifests.update((n) => { + n.packs.push(...m.packs) + n.layers.push(...m.layers.map((l) => ({ ...l, seq: l.seq - top - 1 }))) + return n + }) + await target.manifests.settle() + return { objects: 1 + homes.size, bytes: body.length } + } + + /** + * F: every file a copy of this collection needs, keyed from the location's root + * (`runs//…`, and for a fork the files it reads from its parent's + * prefix, at their own keys), and the manifest to write with them (pending runs + * folded in). A mirror copies these as they are: nothing is rebuilt. + */ + async mirrorFiles(): Promise<{ + /** `optional`: a file older runs have and newer ones don't (a filter, now in the directory). */ + files: { key: string; bytes?: number; optional?: boolean }[] + manifest: Manifest + /** Where the copy's manifest goes: its own number, and the head. */ + manifestKeys: { numbered: string; head: string } + }> { + const home = this.store + if (!(home instanceof HomeStore)) throw new Error('mirrorFiles: layout F only') + const m = await this.manifests.latest() + const registry = await this.registry.read() + const inh = await home.inherited() + const at = (h: string | undefined, key: string) => `runs/${h ?? home.home}/${key}` + const files: { key: string; bytes?: number; optional?: boolean }[] = [] + for (const l of m.layers) { + const h = inh.layers[l.id] + files.push( + { key: at(h, layerKey(l.id)), bytes: l.bytes }, + { key: at(h, layerDirKey(l.id)) }, + { key: at(h, layerFilterKey(l.id)), optional: true }, + ) + } + for (const p of [...m.packs, ...registry.packs]) { + const h = inh.packs[p.id] + files.push({ key: at(h, packKey(p.id)), bytes: p.bytes }, { key: at(h, packDirKey(p.id)) }) + } + for (const k of registry.keys) files.push({ key: at(undefined, k) }) + if (Object.keys(inh.packs).length || Object.keys(inh.layers).length) + files.push({ key: at(undefined, 'inherited.json') }) + const dir = `runs/${home.home}/manifest/${this.name}` + return { + files, + manifest: { ...m, retired: [], folded: [] }, + manifestKeys: { + numbered: `${dir}/${String(m.seq).padStart(12, '0')}.json`, + head: `${dir}/head.json`, + }, + } + } + + /** + * F: delete what a deleted collection wrote, but for files a live fork still + * reads (they go when that fork does); then the same for each collection it + * read from that is gone too, whose forks now hold less. + */ + static async dropCollection( + root: Store, + id: string, + deleted: (collectionId: string) => Promise, + ): Promise<{ objects: number; kept: number }> { + const home = new HomeStore(root, id) + const mine = await home.inherited() + const homes = new Set([...Object.values(mine.packs), ...Object.values(mine.layers)]) + for (const h of homes) await root.delete(forkMarkerKey(h, id)) + const held = await home.forkHeld() + let objects = 0 + let kept = 0 + let cursor: string | undefined + do { + const page = await root.list(`runs/${id}/`, cursor) + const drop = page.keys.filter((k) => { + const rel = k.slice(`runs/${id}/`.length) + // A fork's marker stays while that fork lives; so does what it reads. + const keep = rel.startsWith('forks/') || held.has(fileId(rel) ?? '') + if (keep) kept++ + return !keep + }) + for (let i = 0; i < drop.length; i += 16) + await Promise.all(drop.slice(i, i + 16).map((k) => root.delete(k))) + objects += drop.length + cursor = page.cursor + } while (cursor) + for (const h of homes) + if (await deleted(h)) { + const r = await KeyedLayout.dropCollection(root, h, deleted) + objects += r.objects + kept += r.kept + } + return { objects, kept } + } +} + +/** F: the files a fork read from other collections at fork time, by id: where each lives. */ +export interface Inherited { + packs: Record + layers: Record +} + +const INHERITED_KEY = 'inherited.json' +const forkMarkerKey = (home: string, fork: string) => `runs/${home}/forks/${fork}.json` + +/** The id a pack or index file's key names (`packs/.pk`, `layers/.bf`, …), else null. */ +function fileId(key: string): string | null { + const m = /^(?:packs|layers)\/([^./]+)\./.exec(key) + return m ? m[1]! : null +} + +/** + * F: a collection's own prefix, through which it also reads the files it + * inherited from the collections they live with (a fork's parent, or further + * back). Writes and deletes only ever touch its own prefix: a fork can't delete + * what its parent still uses. + */ +export class HomeStore extends PrefixedStore { + #inherited: { v: Promise; at: number } | null = null + + constructor( + readonly root: Store, + readonly home: string, + ) { + super(root, `runs/${home}`) + } + + /** The files this collection inherited (read once; again after a miss, at most every 2 s). */ + inherited(): Promise { + if (!this.#inherited) { + const v = super + .get(INHERITED_KEY) + .then(async (o) => + o ? (JSON.parse(await o.text()) as Inherited) : { packs: {}, layers: {} }, + ) + this.#inherited = { v, at: Date.now() } + } + return this.#inherited.v + } + + /** Read the inherited files again (a fork just made, in this isolate). */ + forget(): void { + this.#inherited = null + } + + /** Every file of this collection a live fork reads (a fork's listing names its home). */ + async forkHeld(): Promise> { + const held = new Set() + let cursor: string | undefined + do { + const page = await super.list('forks/', cursor) + for (const k of page.keys) { + const fork = k.slice('forks/'.length, -'.json'.length) + const o = await this.root.get(`runs/${fork}/${INHERITED_KEY}`) + if (!o) continue + const inh = JSON.parse(await o.text()) as Inherited + for (const [id, h] of [...Object.entries(inh.packs), ...Object.entries(inh.layers)]) + if (h === this.home) held.add(id) + } + cursor = page.cursor + } while (cursor) + return held + } + + async #owner(key: string): Promise { + const id = fileId(key) + if (!id) return null + const inh = await this.inherited() + return inh.packs[id] ?? inh.layers[id] ?? null + } + + override async get(key: string, range?: { offset: number; length?: number }) { + const owner = await this.#owner(key) + if (owner) return this.root.get(`runs/${owner}/${key}`, range) + const o = await super.get(key, range) + // Not here and not known as inherited: a fork made after this isolate read its list. + if (!o && fileId(key) && this.#inherited && Date.now() - this.#inherited.at > 2000) { + this.forget() + const again = await this.#owner(key) + if (again) return this.root.get(`runs/${again}/${key}`, range) + } + return o + } + + override async head(key: string) { + const owner = await this.#owner(key) + return owner ? this.root.head(`runs/${owner}/${key}`) : super.head(key) + } + + override async delete(key: string) { + // Another collection's file: never deleted from here. + if (await this.#owner(key)) return + return super.delete(key) + } +} + +class KeyedWriter implements RecordWriter { + readonly #packs: PackWriter + #entries: [string, string][] = [] + + constructor(readonly layout: KeyedLayout) { + this.#packs = new PackWriter(layout.store) + } + + add(entry: RecordEntry, line: string): string { + const k = recordKey(entry.key, entry.hash) + const at = this.#packs.append(k, line) + this.#entries.push([k, at]) + return at + } + + async finish(opts: { hold?: boolean } = {}): Promise { + const packs = await this.#packs.finish() + if (!this.#entries.length) return null + this.#entries.sort((a, b) => compareUtf8(a[0], b[0])) + const lw = new RunWriter(this.layout.store, 0) + for (const [k, v] of this.#entries) lw.add(k, v) + this.#entries = [] + const files = await lw.finish() + // Listed before the manifest names their index: a pack the manifest never got + // (a failed update) is one reclaim finds with nothing live, and drops. + await this.layout.registry.add(packs.filter(KeyedLayout.final)) + const run = { layers: files, packs: packs.filter((p) => !KeyedLayout.final(p)) } + // Held: the caller names it later, with the rest of its commit (appendRuns). + if (opts.hold) return run + await this.layout.manifests.append(run.layers, run.packs) + // Merges run after the commit, as their own job (the server's layout.maintain). + return null + } +} diff --git a/packages/protocol/src/repo/layouts/kit.ts b/packages/protocol/src/repo/layouts/kit.ts new file mode 100644 index 00000000..e5119c79 --- /dev/null +++ b/packages/protocol/src/repo/layouts/kit.ts @@ -0,0 +1,1168 @@ +/** + * Storage pieces of layout F, records in packs found through an index + * (record-storage-layout.md). Nothing here changes a hash. + * + * packs records in blocks of ~64 KB, each block its own gzip member, so one + * record costs a range read of one block. A pack is two objects: its + * blocks, and a directory of block offsets written when it closes. + * Records go in key order, each line `\t` (JSON has + * no raw tab), so packs can be merged as streams. + * layers sorted key/value runs in pages of ~16 KB (gzip each), with a + * directory of each page's first key and place. An index is a stack + * of layers, newest first, merged eight at a time. + * manifest the layers and packs in use: one immutable object per change, + * claimed by writing the next sequence number, plus a head hint. + */ +import { compareUtf8 } from '../../utf8.js' +import { gunzipText, gzip } from '../gzip.js' +import type { Lru } from '../lru.js' +import { putJoined, type Store } from '../types.js' + +export const BLOCK_RAW = 64 * 1024 +export const PACK_RAW = 16 * 1024 * 1024 +export const PAGE_RAW = 16 * 1024 +/** Small packs of one tier that trigger a merge. */ +export const FANOUT = 8 +/** Runs of one index level that trigger a merge into the next (fewer layers for a lookup to check). */ +export const LAYER_FANOUT = 4 +/** Entries a level-0 run holds at most; each level up holds LAYER_FANOUT times more. */ +export const LEVEL_BASE = 1024 + +/** + * The level a run of `entries` belongs at (size-tiered): a run lands among runs + * its size, so four small commits merge with each other, not into a large run + * (an initial load's) that happens to share their level. + */ +export const levelFor = (entries: number) => + entries <= LEVEL_BASE + ? 0 + : Math.ceil(Math.log(entries / LEVEL_BASE) / Math.log(LAYER_FANOUT) - 1e-9) +/** Raw bytes of one file of a merged run: a merge holds one file's pages, never the level's. */ +export const RUN_FILE_RAW = 16 * 1024 * 1024 +/** Bytes a merge reads from each input at a time. */ +const MERGE_READ = 1024 * 1024 + +export const newId = () => crypto.randomUUID().replace(/-/g, '') + +/** A record's place: pack, block, line in the block. */ +export const loc = (pack: string, block: number, line: number) => `${pack}.${block}.${line}` +export const parseLoc = (s: string) => { + const [pack, block, line] = s.split('.') + return { pack: pack!, block: Number(block), line: Number(line) } +} + +// --- Packs ---------------------------------------------------------------------------- + +export interface PackInfo { + id: string + /** Size tier: 0 for a commit's packs, higher for merged ones. */ + tier: number + records: number + /** Stored (compressed) bytes, and raw. */ + bytes: number + raw: number +} + +export class PackWriter { + #id = '' + #blocks: Promise[] = [] + #lines: string[] = [] + #raw = 0 + #packRaw = 0 + #records = 0 + #lastKey: string | null = null + readonly done: PackInfo[] = [] + readonly #writing = new Set>() + #error: unknown = null + + constructor( + readonly store: Store, + readonly opts: { tier?: number } = {}, + ) {} + + /** Add a record line under `key`; returns its place. A key below the last starts a new pack. */ + append(key: string, line: string): string { + if (this.#lastKey !== null && compareUtf8(key, this.#lastKey) < 0) this.#close() + this.#lastKey = key + if (!this.#id) this.#id = newId() + this.#lines.push(`${key}\t${line}`) + const at = loc(this.#id, this.#blocks.length, this.#lines.length - 1) + this.#raw += key.length + line.length + 2 + this.#records++ + if (this.#raw >= BLOCK_RAW) this.#seal() + if (this.#packRaw >= PACK_RAW) this.#close() + return at + } + + get pending(): number { + return this.#writing.size + } + + /** + * Wait while the writer holds too much: a pack still writing beside the one + * filling, or more than 2 MB of blocks compressing. Merges call it between + * records, so their output streams out instead of piling up in memory. + */ + async ready(): Promise { + while (this.#writing.size > 1) await Promise.race(this.#writing) + if (this.#blocks.length > 32) await Promise.all(this.#blocks) + if (this.#error) throw this.#error + } + + #seal() { + if (!this.#lines.length) return + this.#blocks.push(gzip(this.#lines.join('\n') + '\n')) + this.#packRaw += this.#raw + this.#lines = [] + this.#raw = 0 + } + + #close() { + this.#seal() + this.#lastKey = null + if (!this.#blocks.length) return + const job = { id: this.#id, blocks: this.#blocks, records: this.#records, raw: this.#packRaw } + this.#id = '' + this.#blocks = [] + this.#records = 0 + this.#packRaw = 0 + const p = this.#write(job) + .catch((err) => { + this.#error ??= err + }) + .finally(() => this.#writing.delete(p)) + this.#writing.add(p) + } + + async #write(job: { id: string; blocks: Promise[]; records: number; raw: number }) { + const parts = await Promise.all(job.blocks) + const offsets = [0] + for (const p of parts) offsets.push(offsets.at(-1)! + p.byteLength) + await putJoined(this.store, packKey(job.id), parts, { ifAbsent: true }) + await this.store.put(packDirKey(job.id), JSON.stringify(offsets), { ifAbsent: true }) + this.done.push({ + id: job.id, + tier: this.opts.tier ?? 0, + records: job.records, + bytes: offsets.at(-1)!, + raw: job.raw, + }) + } + + /** Close the current pack and wait for every write. */ + async finish(): Promise { + this.#close() + while (this.#writing.size) await Promise.race(this.#writing) + if (this.#error) throw this.#error + return this.done + } +} + +export const packKey = (id: string) => `packs/${id}.pk` +export const packDirKey = (id: string) => `packs/${id}.dir` + +/** Reads blocks of packs, through the repository's memory cache. */ +export class PackReader { + constructor( + readonly store: Store, + readonly lru: Lru, + readonly scope: string, + ) {} + + async #dir(pack: string): Promise { + const k = `${this.scope}/packdir:${pack}` + const hit = this.lru.get(k) + if (hit) return (hit as { v: number[] }).v + const o = await this.store.get(packDirKey(pack)) + if (!o) throw new Error(`Missing pack directory ${pack}`) + const v = JSON.parse(await o.text()) as number[] + this.lru.set(k, { v, approxBytes: v.length * 16 }) + return v + } + + /** Every line of one block. */ + async block(pack: string, block: number): Promise { + const k = `${this.scope}/block:${pack}.${block}` + const hit = this.lru.get(k) + if (hit) return (hit as { v: string[] }).v + const dir = await this.#dir(pack) + const from = dir[block] + const to = dir[block + 1] + if (from === undefined || to === undefined) throw new Error(`No block ${block} in pack ${pack}`) + const o = await this.store.get(packKey(pack), { offset: from, length: to - from }) + if (!o) throw new Error(`Missing pack ${pack}`) + const text = await gunzipText(await o.bytes()) + const v = text.split('\n') + v.pop() + for (let i = 0; i < v.length; i++) v[i] = v[i]!.slice(v[i]!.indexOf('\t') + 1) + this.lru.set(k, { v, approxBytes: text.length * 2 }) + return v + } + + /** Lines at many places, read a block once each, `concurrency` blocks at a time. */ + /** + * Lines at many places. Blocks are read in runs: neighbouring blocks of one pack + * (gaps of up to two) come in one range request, as a sorted pack's leaf does. + */ + async lines(places: readonly string[], concurrency = 16): Promise { + const parsed = places.map(parseLoc) + const got = new Map() + const need = new Map>() + for (const p of parsed) { + const k = `${p.pack}.${p.block}` + if (got.has(k)) continue + const hit = this.lru.get(`${this.scope}/block:${k}`) + if (hit) got.set(k, (hit as { v: string[] }).v) + else need.set(p.pack, (need.get(p.pack) ?? new Set()).add(p.block)) + } + const runs: { pack: string; from: number; to: number }[] = [] + for (const [pack, set] of need) { + const blocks = [...set].sort((x, y) => x - y) + for (let i = 0; i < blocks.length;) { + let j = i + while ( + j + 1 < blocks.length && + blocks[j + 1]! - blocks[j]! <= 3 && + blocks[j + 1]! - blocks[i]! < 256 + ) + j++ + runs.push({ pack, from: blocks[i]!, to: blocks[j]! }) + i = j + 1 + } + } + for (let i = 0; i < runs.length; i += concurrency) + await Promise.all(runs.slice(i, i + concurrency).map((r) => this.#run(r, got))) + return parsed.map((p) => { + const line = got.get(`${p.pack}.${p.block}`)?.[p.line] + if (line === undefined) + throw new Error(`No line ${p.line} in block ${p.block} of pack ${p.pack}`) + return line + }) + } + + /** Blocks `from` through `to` of a pack, in one range read. */ + async #run(r: { pack: string; from: number; to: number }, got: Map) { + const dir = await this.#dir(r.pack) + const start = dir[r.from]! + const o = await this.store.get(packKey(r.pack), { + offset: start, + length: dir[r.to + 1]! - start, + }) + if (!o) throw new Error(`Missing pack ${r.pack}`) + const bytes = await o.bytes() + for (let b = r.from; b <= r.to; b++) { + const text = await gunzipText(bytes.subarray(dir[b]! - start, dir[b + 1]! - start)) + const v = text.split('\n') + v.pop() + for (let i = 0; i < v.length; i++) v[i] = v[i]!.slice(v[i]!.indexOf('\t') + 1) + got.set(`${r.pack}.${b}`, v) + this.lru.set(`${this.scope}/block:${r.pack}.${b}`, { v, approxBytes: text.length * 2 }) + } + } + + /** A whole pack's records in order, with their keys, read 8 MB at a time (merges and reclaims). */ + async *all(pack: string): AsyncGenerator<{ key: string; line: string; place: string }> { + const dir = await this.#dir(pack) + for (let b = 0; b < dir.length - 1;) { + let e = b + while (e + 1 < dir.length - 1 && dir[e + 2]! - dir[b]! <= MERGE_READ) e++ + const o = await this.store.get(packKey(pack), { + offset: dir[b]!, + length: dir[e + 1]! - dir[b]!, + }) + if (!o) throw new Error(`Missing pack ${pack}`) + const bytes = await o.bytes() + for (let k = b; k <= e; k++) { + const lines = ( + await gunzipText(bytes.subarray(dir[k]! - dir[b]!, dir[k + 1]! - dir[b]!)) + ).split('\n') + lines.pop() + for (let i = 0; i < lines.length; i++) { + const l = lines[i]! + const t = l.indexOf('\t') + yield { key: l.slice(0, t), line: l.slice(t + 1), place: loc(pack, k, i) } + } + } + b = e + 1 + } + } +} + +// --- Layers --------------------------------------------------------------------------- + +export interface LayerInfo { + id: string + level: number + /** Manifest sequence it was added in: newer layers win. */ + seq: number + entries: number + bytes: number + min: string + max: string + /** + * The run it belongs to: a merge writes one run as several files in key order, + * each with its own directory and filter. Absent: a run of one file. + */ + run?: string +} + +export const runOf = (l: LayerInfo) => l.run ?? l.id + +/** Layers grouped into runs: files in key order, the run's newest sequence, its level. */ +export function runs( + layers: readonly LayerInfo[], +): { id: string; level: number; seq: number; files: LayerInfo[] }[] { + const by = new Map() + for (const l of layers) by.set(runOf(l), [...(by.get(runOf(l)) ?? []), l]) + return [...by].map(([id, files]) => ({ + id, + level: files[0]!.level, + seq: Math.max(...files.map((f) => f.seq)), + files: files.sort((a, b) => compareUtf8(a.min, b.min)), + })) +} + +export const layerKey = (id: string) => `layers/${id}.ly` +export const layerDirKey = (id: string) => `layers/${id}.dir` +export const layerFilterKey = (id: string) => `layers/${id}.bf` + +/** What a layer's filter holds for a key: its id part (before \u0000), or the whole key. */ +export const filterKey = (k: string) => { + const z = k.indexOf('\u0000') + return z < 0 ? k : k.slice(0, z) +} + +/** Two 32-bit hashes of a string (FNV-1a with two seeds), for a Bloom filter. */ +function hashes(s: string): [number, number] { + let a = 2_166_136_261 + let b = 0x9747b28c + for (let i = 0; i < s.length; i++) { + const c = s.charCodeAt(i) + a = Math.imul(a ^ c, 16_777_619) + b = Math.imul(b ^ c, 0x5bd1e995) + } + return [a >>> 0, (b | 1) >>> 0] +} +const FILTER_BITS = 10 +const FILTER_K = 7 + +/** A Bloom filter of `keys` (~1% false positives). */ +export function buildFilter(keys: Iterable, n: number): Uint8Array { + const h = new FilterHashes() + for (const k of keys) h.add(k) + return h.build(n) +} + +/** + * A filter's keys as hash pairs, 8 bytes each, until it's built. Holding the keys + * themselves would hold far more: a key sliced from a block's text keeps the + * whole block alive, so a merge's filter kept every input block it had read. + */ +export class FilterHashes { + #h = new Uint32Array(1024) + size = 0 + + add(k: string): void { + if (this.size * 2 === this.#h.length) { + const grown = new Uint32Array(this.#h.length * 2) + grown.set(this.#h) + this.#h = grown + } + const [h1, h2] = hashes(k) + this.#h[this.size * 2] = h1 + this.#h[this.size * 2 + 1] = h2 + this.size++ + } + + build(n = this.size): Uint8Array { + const m = Math.max(1024, n * FILTER_BITS) + const bits = new Uint8Array(Math.ceil(m / 8) + 4) + new DataView(bits.buffer).setUint32(0, m) + for (let j = 0; j < this.size; j++) { + const h1 = this.#h[j * 2]! + const h2 = this.#h[j * 2 + 1]! + for (let i = 0; i < FILTER_K; i++) { + const bit = (h1 + i * h2) % m + bits[4 + (bit >>> 3)]! |= 1 << (bit & 7) + } + } + return bits + } +} + +/** A copy of `s` that holds no larger string it was sliced from (V8 keeps the parent of a slice). */ +export const detached = (s: string) => (' ' + s).slice(1) + +export function filterHas(filter: Uint8Array, k: string): boolean { + const m = new DataView(filter.buffer, filter.byteOffset).getUint32(0) + const [h1, h2] = hashes(k) + for (let i = 0; i < FILTER_K; i++) { + const bit = (h1 + i * h2) % m + if (!(filter[4 + (bit >>> 3)]! & (1 << (bit & 7)))) return false + } + return true +} + +interface LayerDir { + first: string[] + off: number[] +} + +export class LayerWriter { + readonly id = newId() + #pages: Promise[] = [] + #first: string[] = [] + #lines: string[] = [] + #raw = 0 + entries = 0 + #min: string | null = null + #last: string | null = null + /** Distinct filter keys' hashes, for the layer's Bloom filter (keys come sorted, so repeats are adjacent). */ + readonly #filtered = new FilterHashes() + #lastFiltered: string | null = null + + constructor( + readonly store: Store, + readonly level: number, + ) {} + + /** Add in key order (a repeated key keeps the first). */ + add(key: string, value: string): void { + if (this.#last !== null) { + const c = compareUtf8(key, this.#last) + if (c < 0) throw new Error(`Layer keys out of order: ${key} after ${this.#last}`) + if (c === 0) return + } + if (!this.#lines.length) this.#first.push(detached(key)) + this.#min ??= detached(key) + this.#last = key + this.#lines.push(`${key}\t${value}`) + const f = filterKey(key) + if (f !== this.#lastFiltered) { + this.#filtered.add(f) + this.#lastFiltered = f + } + this.#raw += key.length + value.length + 2 + this.entries++ + if (this.#raw >= PAGE_RAW) this.#seal() + } + + #seal() { + if (!this.#lines.length) return + this.#pages.push(gzip(this.#lines.join('\n') + '\n')) + this.#lines = [] + this.#raw = 0 + } + + async finish(seq = 0): Promise { + this.#seal() + if (!this.entries) return null + const pages = await Promise.all(this.#pages) + const off = [0] + for (const p of pages) off.push(off.at(-1)! + p.byteLength) + await putJoined(this.store, layerKey(this.id), pages, { ifAbsent: true }) + // The directory and the Bloom filter in one object: a cold lookup reads one, not two. + const dir = new TextEncoder().encode( + JSON.stringify({ first: this.#first, off } satisfies LayerDir), + ) + const filter = this.#filtered.build() + const both = new Uint8Array(4 + dir.byteLength + filter.byteLength) + new DataView(both.buffer).setUint32(0, dir.byteLength) + both.set(dir, 4) + both.set(filter, 4 + dir.byteLength) + await this.store.put(layerDirKey(this.id), both, { ifAbsent: true }) + return { + id: this.id, + level: this.level, + seq, + entries: this.entries, + bytes: off.at(-1)!, + min: this.#min!, + max: detached(this.#last!), + } + } +} + +/** + * A sorted run written as files of at most RUN_FILE_RAW each, so a merge never + * holds a level. Its level is at least `level`, and as high as its size puts it. + */ +export class RunWriter { + readonly id = newId() + readonly #done: Promise[] = [] + #file: LayerWriter | null = null + #raw = 0 + #entries = 0 + + constructor( + readonly store: Store, + readonly level: number, + ) {} + + /** Wait while a finished file is still writing: a merge holds at most the one filling and one writing. */ + async ready(): Promise { + while (this.#done.length > 1) { + const pending = this.#done.slice(0, -1) + await Promise.all(pending) + this.#settled.push(...pending) + this.#done.splice(0, pending.length) + } + } + + /** Files already written, kept until finish. */ + readonly #settled: Promise[] = [] + + add(key: string, value: string): void { + if (this.#file && this.#raw >= RUN_FILE_RAW) { + this.#done.push(this.#file.finish()) + this.#file = null + } + if (!this.#file) { + this.#file = new LayerWriter(this.store, this.level) + this.#raw = 0 + } + this.#file.add(key, value) + this.#raw += key.length + value.length + 2 + this.#entries++ + } + + async finish(seq = 0): Promise { + if (this.#file) this.#done.push(this.#file.finish()) + this.#file = null + const files = (await Promise.all([...this.#settled, ...this.#done])).filter( + (f): f is LayerInfo => f !== null, + ) + const level = Math.max(this.level, levelFor(this.#entries)) + return files.map((f) => ({ + ...f, + seq, + level, + ...(files.length > 1 ? { run: this.id } : {}), + })) + } +} + +/** Reads layers a page at a time, through the repository's memory cache. */ +export class LayerReader { + constructor( + readonly store: Store, + readonly lru: Lru, + readonly scope: string, + ) {} + + async #dir(id: string): Promise { + const k = `${this.scope}/layerdir:${id}` + const hit = this.lru.get(k) + if (hit) return (hit as { v: LayerDir }).v + const o = await this.store.get(layerDirKey(id)) + if (!o) throw new Error(`Missing layer directory ${id}`) + const b = await o.bytes() + let v: LayerDir + let filter: Uint8Array | null = null + if (b[0] === 0x7b) + v = JSON.parse(new TextDecoder().decode(b)) as LayerDir // `{`: an older layer, filter apart + else { + const n = new DataView(b.buffer, b.byteOffset).getUint32(0) + v = JSON.parse(new TextDecoder().decode(b.subarray(4, 4 + n))) as LayerDir + filter = b.slice(4 + n) + this.lru.set(`${this.scope}/filter:${id}`, { v: filter, approxBytes: filter.byteLength }) + } + this.lru.set(k, { v, approxBytes: v.first.reduce((n, f) => n + f.length * 2 + 16, 0) }) + return v + } + + async #page(id: string, dir: LayerDir, i: number): Promise<[string, string][]> { + const k = `${this.scope}/page:${id}.${i}` + const hit = this.lru.get(k) + if (hit) return (hit as { v: [string, string][] }).v + const o = await this.store.get(layerKey(id), { + offset: dir.off[i]!, + length: dir.off[i + 1]! - dir.off[i]!, + }) + if (!o) throw new Error(`Missing layer ${id}`) + const text = await gunzipText(await o.bytes()) + const v = text + .split('\n') + .filter(Boolean) + .map((l) => { + const t = l.indexOf('\t') + return [l.slice(0, t), l.slice(t + 1)] as [string, string] + }) + this.lru.set(k, { v, approxBytes: text.length * 3 }) + return v + } + + async #filter(id: string): Promise { + const k = `${this.scope}/filter:${id}` + const hit = this.lru.get(k) + if (hit) return (hit as { v: Uint8Array | null }).v + // The directory carries the filter (newer layers); reading it caches both. + await this.#dir(id) + const again = this.lru.get(k) + if (again) return (again as { v: Uint8Array | null }).v + const o = await this.store.get(layerFilterKey(id)) + const v = o ? await o.bytes() : null + this.lru.set(k, { v, approxBytes: v?.byteLength ?? 64 }) + return v + } + + /** + * Entries whose filter key is one of `ids` (records of those ids), reading only + * pages the layer's filter says may hold them. + */ + async lookup(layer: LayerInfo, ids: readonly string[]): Promise<[string, string][]> { + const inRange = ids.filter( + (id) => compareUtf8(layer.max, id) >= 0 && compareUtf8(layer.min, `${id}\u0000\uffff`) <= 0, + ) + if (!inRange.length) return [] + const filter = await this.#filter(layer.id) + const maybe = filter ? inRange.filter((id) => filterHas(filter, id)) : inRange + if (!maybe.length) return [] + const dir = await this.#dir(layer.id) + const pages = new Set() + for (const id of maybe) { + // Pages whose range can hold id's keys: from the last page starting at or before id. + let lo = 0 + let hi = dir.first.length - 1 + while (lo < hi) { + const mid = (lo + hi + 1) >> 1 + if (compareUtf8(dir.first[mid]!, id) <= 0) lo = mid + else hi = mid - 1 + } + pages.add(lo) + for ( + let p = lo + 1; + p < dir.first.length && compareUtf8(dir.first[p]!, `${id}\u0000\uffff`) <= 0; + p++ + ) + pages.add(p) + } + const want = new Set(maybe) + const out: [string, string][] = [] + const read = await Promise.all([...pages].map((p) => this.#page(layer.id, dir, p))) + for (const page of read) for (const e of page) if (want.has(filterKey(e[0]))) out.push(e) + return out + } + + /** Entries with keys in [from, to], in key order. */ + async range(layer: LayerInfo, from: string, to: string): Promise<[string, string][]> { + if (compareUtf8(layer.max, from) < 0 || compareUtf8(layer.min, to) > 0) return [] + const dir = await this.#dir(layer.id) + // The last page whose first key is ≤ from, through the last whose first key is ≤ to. + let lo = 0 + while (lo + 1 < dir.first.length && compareUtf8(dir.first[lo + 1]!, from) <= 0) lo++ + let hi = lo + while (hi + 1 < dir.first.length && compareUtf8(dir.first[hi + 1]!, to) <= 0) hi++ + const out: [string, string][] = [] + const pages = await Promise.all( + Array.from({ length: hi - lo + 1 }, (_, j) => this.#page(layer.id, dir, lo + j)), + ) + for (const page of pages) + for (const e of page) + if (compareUtf8(e[0], from) >= 0 && compareUtf8(e[0], to) <= 0) out.push(e) + return out + } + + /** The first key of each page, with the page's share of the layer's entries (where to cut it into ranges). */ + async pageKeys(layer: LayerInfo): Promise<{ keys: string[]; perPage: number }> { + const dir = await this.#dir(layer.id) + return { keys: dir.first, perPage: layer.entries / Math.max(1, dir.first.length) } + } + + /** + * Every entry in order (keys in [from, to) when given), read 2 MB at a time, + * bypassing the cache (merges, reclaim's key ranges). + */ + async *all(layer: LayerInfo, from?: string, to?: string): AsyncGenerator<[string, string]> { + if (from !== undefined && compareUtf8(layer.max, from) < 0) return + if (to !== undefined && compareUtf8(layer.min, to) >= 0) return + const dir = await this.#dir(layer.id) + const off = dir.off + let start = 0 + if (from !== undefined) + while (start + 1 < dir.first.length && compareUtf8(dir.first[start + 1]!, from) <= 0) start++ + for (let p = start; p < dir.first.length;) { + if (to !== undefined && compareUtf8(dir.first[p]!, to) >= 0) return + let e = p + while (e + 1 < dir.first.length && off[e + 2]! - off[p]! <= MERGE_READ) e++ + const o = await this.store.get(layerKey(layer.id), { + offset: off[p]!, + length: off[e + 1]! - off[p]!, + }) + if (!o) throw new Error(`Missing layer ${layer.id}`) + const bytes = await o.bytes() + for (let k = p; k <= e; k++) { + for (const l of ( + await gunzipText(bytes.subarray(off[k]! - off[p]!, off[k + 1]! - off[p]!)) + ).split('\n')) { + if (!l) continue + const t = l.indexOf('\t') + const k = l.slice(0, t) + if (from !== undefined && compareUtf8(k, from) < 0) continue + if (to !== undefined && compareUtf8(k, to) >= 0) return + yield [k, l.slice(t + 1)] + } + } + p = e + 1 + } + } +} + +/** A run's entries in order (keys in [from, to) when given): its files one after another. */ +export async function* runEntries( + reader: LayerReader, + files: readonly LayerInfo[], + from?: string, + to?: string, +): AsyncGenerator<[string, string]> { + for (const f of files) yield* reader.all(f, from, to) +} + +/** Merge sorted streams, newest first winning a repeated key. */ +export async function* mergeSorted( + streams: AsyncIterable<[string, V]>[], +): AsyncGenerator<[string, V]> { + const its = streams.map((s) => s[Symbol.asyncIterator]()) + const heads = await Promise.all(its.map((it) => it.next())) + for (;;) { + let best = -1 + for (let i = 0; i < heads.length; i++) { + const h = heads[i]! + if (h.done) continue + if (best < 0 || compareUtf8(h.value[0], (heads[best]!.value as [string, V])[0]) < 0) best = i + } + if (best < 0) return + const key = (heads[best]!.value as [string, V])[0] + yield heads[best]!.value as [string, V] + // Skip the same key in older streams (and advance the winner). + for (let i = 0; i < heads.length; i++) { + while (!heads[i]!.done && (heads[i]!.value as [string, V])[0] === key) + heads[i] = await its[i]!.next() + } + } +} + +// --- Manifest ------------------------------------------------------------------------- + +export interface Manifest { + seq: number + layers: LayerInfo[] + packs: PackInfo[] + /** Objects replaced by a merge, deleted once older than the grace period. */ + retired: { key: string; at: number }[] + /** Counters of merge work. */ + stats: Record + /** + * Pending objects (F) already folded in, until their deletion is seen: one whose + * delete failed must never be folded again once its layers are merged away. + */ + folded?: string[] +} + +const empty = (): Manifest => ({ seq: 0, layers: [], packs: [], retired: [], stats: {} }) +const pad = (n: number) => String(n).padStart(12, '0') + +/** The least time between writes of a manifest's head: R2's one write a second a key (tests set it lower). */ +export const headConfig = { intervalMs: 1000 } + +/** + * Folded pending objects one manifest update deletes; the rest stay + * listed in `folded` and go in later updates (a 12M load's 354 units, all in one job's + * calls, took it past a Worker's 10,000 subrequests). + */ +export const pendingConfig = { deletesPerUpdate: 500 } + +/** The sequence a new newest layer takes: past the manifest's number and every layer's. */ +export const topSeq = (m: Manifest) => Math.max(m.seq, ...m.layers.map((l) => l.seq)) + +/** A commit unit's layers and packs, written apart from the manifest until a merge folds them in. */ +export interface PendingRun { + layers: LayerInfo[] + packs: PackInfo[] +} + +/** + * A pending object: one unit's run, or a commit's (a tree's units' runs, in key + * order), each folded in as its own run, as if written one an object. + */ +type PendingObject = PendingRun | { runs: PendingRun[] } +const runsOf = (o: PendingObject): PendingRun[] => ('runs' in o ? o.runs : [o]) + +/** The pending objects a view listed, and which of them it folded in. */ +interface Pending { + listed: string[] + folded: string[] +} + +/** + * Packs no merge will take again, kept out of the manifest: every reader and + * writer reads the manifest whole, so a list of every pack there (tens of + * thousands at Crossref's size) would make each commit and cold read pay for the + * whole store. They're listed instead in write-once objects that only reclaim + * reads, and reclaim rewrites as one. + */ +export class PackRegistry { + constructor( + readonly store: Store, + readonly name: string, + ) {} + + get #prefix() { + return `manifest/${this.name}/packs/` + } + + /** List `packs` in a new object; its key (null for none). */ + async add(packs: readonly PackInfo[]): Promise { + if (!packs.length) return null + const key = `${this.#prefix}${newId()}.json` + await this.store.put(key, JSON.stringify(packs), { ifAbsent: true }) + return key + } + + async read(): Promise<{ keys: string[]; packs: PackInfo[] }> { + const keys: string[] = [] + let cursor: string | undefined + do { + const page = await this.store.list(this.#prefix, cursor) + keys.push(...page.keys) + cursor = page.cursor + } while (cursor) + const packs: PackInfo[] = [] + for (let i = 0; i < keys.length; i += 16) + for (const o of await Promise.all(keys.slice(i, i + 16).map((k) => this.store.get(k)))) + if (o) packs.push(...(JSON.parse(await o.text()) as PackInfo[])) + return { keys, packs } + } + + /** Replace the objects `keys` with one listing `packs`. */ + async replace(keys: readonly string[], packs: readonly PackInfo[]): Promise { + await this.add(packs) + for (let i = 0; i < keys.length; i += 16) + await Promise.all(keys.slice(i, i + 16).map((k) => this.store.delete(k))) + } +} + +/** Pending objects this process has written: orders those written in one millisecond. */ +let appended = 0 + +export class ManifestStore { + #cached: { m: Manifest; at: number } | null = null + /** Pending objects read, by key: each is written once and never changes. */ + readonly #pendingRuns = new Map() + /** When the head was last written, as the head says (it's written at most once a second). */ + #headAt = 0 + /** The manifest number the head was last seen or written at. */ + #headSeq = 0 + + /** + * A commit unit's layers and packs go in an object of their own (a new key each), + * folded into the manifest by the next update, which only merges make (one job + * at a time per collection). Nothing a commit writes is ever written twice, so no + * key is written more than once a second however many of a push's units finish + * together (a manifest object every unit rewrote had 69 such writes in the bench). + */ + constructor( + readonly store: Store, + readonly name: string, + ) {} + + get #pendingPrefix() { + return `manifest/${this.name}/pending/` + } + + /** + * `m` with the pending runs not yet in it, newest last (keys sort by time), each + * a sequence past everything in `m`: what the next update will write. + */ + async #withPending(m: Manifest): Promise<{ m: Manifest; pending: Pending }> { + const pending: Pending = { listed: [], folded: [] } + const keys: string[] = [] + let cursor: string | undefined + do { + const page = await this.store.list(this.#pendingPrefix, cursor) + keys.push(...page.keys) + cursor = page.cursor + } while (cursor) + keys.sort() + const done = new Set(m.folded ?? []) + const fresh = keys.filter((k) => !done.has(k)) + pending.listed = keys + if (!fresh.length) return { m, pending } + // Read once an isolate: every manifest read sees them all, several times a job. + const listed = new Set(keys) + for (const k of this.#pendingRuns.keys()) if (!listed.has(k)) this.#pendingRuns.delete(k) + const runs: (PendingObject | null)[] = [] + for (let i = 0; i < fresh.length; i += 16) + runs.push( + ...(await Promise.all( + fresh.slice(i, i + 16).map(async (k) => { + const known = this.#pendingRuns.get(k) + if (known) return known + // Folded and deleted since the listing: already in a newer manifest. + const o = await this.store.get(k) + if (!o) return null + const r = JSON.parse(await o.text()) as PendingObject + this.#pendingRuns.set(k, r) + return r + }), + )), + ) + const have = new Set(m.layers.map((l) => l.id)) + const havePacks = new Set(m.packs.map((p) => p.id)) + const out = { ...m, layers: [...m.layers], packs: [...m.packs] } + let seq = topSeq(m) + for (let i = 0; i < runs.length; i++) { + const o = runs[i] + if (!o) continue + // A run already folded in (an object written twice, or folded before) is skipped. + for (const r of runsOf(o)) { + if (r.layers.some((l) => have.has(l.id))) continue + seq++ + for (const l of r.layers) { + out.layers.push({ ...l, seq }) + have.add(l.id) + } + for (const p of r.packs) + if (!havePacks.has(p.id)) { + out.packs.push(p) + havePacks.add(p.id) + } + } + pending.folded.push(fresh[i]!) + } + return { m: out, pending } + } + + #key(seq: number) { + return `manifest/${this.name}/${pad(seq)}.json` + } + + async #read(seq: number): Promise { + if (seq === 0) return empty() + // Two writers racing for one number can leave a local file store's reader with + // a cut-short read (it sizes the file, then opens the other writer's): read again. + for (let attempt = 0; ; attempt++) { + const o = await this.store.get(this.#key(seq)) + if (!o) return null + try { + return JSON.parse(await o.text()) as Manifest & { token?: string } + } catch (err) { + if (attempt >= 5) throw err + await new Promise((r) => setTimeout(r, 10)) + } + } + } + + /** + * The head: the newest manifest as its last writer left it (the whole manifest, + * so a reader needs one request), or an older head's bare `{seq}`. + */ + async #head(): Promise { + for (let attempt = 0; ; attempt++) { + const h = await this.store.get(`manifest/${this.name}/head.json`) + if (!h) return null + try { + const v = JSON.parse(await h.text()) as Partial & { seq: number; at?: number } + this.#headAt = v.at ?? 0 + this.#headSeq = v.seq + if (v.layers) return v as Manifest + return (await this.#read(v.seq)) ?? empty() + } catch (err) { + // A local file store can cut a read short while the head is replaced (see #read). + if (attempt >= 5) throw err + await new Promise((r) => setTimeout(r, 10)) + } + } + } + + /** The newest manifest: the head, then any claimed after it (a writer may not have moved the head yet). */ + async latest(): Promise { + return (await this.#latest()).m + } + + async #latest(): Promise<{ m: Manifest; pending: Pending; stored: Manifest }> { + // The head moves at most once a second, so it can be many manifests behind: find + // the newest by listing them (a few dozen keys), not from the head. + let m = await this.#listed() + for (;;) { + const next = await this.#read(m.seq + 1) + if (!next) break + m = next + } + const v = await this.#withPending(m) + this.#cached = { m: v.m, at: Date.now() } + return { ...v, stored: m } + } + + /** The newest manifest by listing their numbers. */ + async #listed(): Promise { + const prefix = `manifest/${this.name}/0` + for (let attempt = 0; ; attempt++) { + const keys: string[] = [] + let cursor: string | undefined + do { + const page = await this.store.list(prefix, cursor) + keys.push(...page.keys) + cursor = page.cursor + } while (cursor) + const top = keys.sort().at(-1) + if (!top) return empty() + const m = await this.#read(Number(top.slice(`manifest/${this.name}/`.length, -5))) + // Deleted as old history between the listing and the read: list again. + if (m || attempt >= 5) return m ?? empty() + } + } + + /** + * Bring the head up to the newest manifest, waiting out R2's second a key if it + * was just written. Every job that changes the manifest ends with it, so readers' + * heads are never left behind. + */ + async settle(): Promise { + const { stored } = await this.#latest() + if (stored.seq === 0) return + await this.#moveHead(stored, true) + } + + /** + * Write the head as `m`, at most once a second whoever writes it. + * Split merges' swaps, reclaim and forks write it from their own jobs, beside the + * maintain job, so when it was last written is read from the head itself: an + * isolate that hadn't read it would think it free (round 17: a write R2 would + * refuse). With `wait`, a head written under a second ago is waited out; without, + * it's left for the job's settle. A head left behind costs readers a listing, + * never a record, so a refused write isn't the job's failure. + */ + async #moveHead(m: Manifest, wait: boolean): Promise { + // This isolate wrote or read it under a second ago: it's no older in storage. + if (!wait && Date.now() - this.#headAt < headConfig.intervalMs) return + for (let attempt = 0; attempt < 5; attempt++) { + await this.#head() + if (this.#headSeq >= m.seq) return + const left = headConfig.intervalMs - (Date.now() - this.#headAt) + if (left > 0) { + if (!wait) return + // Then read it again: another job waiting on the same head may have moved it. + // A random share of a second more, so jobs waiting together don't wake together. + await new Promise((r) => setTimeout(r, left + (Math.random() * headConfig.intervalMs) / 2)) + continue + } + const now = Date.now() + try { + await this.store.put(`manifest/${this.name}/head.json`, JSON.stringify({ ...m, at: now })) + } catch { + return + } + this.#headAt = now + this.#headSeq = m.seq + return + } + } + + /** + * The manifest for a reader: this isolate's copy when younger than `maxAgeMs`, + * else the head alone (one request). A reader that misses a record asks + * `latest()`, which also checks for newer ones. + */ + async current(maxAgeMs = 2000): Promise { + if (this.#cached && Date.now() - this.#cached.at < maxAgeMs) return this.#cached.m + // A writer wants the newest: the head may be behind. + if (maxAgeMs === 0) return this.latest() + const m = (await this.#head()) ?? empty() + this.#cached = { m, at: Date.now() } + return m + } + + /** + * A commit's runs, held back by its units (a tree's, in key order), as one pending + * object: a push of thousands of units leaves a few objects, not one a unit. + */ + async appendRuns(runs: PendingRun[]): Promise { + if (!runs.length) return + await this.#putPending({ runs }) + } + + /** Add a commit unit's new layers (the newest) and packs, as a pending object of its own. */ + async append(layers: LayerInfo[], packs: PackInfo[]): Promise { + await this.#putPending({ layers, packs }) + } + + async #putPending(o: PendingObject): Promise { + // Keys sort by time: a later commit's runs fold in after (and win over) an earlier's. + // Within a millisecond, by this process's count: two commits a process makes in + // one millisecond (tests) would otherwise fold in their random ids' order. + const key = `${this.#pendingPrefix}${String(Date.now()).padStart(15, '0')}-${String(++appended).padStart(9, '0')}-${newId()}.json` + await this.store.put(key, JSON.stringify(o), { ifAbsent: true }) + this.#cached = null + } + + /** Apply `change` to the newest manifest and claim the next number; again on a race. */ + async update(change: (m: Manifest) => Manifest | null): Promise { + for (let attempt = 0; attempt < 100; attempt++) { + const { m, pending } = await this.#latest() + const next = change(structuredClone(m)) + if (!next) return m + next.seq = m.seq + 1 + // Folded before and still there (a delete not yet seen), and folded now. + const listed = new Set(pending.listed) + next.folded = [...(m.folded ?? []).filter((k) => listed.has(k)), ...pending.folded] + const token = newId() + await this.store.put(this.#key(next.seq), JSON.stringify({ ...next, token }), { + ifAbsent: true, + }) + const back = (await this.#read(next.seq)) as (Manifest & { token?: string }) | null + if (back?.token !== token) { + await new Promise((r) => setTimeout(r, 5 + Math.random() * 20 * (attempt + 1))) + continue + } + // The head carries the whole manifest: readers need one request. It's written at + // most once a second (R2's limit a key): a reader whose head is behind finds the + // newer manifests by number, and misses send it there. + await this.#moveHead(next, false) + this.#cached = { m: next, at: Date.now() } + // Some at a time: those left stay in `folded` (still listed) for the next update. + const gone = next.folded.slice(0, pendingConfig.deletesPerUpdate) + for (let i = 0; i < gone.length; i += 16) + await Promise.all(gone.slice(i, i + 16).map((k) => this.store.delete(k))) + // Older manifests are history nobody reads: keep the last 25 or so. + if (next.seq % 25 === 0) + for (let s = Math.max(1, next.seq - 50); s <= next.seq - 25; s++) + await this.store.delete(this.#key(s)) + return next + } + throw new Error(`Manifest ${this.name}: too many concurrent updates`) + } +} + +/** The index a stack of layers forms: newest wins. */ +export class LayeredIndex { + readonly reader: LayerReader + + constructor( + readonly store: Store, + readonly lru: Lru, + readonly scope: string, + readonly manifests: ManifestStore, + readonly filter: (l: LayerInfo) => boolean = () => true, + ) { + this.reader = new LayerReader(store, lru, scope) + } + + /** Values for the keys of `ids` (by filter key), newest layer winning, from manifest `m`. */ + async lookup(m: Manifest, ids: readonly string[]): Promise> { + const layers = m.layers.filter(this.filter).sort((a, b) => b.seq - a.seq) + const results = await Promise.all(layers.map((l) => this.reader.lookup(l, ids))) + const out = new Map() + for (const r of results) for (const [k, v] of r) if (!out.has(k)) out.set(k, v) + return out + } + + /** Values for keys in [from, to], newest layer winning, from manifest `m`. */ + async range(m: Manifest, from: string, to: string): Promise> { + const layers = m.layers.filter(this.filter).sort((a, b) => b.seq - a.seq) + const results = await Promise.all(layers.map((l) => this.reader.range(l, from, to))) + const out = new Map() + for (const r of results) for (const [k, v] of r) if (!out.has(k)) out.set(k, v) + return out + } +} diff --git a/packages/protocol/src/repo/layouts/reclaim.ts b/packages/protocol/src/repo/layouts/reclaim.ts new file mode 100644 index 00000000..36bfa631 --- /dev/null +++ b/packages/protocol/src/repo/layouts/reclaim.ts @@ -0,0 +1,120 @@ +import { type RecordEntry, recordTree } from '../../tree/node.js' +/** + * Reclaim's mark for a key range (keyed layouts, record-storage-layout.md + * "Reclaim needs a mark again"). A leaf covers its records, so today's cleanup + * marks nodes; a packed layout has to know which records are live, one by one, + * and at Crossref's size that set doesn't fit one isolate. So reclaim works a + * key range at a time: each range's job walks every retained version's record + * trees only where they overlap the range, and keeps the live records as 52-bit + * hashes, 8 bytes each. A collision can only keep a dead record, never drop a + * live one. + */ +import { compareUtf8 } from '../../utf8.js' +import type { Repo } from '../repo.js' + +/** Live records of one key range, as sorted 52-bit hashes of (id, record hash). */ +export class LiveHashes { + #h = new Float64Array(4096) + size = 0 + #sorted = true + + add(id: string, hash: string): void { + if (this.size === this.#h.length) { + const grown = new Float64Array(this.#h.length * 2) + grown.set(this.#h) + this.#h = grown + } + this.#h[this.size++] = hash52(id, hash) + this.#sorted = false + } + + has(id: string, hash: string): boolean { + if (!this.#sorted) { + this.#h = this.#h.subarray(0, this.size).sort() + this.#sorted = true + } + const x = hash52(id, hash) + let lo = 0 + let hi = this.size - 1 + while (lo <= hi) { + const mid = (lo + hi) >> 1 + const v = this.#h[mid]! + if (v === x) return true + if (v < x) lo = mid + 1 + else hi = mid - 1 + } + return false + } +} + +/** FNV-1a with two seeds over id, a separator and hash, folded to 52 bits (exact in a double). */ +function hash52(id: string, hash: string): number { + let a = 2_166_136_261 + let b = 0x9747b28c + const step = (c: number) => { + a = Math.imul(a ^ c, 16_777_619) + b = Math.imul(b ^ c, 0x5bd1e995) + } + for (let i = 0; i < id.length; i++) step(id.charCodeAt(i)) + step(0) + for (let i = 0; i < hash.length; i++) step(hash.charCodeAt(i)) + return (a >>> 0) * 1_048_576 + ((b >>> 0) >>> 12) +} + +/** A key range: ids in [lo, hi); null is open. */ +export interface KeyRange { + lo: string | null + hi: string | null +} + +export const inRange = (r: KeyRange, k: string) => + (r.lo === null || compareUtf8(k, r.lo) >= 0) && (r.hi === null || compareUtf8(k, r.hi) < 0) + +/** + * Mark the records of `range` that the given record tree roots hold. Each node is + * read once however many versions share it, and only nodes whose keys overlap + * the range are read. + */ +export async function markRange( + repo: Repo, + treeRoots: Iterable, + range: KeyRange, +): Promise<{ live: LiveHashes; nodesRead: number }> { + const live = new LiveHashes() + const seen = new Set() + let nodesRead = 0 + const walk = async (hash: string): Promise => { + if (seen.has(hash)) return + seen.add(hash) + nodesRead++ + const node = await repo.decoded(recordTree, hash) + if (node.kind === 'leaf') { + for (const e of node.entries as RecordEntry[]) + if (inRange(range, e.key)) live.add(e.key, e.hash) + return + } + // A child holds keys after the one before it's last, through its own last. + const due: string[] = [] + for (let i = 0; i < node.children.length; i++) { + const c = node.children[i]! + const prev = i > 0 ? node.children[i - 1]!.lastKey : null + if (range.hi !== null && prev !== null && compareUtf8(prev, range.hi) >= 0) break + if (range.lo !== null && compareUtf8(c.lastKey, range.lo) < 0) continue + due.push(c.hash) + } + for (let i = 0; i < due.length; i += 16) await Promise.all(due.slice(i, i + 16).map(walk)) + } + for (const root of treeRoots) await walk(root) + return { live, nodesRead } +} + +/** Every record tree root of the given versions (public and private sets, every type). */ +export async function recordRoots(repo: Repo, versionHashes: Iterable): Promise { + const out = new Set() + for (const v of versionHashes) { + const root = await repo.root(v) + const sets = [root.public, ...(root.private ? [await repo.privateSet(root.private)] : [])] + for (const s of sets) for (const t of Object.values(s.types)) if (t.root) out.add(t.root) + } + return [...out] +} diff --git a/packages/protocol/src/repo/layouts/types.ts b/packages/protocol/src/repo/layouts/types.ts new file mode 100644 index 00000000..8fc847bb --- /dev/null +++ b/packages/protocol/src/repo/layouts/types.ts @@ -0,0 +1,75 @@ +/** + * Where a repository keeps record bytes (record-storage-layout.md). `today` is no + * RecordStore at all (leaf bodies, repo.ts). `f` takes records out of leaves, so a + * commit writes only the records that changed and a leaf's node lists what it + * holds. Neither changes a node, a root or any hash. + */ +import type { RecordEntry } from '../../tree/node.js' +import type { PendingRun } from './kit.js' + +export const LAYOUTS = ['today', 'f'] as const +export type LayoutName = (typeof LAYOUTS)[number] + +/** A record entry as layout F carries it through a build: with its place once stored. */ +export type Placed = RecordEntry & { loc?: string } + +/** Records still in use, from a mark over every retained version (storage reclaim). */ +export interface LiveSet { + /** `${id}\u0000${hash}` of every record in a retained version. */ + records: Set +} + +/** A record set: a version's public records, or its private ones (file-sets.ts SetName). */ +export type RecordSet = 'public' | 'private' + +/** + * A collection's records in layout F, a store a set: its public records and its + * private ones never share a pack or an index run, so a mirror of the public set + * copies files that hold nothing else. + */ +export interface RecordSets { + readonly public: RecordStore + readonly private: RecordStore +} + +export interface RecordStore { + readonly name: LayoutName + /** The stored lines of a leaf's entries, in entry order (out-of-line pointers unresolved). */ + leafLines(leafHash: string, entries: readonly RecordEntry[]): Promise + /** One entry's stored line: a changed record's old body, which file references need. */ + entryLine(leafHash: string, entries: readonly RecordEntry[], i: number): Promise + /** The stored lines of some of a leaf's entries (by index), reading only those records. */ + linesAt( + leafHash: string, + entries: readonly RecordEntry[], + at: readonly number[], + ): Promise + /** A writer for one commit unit or build. */ + writer(): RecordWriter + /** + * Merges due after a write (layers, packs), run in the writer's own job as a commit's + * tail. True when a budget stopped it with work perhaps left: queue another. + */ + maintain(): Promise + /** Drop what `live` doesn't hold, rewriting packs that are mostly dead. Returns counts. */ + reclaim(live: LiveSet): Promise> + /** Name a commit's held runs (RecordWriter.finish with `hold`) in the index, together. */ + appendRuns?(runs: HeldRun[]): Promise + /** Give a fork its records: it lists the parent's files and reads them where they are. */ + forkInto?(target: RecordStore): Promise> +} + +export interface RecordWriter { + /** Store one record line (a spilled or new entry): its place. */ + add(entry: RecordEntry, line: string): string + /** + * Write what's held: packs, the index layer, then the manifest. With `hold`, the + * manifest isn't told: the run is returned, for the commit to name with its others. + */ + finish(opts?: { hold?: boolean }): Promise +} + +/** A run a commit unit wrote and held back from the manifest (layout F). */ +export type HeldRun = PendingRun + +export const recordKey = (id: string, hash: string) => `${id}\u0000${hash}` diff --git a/packages/protocol/src/repo/repo.ts b/packages/protocol/src/repo/repo.ts index 209a4af1..6175f363 100644 --- a/packages/protocol/src/repo/repo.ts +++ b/packages/protocol/src/repo/repo.ts @@ -41,7 +41,19 @@ import { versionDigest, versionHash, } from '../format.js' +import { compareUtf8 } from '../utf8.js' import { chunkLines, gunzipChunks, gunzipStream, gunzipText, gzip, splitLines } from './gzip.js' +import { + type HeldRun, + type LayoutName, + MissingRecordError, + type Placed, + type RecordSet, + type RecordSets, + recordStore, + type RecordStore, + type RecordWriter, +} from './layouts/index.js' import { Lru } from './lru.js' import { type Store, type Cache, noCache, putJoined } from './types.js' @@ -133,8 +145,23 @@ export interface RepoOptions { lru?: Lru /** Size of a private memory cache when `lru` isn't given (default 8 MB). */ memoryBytes?: number + /** + * Where record bytes live (record-storage-layout.md): `today` (default) is leaf + * bodies; `f` is packs found through an index, per collection (`recordScope`). + * No hash depends on it. + */ + layout?: LayoutName + /** Layout F keeps its packs and index per collection: the collection this repository is for. */ + recordScope?: string + /** `delta` stores a rewritten leaf node as its changes against an earlier full node. */ + nodes?: 'full' | 'delta' } +/** A stored node that is a delta: `{"b":,"p":[tuples put],"d":[keys removed]}`. */ +const DELTA_PREFIX = '{"b":"' +/** A delta larger than this share of the full node is stored full instead (a new keyframe). */ +const DELTA_MAX_SHARE = 0.25 + /** Open the repository in a store. */ export function openRepo(store: Store, opts: RepoOptions = {}): Repo { return new Repo(store, opts) @@ -145,6 +172,12 @@ export class Repo { readonly lru: Lru readonly scope: string readonly trusted: boolean + /** Where records live when they aren't in leaf bodies, a store a set (null: today's layout). */ + readonly records: RecordSets | null + readonly nodeDeltas: boolean + /** The layout it was opened in, and for layout F the collection (see `reopen`). */ + readonly layout: LayoutName + readonly recordScope: string | undefined constructor( readonly blobs: Store, @@ -154,6 +187,26 @@ export class Repo { this.lru = opts.lru ?? repoLru(opts.memoryBytes ?? DEFAULT_MEMORY_BYTES) this.scope = opts.scope ?? '' this.trusted = opts.trusted ?? false + const layout = opts.layout ?? 'today' + this.layout = layout + this.recordScope = opts.recordScope + this.records = recordStore( + layout, + this.lru, + `${this.scope}/${layout}/${opts.recordScope ?? ''}`, + opts.recordScope ? { root: blobs, home: opts.recordScope } : undefined, + ) + this.nodeDeltas = opts.nodes === 'delta' + } + + /** The same repository opened again with other options (fsck's untrusted copy), its layout kept. */ + reopen(opts: Omit): Repo { + return new Repo(this.blobs, { + layout: this.layout, + ...(this.recordScope ? { recordScope: this.recordScope } : {}), + nodes: this.nodeDeltas ? 'delta' : 'full', + ...opts, + }) } #ck(key: string) { @@ -224,17 +277,72 @@ export class Repo { async #nodeJson(hash: string, key: string): Promise { const bytes = await this.#immutable(key, async (b) => { - if (sha256Hex(await gunzipText(b)) !== hash) + const text = await gunzipText(b) + // A delta is checked once it's expanded, below. + if (!text.startsWith(DELTA_PREFIX) && sha256Hex(text) !== hash) throw new IntegrityError(`Node ${hash} fails its hash`) }) if (!bytes) throw new Error(`Missing node ${hash}`) - const json = await gunzipText(bytes) + let json = await gunzipText(bytes) + const delta = json.startsWith(DELTA_PREFIX) + this.lru.set(this.#ck(`base:${hash}`), { + v: delta ? (JSON.parse(json) as { b: string }).b : '', + approxBytes: 128, + }) + if (delta) { + json = await this.#expand(json) + if (sha256Hex(json) !== hash) + throw new IntegrityError(`Node ${hash} fails its hash (from a delta)`) + } this.lru.set(this.#ck(key), json) return json } - async putNode(hash: string, json: string): Promise { - await this.blobs.put(keys.node(hash), await gzip(json), { + /** A node as stored: full JSON, or a delta. */ + async #storedNode(hash: string): Promise { + const bytes = await this.#immutable(keys.node(hash)) + if (!bytes) throw new Error(`Missing node ${hash}`) + return gunzipText(bytes) + } + + async #expand(stored: string): Promise { + const d = JSON.parse(stored) as { b: string; p: unknown[][]; d: string[] } + const tuples = new Map(leafTuples(await this.nodeJson(d.b)).map((t) => [tupleKey(t), t])) + for (const k of d.d) tuples.delete(k) + for (const t of d.p) tuples.set(t[0] as string, JSON.stringify(t)) + const keysInOrder = [...tuples.keys()].sort(compareUtf8) + return '{"e":[' + keysInOrder.map((k) => tuples.get(k)!).join(',') + '],"t":"leaf"}' + } + + /** The full node a delta of `base` diffs against: `base` itself, or its own base. */ + async deltaBase(hash: string): Promise { + const hit = this.lru.get(this.#ck(`base:${hash}`)) + if (hit) return (hit as { v: string }).v || null + const stored = await this.#storedNode(hash) + return stored.startsWith(DELTA_PREFIX) ? (JSON.parse(stored) as { b: string }).b : null + } + + /** + * Write a node. With node deltas, a rewritten leaf (`base`: the leaf it replaces) + * is stored as its changes against the base's full node, while they stay small. + */ + async putNode(hash: string, json: string, base?: string): Promise { + let stored = json + if (this.nodeDeltas && base && json.endsWith('],"t":"leaf"}')) { + const frame = (await this.deltaBase(base)) ?? base + const old = new Map(leafTuples(await this.nodeJson(frame)).map((t) => [tupleKey(t), t])) + const put: unknown[] = [] + const seen = new Set() + for (const t of leafTuples(json)) { + const k = tupleKey(t) + seen.add(k) + if (old.get(k) !== t) put.push(JSON.parse(t)) + } + const del = [...old.keys()].filter((k) => !seen.has(k)) + const delta = JSON.stringify({ b: frame, p: put, d: del }) + if (delta.length <= json.length * DELTA_MAX_SHARE) stored = delta + } + await this.blobs.put(keys.node(hash), await gzip(stored), { contentType: 'application/gzip', ifAbsent: true, }) @@ -257,14 +365,83 @@ export class Repo { * A leaf's body lines, in entry order, with out-of-line records resolved. On an * untrusted location, each line is checked against the leaf's record hashes. */ - async bodyLines(leaf: { hash: string; entries: readonly RecordEntry[] }): Promise { + async bodyLines( + leaf: { hash: string; entries: readonly RecordEntry[] }, + /** Layout F: the set the leaf is in (else public, then private). */ + set?: RecordSet, + ): Promise { const key = keys.body(leaf.hash) const hit = this.lru.get(this.#ck(key)) if (hit) return (hit as { lines: string[] }).lines // Two readers of one leaf at once decompress it once. + if (this.records) return this.#shared(`lines:${key}`, () => this.#packedLines(leaf, key, set)) return this.#shared(`lines:${key}`, () => this.#bodyLines(leaf, key)) } + /** + * Read from the set's record store; a reader that doesn't know the leaf's set + * tries the public set, then the private one (a miss costs a second lookup). + */ + async #inSet(set: RecordSet | undefined, read: (r: RecordStore) => Promise): Promise { + const sets = this.records! + if (set) return read(sets[set]) + try { + return await read(sets.public) + } catch (err) { + if (!(err instanceof MissingRecordError)) throw err + return read(sets.private) + } + } + + async #packedLines( + leaf: { hash: string; entries: readonly RecordEntry[] }, + key: string, + set: RecordSet | undefined, + ): Promise { + const lines = await this.#resolve( + await this.#inSet(set, (r) => r.leafLines(leaf.hash, leaf.entries)), + ) + // A packed layout's records come from its packs, not a body the leaf hashes: on an + // untrusted location (a mirror read by a third party) each is checked as a body's is. + if (!this.trusted) + lines.forEach((l, i) => { + const e = leaf.entries[i]! + if (sha256Hex(l) !== e.hash) + throw new IntegrityError(`Leaf ${leaf.hash}: record ${i} fails its hash`) + if (utf8ByteLength(l) !== e.size || !l.startsWith(`{"id":${JSON.stringify(e.key)},"type":`)) + throw new IntegrityError(`Leaf ${leaf.hash}: record ${i} isn't its entry's record`) + }) + this.lru.set(this.#ck(key), { lines, approxBytes: lines.reduce((n, l) => n + l.length * 2, 0) }) + return lines + } + + /** + * Some of a leaf's records (by entry index), resolved. A packed layout reads only + * those; leaf bodies are read whole (or from memory) and picked from. + */ + async linesAt( + leaf: { hash: string; entries: readonly RecordEntry[] }, + at: readonly number[], + set?: RecordSet, + ): Promise { + if (!this.records) { + const lines = await this.bodyLines(leaf) + return at.map((i) => lines[i]!) + } + return this.#resolve(await this.#inSet(set, (r) => r.linesAt(leaf.hash, leaf.entries, at))) + } + + /** One entry's record line (resolved): a changed record's old body, in a packed layout. */ + async entryLine( + leaf: { hash: string; entries: readonly RecordEntry[] }, + i: number, + set?: RecordSet, + ): Promise { + if (!this.records) return (await this.bodyLines(leaf))[i]! + const line = await this.#inSet(set, (r) => r.entryLine(leaf.hash, leaf.entries, i)) + return line.startsWith(REF_PREFIX) ? this.#outOfLine(line) : line + } + async #bodyLines( leaf: { hash: string; entries: readonly RecordEntry[] }, key: string, @@ -311,11 +488,19 @@ export class Repo { entries: readonly RecordEntry[] }, onRead?: (r: BodyRead) => void, - opts: { pointers?: boolean } = {}, + opts: { pointers?: boolean; set?: RecordSet } = {}, ): AsyncGenerator { const key = keys.body(leaf.hash) // The memory cache holds lines resolved. const hit = opts.pointers && this.trusted ? undefined : this.lru.get(this.#ck(key)) + if (this.records && !hit) { + // A packed layout: 256 records at a time, so a leaf of large records is never held whole. + for (let i = 0; i < leaf.entries.length; i += 256) { + const at = Array.from({ length: Math.min(256, leaf.entries.length - i) }, (_, j) => i + j) + yield* await this.linesAt(leaf, at, opts.set) + } + return + } if (hit || !this.trusted) { onRead?.({ leaf: leaf.hash, from: hit ? 'memory' : 'whole', storedBytes: null }) yield* hit ? (hit as { lines: string[] }).lines : await this.bodyLines(leaf) @@ -358,6 +543,7 @@ export class Repo { * pointers; callers that need full records use bodyLines. */ async rawBody(leafHash: string): Promise { + if (this.records) throw new Error('rawBody: records are not in leaf bodies in this layout') const bytes = await this.#immutable(keys.body(leafHash)) if (!bytes) throw new Error(`Missing body for leaf ${leafHash}`) return bytes @@ -489,6 +675,8 @@ export class RepoSource implements NodeSource { onBodyRead?: (r: BodyRead) => void /** `leafStream` keeps out-of-line records as pointers (Repo.bodyLineStream). */ pointers?: boolean + /** Layout F: the record set this tree is in, so a read looks in its index alone. */ + set?: RecordSet } = {}, ) {} @@ -501,10 +689,53 @@ export class RepoSource implements NodeSource { if (node.kind !== 'leaf') throw new Error(`Node ${hash} is not a leaf`) if (this.spec !== (recordTree as TreeSpec)) return node.entries.slice() const entries = node.entries as RecordEntry[] - const lines = await this.repo.bodyLines({ hash, entries }) + const lines = await this.repo.bodyLines({ hash, entries }, this.opts.set) return entries.map((e, i) => ({ ...e, body: lines[i]! })) as E[] } + /** + * Layout F's leaf entries without bodies, for a merge: unchanged records aren't + * read or written again. Absent for today's layout. + */ + get sparseStream(): ((hash: string) => AsyncGenerator) | undefined { + const records = this.repo.records + if (!records || this.spec !== (recordTree as TreeSpec)) return undefined + return async function* (this: RepoSource, hash: string) { + const node = await this.node(hash) + if (node.kind !== 'leaf') throw new Error(`Node ${hash} is not a leaf`) + for (const e of node.entries as RecordEntry[]) + yield { key: e.key, hash: e.hash, size: e.size } as E + }.bind(this) + } + + /** Some of a leaf's entries with bodies, reading only those records (packed layouts only). */ + get payloadsAt(): + | ((leafHash: string, entries: readonly E[], at: readonly number[]) => Promise) + | undefined { + if (!this.repo.records || this.spec !== (recordTree as TreeSpec)) return undefined + return async (leafHash, entries, at) => { + const lines = await this.repo.linesAt( + { hash: leafHash, entries: entries as unknown as RecordEntry[] }, + at, + this.opts.set, + ) + return at.map((i, j) => ({ ...(entries[i] as unknown as RecordEntry), body: lines[j]! }) as E) + } + } + + /** An entry from `sparseStream` with its body: a changed record's old version. */ + async withPayload(leafHash: string, e: E): Promise { + const node = await this.node(leafHash) + if (node.kind !== 'leaf') throw new Error(`Node ${leafHash} is not a leaf`) + const entries = node.entries as RecordEntry[] + const r = e as unknown as RecordEntry + const i = entries.findIndex((x) => x.key === r.key) + return { + ...r, + body: await this.repo.entryLine({ hash: leafHash, entries }, i, this.opts.set), + } as E + } + /** A leaf's entries with bodies, one at a time (Repo.bodyLineStream). */ async *leafStream(hash: string): AsyncGenerator { const node = await this.node(hash) @@ -517,6 +748,7 @@ export class RepoSource implements NodeSource { let i = 0 for await (const body of this.repo.bodyLineStream({ hash, entries }, this.opts.onBodyRead, { pointers: this.opts.pointers ?? false, + ...(this.opts.set ? { set: this.opts.set } : {}), })) { yield { ...entries[i++]!, body } as E } @@ -551,6 +783,10 @@ export class RepoSink implements TreeSink { /** Raw body bytes of the writes in flight. */ #pendingBytes = 0 readonly written: string[] = [] + /** Layout F's writer for this sink's records: its set's (null for leaf bodies, or none yet). */ + #records: RecordWriter | null = null + /** The leaf being rewritten: the base of the next leaf node's delta. */ + #base: string | undefined constructor( readonly repo: Repo, @@ -561,9 +797,27 @@ export class RepoSink implements TreeSink { pendingBytes?: number /** Raw text of members compressing before `settle()` waits (default 2 MB). */ compressingBytes?: number + /** Layout F: the record set this sink's records go to (needed to write any). */ + set?: RecordSet + /** Layout F: hold each run back from the manifest (`held`), for the commit to name. */ + holdRecords?: boolean } = {}, ) {} + /** Runs held back (`holdRecords`), in the order written. */ + readonly held: HeldRun[] = [] + + /** Layout F's writer for this sink's set, made on first use; null for leaf bodies. */ + get #writer(): RecordWriter | null { + if (!this.repo.records) return null + if (!this.opts.set) throw new Error('RepoSink: layout F writes records to a set; pass `set`') + return (this.#records ??= this.repo.records[this.opts.set].writer()) + } + + rewriting(base: string): void { + this.#base = base + } + #run(key: string, work: () => Promise, bytes = 0) { this.#pendingBytes += bytes const p = work() @@ -593,12 +847,29 @@ export class RepoSink implements TreeSink { } spill(entries: readonly E[]): void { - if (this.opts.bodyOf) this.#members.push(this.#member(entries)) + if (!this.opts.bodyOf) return + if (this.repo.records) { + for (const e of entries) this.#place(e) + return + } + this.#members.push(this.#member(entries)) + } + + /** Store a new record in the packed layout, and note its place on the entry. */ + #place(e: E) { + const p = e as unknown as Placed + if (p.loc !== undefined || p.body === undefined) return + p.loc = this.#writer!.add(p, p.body) } leaf(desc: NodeDesc, json: string, entries: readonly E[], spilled: number): void { - this.#run(keys.node(desc.hash), () => this.repo.putNode(desc.hash, json)) + const base = this.#base + this.#run(keys.node(desc.hash), () => this.repo.putNode(desc.hash, json, base)) if (!this.opts.bodyOf) return + if (this.repo.records) { + for (const e of entries.slice(spilled)) this.#place(e) + return + } const tail = entries.slice(spilled) const members = [...this.#members, ...(tail.length > 0 ? [this.#member(tail)] : [])] const bytes = this.#memberBytes @@ -644,6 +915,11 @@ export class RepoSink implements TreeSink { async flush(): Promise { while (this.#pending.size > 0) await Promise.race(this.#pending) if (this.#error) throw this.#error + if (this.#records) { + const run = await this.#records.finish({ hold: this.opts.holdRecords ?? false }) + if (run) this.held.push(run) + this.#records = null + } } // Read-only, for measuring what a sink holds (scripts/memory.ts). @@ -686,3 +962,10 @@ export const recordPayloadBytes = (e: RecordEntry) => e.body?.length ?? 0 export const dropRecordBody = (e: RecordEntry) => { delete e.body } + +/** A leaf node's entry tuples, as their canonical JSON. */ +function leafTuples(json: string): string[] { + return (JSON.parse(json) as { e: unknown[] }).e.map((t) => JSON.stringify(t)) +} + +const tupleKey = (tuple: string) => (JSON.parse(tuple) as [string])[0] diff --git a/packages/protocol/src/repo/sync.ts b/packages/protocol/src/repo/sync.ts index 90264dd1..181c97c6 100644 --- a/packages/protocol/src/repo/sync.ts +++ b/packages/protocol/src/repo/sync.ts @@ -39,8 +39,8 @@ import { versionDigest, versionHash, } from '../format.js' -import { gunzipText, splitLines } from './gzip.js' -import { IntegrityError, keys, outOfLineHash, type Repo } from './repo.js' +import { gunzipText, gzip, splitLines } from './gzip.js' +import { IntegrityError, keys, OUT_OF_LINE_BYTES, outOfLineHash, type Repo } from './repo.js' /** Which sets a sync moves: `public`, or `all` (public and private). */ export type SyncSets = 'public' | 'all' @@ -177,17 +177,31 @@ export async function versionWork( * One tree's new objects, top down: nodes, and after each record leaf its * out-of-line records and body. `resumeKey` marks the objects after which a * walk can stop and resume with `after` (each leaf's last key). + * + * Layout F keeps no leaf bodies. A mirror copies its record files as they are + * (KeyedLayout.mirrorFiles), so by default its leaves come without one; with + * `bodies`, each is built from the leaf's stored records (out-of-line ones as + * pointers, as a body stores them), for a receiver that takes today's format. */ export async function* treeObjects( repo: Repo, tree: SyncTree, after: string | null = null, + opts: { bodies?: boolean } = {}, ): AsyncGenerator { const source = tree.kind === 'records' ? nodesOnly(repo, recordTree) : nodesOnly(repo, fileTree) for await (const n of newNodes(source as NodeSource, tree.base, tree.target, { after, })) { - const node = { key: keys.node(n.hash), bytes: await raw(repo, keys.node(n.hash)) } + // A receiver checks each node against its hash: one stored as a delta (node + // deltas) goes in full. A mirror copies it as stored, with the node it's against. + const node = { + key: keys.node(n.hash), + bytes: + opts.bodies && repo.nodeDeltas + ? await gzip(await repo.nodeJson(n.hash)) + : await raw(repo, keys.node(n.hash)), + } if (n.level !== 0) { yield node continue @@ -196,6 +210,26 @@ export async function* treeObjects( yield { ...node, resumeKey: n.lastKey } continue } + // A packed layout's records aren't in leaf bodies: the layout's own files carry + // them (KeyedLayout.mirrorFiles), but for records over OUT_OF_LINE_BYTES, objects + // of their own as in any layout. The leaf's entries say which (by size, the one + // threshold every writer uses), with no record read. + if (repo.records) { + const leaf = await repo.decoded(recordTree, n.hash) + const entries = leaf.kind === 'leaf' ? leaf.entries : [] + if (opts.bodies) yield node + for (const e of entries) + if (e.size > OUT_OF_LINE_BYTES) + yield { key: keys.record(e.hash), bytes: await raw(repo, keys.record(e.hash)) } + if (!opts.bodies) { + yield { ...node, resumeKey: n.lastKey } + continue + } + const lines = await repo.records[tree.set].leafLines(n.hash, entries) + const body = await gzip(lines.join('\n') + '\n') + yield { key: keys.body(n.hash), bytes: body, resumeKey: n.lastKey } + continue + } yield node const body = await raw(repo, keys.body(n.hash)) for (const line of splitLines(await gunzipText(body))) { @@ -215,7 +249,9 @@ export async function* packVersion( const work = await versionWork(repo, target, opts) for (const key of work.schemas) yield { key, bytes: await raw(repo, key) } for (const tree of work.trees) { - for await (const { key, bytes } of treeObjects(repo, tree)) yield { key, bytes } + // A receiver takes leaf bodies: layout F's are built from its records. + for await (const { key, bytes } of treeObjects(repo, tree, null, { bodies: true })) + yield { key, bytes } } if (work.privateSet) yield { key: work.privateSet, bytes: await raw(repo, work.privateSet) } yield { key: work.rootKey, bytes: await raw(repo, work.rootKey) } diff --git a/packages/protocol/src/stores/fs.ts b/packages/protocol/src/stores/fs.ts index 5265846d..03ba25b1 100644 --- a/packages/protocol/src/stores/fs.ts +++ b/packages/protocol/src/stores/fs.ts @@ -45,6 +45,9 @@ interface NodeModules { type GetBuiltin = (id: string) => unknown let nodeModules: NodeModules | undefined /** Node's modules, through `process.getBuiltinModule` so no bundler sees a `node:` import. */ +/** Objects up to this size are read whole by `get` (see there). */ +const EAGER_BYTES = 1024 * 1024 + const node = async (): Promise => { if (nodeModules) return nodeModules const get = (globalThis as { process?: { getBuiltinModule?: GetBuiltin } }).process @@ -63,6 +66,23 @@ const hex = (b: ArrayBuffer) => [...new Uint8Array(b)].map((x) => x.toString(16).padStart(2, '0')).join('') const unhex = (s: string) => new Uint8Array((s.match(/../g) ?? []).map((x) => parseInt(x, 16))) +/** + * Move a written temporary file to its key. With `ifAbsent`, a hard link, which + * fails if the key exists: a check first and a rename after would let two + * writers both win (two maintain jobs both holding the lease, round 12 at 20,000 + * collections), where R2's conditional put lets one. + */ +async function settle(fsp: typeof NodeFsp, tmp: string, path: string, ifAbsent?: boolean) { + if (!ifAbsent) return fsp.rename(tmp, path) + try { + await fsp.link(tmp, path) + } catch (err) { + if ((err as { code?: string }).code !== 'EEXIST') throw err + } finally { + await fsp.rm(tmp, { force: true }) + } +} + /** A store in a directory; with `presign`, one that presigns URLs served by the app. */ export function fileStore(root: string): FileStore export function fileStore(root: string, presign: FileStoreOptions): FileStore & PresigningStore @@ -122,6 +142,27 @@ export class FileStore implements Store { await fh.close() } } + // A small object is read now: deleted after this get, it still reads, as on R2 (a + // maintain job read a merge plan the merge's last job had just deleted). Large ones + // stay lazy: nothing deletes them while they're read (retired files wait out a grace). + if (size <= EAGER_BYTES) { + let buf: Uint8Array + try { + buf = await read() + } catch (err) { + // Deleted between the stat and the read: as if the get came after. + if ((err as { code?: string }).code === 'ENOENT') return null + throw err + } + return { + ...head, + get body() { + return new Blob([buf]).stream() + }, + bytes: async () => buf, + text: async () => new TextDecoder().decode(buf), + } + } return { ...head, get body() { @@ -157,7 +198,7 @@ export class FileStore implements Store { // Write then rename, so a reader never sees a partial object. const tmp = `${path}.${crypto.randomUUID()}.tmp` await fsp.writeFile(tmp, body) - await fsp.rename(tmp, path) + await settle(fsp, tmp, path, opts.ifAbsent) if (opts.contentType) await fsp.writeFile(path + '.__type', opts.contentType) } @@ -175,7 +216,7 @@ export class FileStore implements Store { await fsp.writeFile(tmp, new Uint8Array(0)) for await (const part of parts) await fsp.appendFile(tmp, part) await opts.check?.() - await fsp.rename(tmp, path) + await settle(fsp, tmp, path, opts.ifAbsent) } catch (err) { await fsp.rm(tmp, { force: true }) throw err @@ -193,18 +234,25 @@ export class FileStore implements Store { const { fsp, path: P } = await node() const root = P.resolve(this.root) const out: string[] = [] + // From the deepest folder the prefix names, into only the folders that can hold its + // keys: walking the whole bucket made a listing cost every file in it (the scale + // bench's 20,000 collections: most of each F maintain job, which lists a collection's + // manifests; R2 lists a prefix without that). + const start = P.resolve(root, prefix.slice(0, prefix.lastIndexOf('/') + 1)) + if (start !== root && !start.startsWith(root + P.sep)) return { keys: [], info: [] } const walk = async (dir: string) => { const entries = await fsp.readdir(dir, { withFileTypes: true }).catch(() => []) for (const e of entries) { const p = P.join(dir, e.name) - if (e.isDirectory()) await walk(p) - else if (!e.name.endsWith('.__type') && !e.name.endsWith('.tmp')) { - const key = P.relative(root, p).split(P.sep).join('/') + const key = P.relative(root, p).split(P.sep).join('/') + if (e.isDirectory()) { + if (`${key}/`.startsWith(prefix) || prefix.startsWith(`${key}/`)) await walk(p) + } else if (!e.name.endsWith('.__type') && !e.name.endsWith('.tmp')) { if (key.startsWith(prefix) && (!cursor || key > cursor)) out.push(key) } } } - await walk(root) + await walk(start) out.sort() const page = out.slice(0, 1000) const info = await Promise.all( diff --git a/packages/protocol/src/tree/assemble.ts b/packages/protocol/src/tree/assemble.ts index 84cd39de..37c7f9e6 100644 --- a/packages/protocol/src/tree/assemble.ts +++ b/packages/protocol/src/tree/assemble.ts @@ -95,8 +95,14 @@ export async function assembleTree( } stats.readNodes++ if (desc.level === 0) { - // Read one entry at a time: a leaf of large records is never held whole. - const it = leafEntryStream(source, desc.hash)[Symbol.asyncIterator]() + // Read one entry at a time: a leaf of large records is never held whole. In a + // layout that keeps records out of leaves (repo/layouts), its entries pass + // through without their records, as a merge's do; and a rewritten leaf may + // be stored against this one (node deltas). + sink.rewriting?.(desc.hash) + const it = (source.sparseStream ?? ((h: string) => leafEntryStream(source, h)))(desc.hash)[ + Symbol.asyncIterator + ]() const nextEntry = async () => { const r = await it.next() return r.done ? undefined : r.value diff --git a/packages/protocol/src/tree/builder.ts b/packages/protocol/src/tree/builder.ts index 8cae015e..8940c98e 100644 --- a/packages/protocol/src/tree/builder.ts +++ b/packages/protocol/src/tree/builder.ts @@ -48,6 +48,8 @@ export interface TreeSink { * waits on writes, so a leaf read isn't held up by the previous leaf's write. */ settle?(): Promise | undefined + /** The base leaf being rewritten now (node deltas diff the next leaf against it). */ + rewriting?(base: string): void } export interface BuilderOptions { diff --git a/packages/protocol/src/tree/merge.ts b/packages/protocol/src/tree/merge.ts index 3cf6f069..a5ef1611 100644 --- a/packages/protocol/src/tree/merge.ts +++ b/packages/protocol/src/tree/merge.ts @@ -54,6 +54,11 @@ export interface KeyRange { export interface MergeOptions extends BuilderOptions { /** Called for every effective change, in key order: (old, new). */ onChange?: (before: E | null, after: E | null) => void + /** + * Whether `onChange` needs an old entry's payload (default true). A layout that + * keeps records out of leaves then reads it; false skips that read. + */ + oldPayloads?: boolean /** * Merge only this range (a commit unit). Requires `leafOutput`, and every * change must lie in the range. @@ -97,6 +102,8 @@ export function emptyStats(): MergeStats { export class Peekable { readonly #it: Iterator | AsyncIterator #head: IteratorResult | undefined + /** Items taken and put back, served before the iterator. */ + #back: T[] = [] constructor(src: Iterable | AsyncIterable) { this.#it = @@ -106,15 +113,26 @@ export class Peekable { } async peek(): Promise { + if (this.#back.length) return this.#back[0] this.#head ??= await this.#it.next() return this.#head.done ? undefined : this.#head.value } async next(): Promise { + if (this.#back.length) return this.#back.shift() const v = await this.peek() this.#head = undefined return v } + + /** The items while `ok` holds, left in place: a look further ahead than peek. */ + async lookAhead(ok: (v: T) => boolean): Promise { + const out: T[] = [] + for (let v = await this.peek(); v !== undefined && ok(v); v = await this.peek()) + out.push((await this.next())!) + this.#back = [...out, ...this.#back] + return out + } } export async function mergeTree( @@ -167,9 +185,42 @@ export async function mergeTree( } } + // Layouts that keep records out of leaves give entries without bodies; a + // changed entry's old body is fetched only when a change callback needs it. + const sparse = source.sparseStream + const needOld = !!sparse && !!opts.onChange && !!source.withPayload && opts.oldPayloads !== false + const withOld = async (desc: NodeDesc, old: E) => + needOld ? source.withPayload!(desc.hash, old) : old + + /** + * A sparse leaf's old records that the changes landing in it replace or remove, + * read in one batch (file references need them), by key. + */ + const oldBodies = async (desc: NodeDesc, olds: E[]): Promise> => { + const keys = new Set( + (await pending.lookAhead((c) => compareUtf8(c.key, desc.lastKey) <= 0)).map((c) => c.key), + ) + const at = olds.flatMap((o, i) => (keys.has(spec.key(o)) ? [i] : [])) + if (!at.length) return new Map() + const got = await source.payloadsAt!(desc.hash, olds, at) + return new Map(got.map((e) => [spec.key(e), e])) + } + const rewriteLeaf = async (desc: NodeDesc) => { stats.readNodes++ - for await (const old of leafEntryStream(source, desc.hash)) { + sink.rewriting?.(desc.hash) + let entries: AsyncIterable | E[] = sparse + ? sparse(desc.hash) + : leafEntryStream(source, desc.hash) + let olds: Map | null = null + if (needOld && source.payloadsAt) { + const list: E[] = [] + for await (const e of entries) list.push(e) + olds = await oldBodies(desc, list) + entries = list + } + const withOldOf = async (old: E) => olds?.get(spec.key(old)) ?? withOld(desc, old) + for await (const old of entries) { const oldKey = spec.key(old) // A leaf straddling a unit's range edge: keep only the range's entries. if (range && !inRange(range, oldKey)) continue @@ -186,13 +237,13 @@ export async function mergeTree( await nextChange() if (!c.entry) { stats.removed++ - opts.onChange?.(old, null) + opts.onChange?.(await withOldOf(old), null) } else if (spec.same(old, c.entry)) { stats.unchanged++ await add(old) } else { stats.updated++ - opts.onChange?.(old, c.entry) + opts.onChange?.(await withOldOf(old), c.entry) await add(c.entry) } } else { diff --git a/packages/protocol/src/tree/read.ts b/packages/protocol/src/tree/read.ts index dce10b0a..3a55cbec 100644 --- a/packages/protocol/src/tree/read.ts +++ b/packages/protocol/src/tree/read.ts @@ -183,12 +183,16 @@ export async function* iterate( } return } + // A packed layout reads records only as they're yielded, 64 at a time: leaves + // ahead hold their entries, never their records. + const lazy = opts.payloads ? source.payloadsAt : undefined const load = (h: string) => - opts.payloads + opts.payloads && !lazy ? source.leafEntries(h) : source.node(h).then((n) => (n.kind === 'leaf' ? n.entries : [])) const ahead = Math.max(1, opts.prefetch ?? 4) - const queue: { entries: Promise; skip: number; after: string | undefined }[] = [] + const queue: { hash: string; entries: Promise; skip: number; after: string | undefined }[] = + [] const leaves = leavesFrom(source, await rootDesc(source, root), opts.after, opts.offset ?? 0) let exhausted = false const fill = async () => { @@ -200,7 +204,7 @@ export async function* iterate( } const entries = load(r.value.leaf.hash) entries.catch(() => {}) // surfaced when awaited below - queue.push({ entries, skip: r.value.skip, after: r.value.after }) + queue.push({ hash: r.value.leaf.hash, entries, skip: r.value.skip, after: r.value.after }) } } for (;;) { @@ -213,6 +217,61 @@ export async function* iterate( while (i < entries.length && compareUtf8(spec.key(entries[i]!), next.after) <= 0) i++ } i += next.skip + if (lazy) { + // Records in batches of up to 1 MB, read six at a time ahead of the reader, + // across leaves: about 6 MB of records ahead whatever the leaves (4 × 4 MB + // passed 128 MB with decoded strings at 12M), in enough parallel reads that a + // scan rarely waits (3 × 2 MB held as much but made exports 60% slower). + const leafBatches = function* (hash: string, list: E[], from: number) { + for (let at = from; at < list.length;) { + const idx: number[] = [] + let bytes = 0 + while ( + at < list.length && + idx.length < 2048 && + (idx.length === 0 || bytes < 1024 * 1024) + ) { + bytes += spec.bytes(list[at]!) + idx.push(at++) + } + yield { hash, list, idx } + } + } + const window: Promise[] = [] + const source1 = leafBatches(next.hash, entries, i) + const more = async function* () { + yield* source1 + // Then the leaves already queued behind this one, and the rest. + for (;;) { + await fill() + const n = queue.shift() + if (!n) return + const list = await n.entries + let j = 0 + if (n.after !== undefined) + while (j < list.length && compareUtf8(spec.key(list[j]!), n.after) <= 0) j++ + yield* leafBatches(n.hash, list, j + n.skip) + } + } + const batches = more() + const start = async () => { + const r = await batches.next() + if (r.done) return false + const p = lazy(r.value.hash, r.value.list, r.value.idx) + p.catch(() => {}) + window.push(p) + return true + } + while (window.length < 6 && (await start())) { + // Fill the window. + } + while (window.length) { + const got = await window.shift()! + await start() + yield* got + } + return + } for (; i < entries.length; i++) yield entries[i]! } } diff --git a/packages/protocol/src/tree/source.ts b/packages/protocol/src/tree/source.ts index 07443be6..df3431b8 100644 --- a/packages/protocol/src/tree/source.ts +++ b/packages/protocol/src/tree/source.ts @@ -18,6 +18,17 @@ export interface NodeSource { * large records is never held whole. Readers fall back to `leafEntries`. */ leafStream?(hash: string): AsyncIterable + /** + * A leaf's entries without payloads, for a merge in a layout that doesn't keep + * records in leaves (repo/layouts): unchanged entries pass through unread. + */ + readonly sparseStream?: ((hash: string) => AsyncIterable) | undefined + /** An entry from `sparseStream` with its payload (a changed entry's old version). */ + withPayload?(leafHash: string, e: E): Promise + /** Some of a leaf's entries with payloads, reading only those (packed layouts). */ + readonly payloadsAt?: + | ((leafHash: string, entries: readonly E[], at: readonly number[]) => Promise) + | undefined } /** A leaf's entries with payloads: streamed where the source can. */ diff --git a/packages/protocol/test/repo/layouts.test.ts b/packages/protocol/test/repo/layouts.test.ts new file mode 100644 index 00000000..3131e83e --- /dev/null +++ b/packages/protocol/test/repo/layouts.test.ts @@ -0,0 +1,630 @@ +/** + * Layout F's index pieces (repo/layouts): a layer's Bloom + * filter holds every key it was built from, and a lookup through the filters + * finds exactly what a range read of the same keys finds, newest layer winning. + */ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { + headConfig, + HomeStore, + KeyedLayout, + layoutConfig, + LiveHashes, + memoryStore, + type MergePlan, +} from '../../src/index.js' +import { + buildFilter, + filterHas, + levelFor, + LayeredIndex, + ManifestStore, + pendingConfig, + runs, + RunWriter, +} from '../../src/repo/layouts/kit.js' +import { repoLru } from '../../src/repo/repo.js' + +// F's manifest head is written at most once a second: tests wait a fortieth of that. +beforeEach(() => { + headConfig.intervalMs = 25 +}) +afterEach(() => { + headConfig.intervalMs = 1000 +}) + +/** A collection's layout F over `store`: its files under `runs//`. */ +const fLayout = (store = memoryStore(), home = 'c1', scope = 's') => + new KeyedLayout(new HomeStore(store, home), repoLru(8 << 20), scope) + +const ids = Array.from( + { length: 20_000 }, + (_, i) => `10.${1000 + (i % 50)}/x${String(i).padStart(6, '0')}`, +).sort() + +describe('layer filters', () => { + it('hold every key they were built from', () => { + const keys = ids.slice(0, 12_000) + const f = buildFilter(new Set(keys), keys.length) + expect(keys.filter((k) => !filterHas(f, k))).toEqual([]) + }) + + it('reject most keys they were not built from', () => { + const f = buildFilter(new Set(ids.slice(0, 10_000)), 10_000) + const false_ = ids.slice(10_000).filter((k) => filterHas(f, k)).length + expect(false_ / 10_000).toBeLessThan(0.03) + }) +}) + +describe('reclaim a key range at a time', () => { + // Small lines make packs merges still take (listed in the manifest); big ones make + // packs no merge takes (listed in the registry instead). + const noise = (i: number) => { + let x = (i + 1) * 2_654_435_761 + let out = '' + for (let j = 0; j < 400; j++) { + x = (x * 1_103_515_245 + 12_345) % 2_147_483_648 + out += x.toString(36) + } + return out + } + it.each([ + ['small packs', false], + ['registry packs', true], + ])('keeps exactly the live records, rewriting mostly-dead packs (%s)', async (_, big) => { + const layout = fLayout() + const n = 3_000 + const entry = (i: number, v: number) => ({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 }) + const line = (i: number, v: number) => `line ${v}.${i}${big ? ` ${noise(i * 7 + v)}` : ''}` + const write = async (from: number, to: number, v: number) => { + const w = layout.writer() + for (let i = from; i < to; i++) w.add(entry(i, v), line(i, v)) + await w.finish() + } + await write(0, n, 1) + // Most of the first write's records change: its packs become mostly dead. + await write(0, 2_000, 2) + const listed = (await layout.manifests.latest()).packs.length + const registered = (await layout.registry.read()).packs.length + if (big) expect(registered).toBeGreaterThan(0) + else expect(registered).toBe(0) + expect(listed + registered).toBe((await layout.allPacks()).length) + + const live = new LiveHashes() + const want = Array.from({ length: n }, (_, i) => entry(i, i < 2_000 ? 2 : 1)) + for (const e of want) live.add(e.key, e.hash) + expect(live.has(want[0]!.key, 'h1.0')).toBe(false) + expect(live.has(want[0]!.key, want[0]!.hash)).toBe(true) + + const plan = await layout.reclaimPlan(500) + expect(plan.ranges.length).toBeGreaterThan(3) + const done = [] + for (let i = 0; i < plan.ranges.length; i++) done.push(await layout.reclaimRange(plan, i, live)) + expect(done.at(-1)).toBe(true) + expect(done.slice(0, -1).every((d) => !d)).toBe(true) + const swapped = await layout.reclaimSwap(plan) + expect(swapped?.stats.entriesKept).toBe(n) + expect(swapped?.stats.entriesDropped).toBe(2_000) + expect(swapped!.rewrite.length).toBeGreaterThan(0) + // A second caller doesn't swap again. + expect(await layout.reclaimSwap(plan)).toBeNull() + for (const id of swapped!.rewrite) await layout.reclaimRewrite([id]) + + const packs = await layout.allPacks() + expect(packs.some((p) => swapped!.rewrite.includes(p.id))).toBe(false) + expect(packs.reduce((s, p) => s + p.records, 0)).toBe(n) + const lines = await layout.leafLines('leaf', want) + expect(lines).toEqual(want.map((_, i) => line(i, i < 2_000 ? 2 : 1))) + }) +}) + +describe('concurrent commits', () => { + it('are none of them lost, and a merge removes only what it read', async () => { + const layout = fLayout() + const entry = (i: number) => ({ key: ids[i]!, hash: `h.${i}`, size: 10 }) + await Promise.all( + Array.from({ length: 12 }, async (_, k) => { + const w = layout.writer() + for (let i = k * 100; i < k * 100 + 100; i++) w.add(entry(i), `line ${i}`) + await w.finish() + }), + ) + expect((await layout.manifests.latest()).layers.length).toBe(12) + await layout.maintain() + const want = Array.from({ length: 1_200 }, (_, i) => entry(i)) + expect(await layout.leafLines('leaf', want)).toEqual(want.map((_, i) => `line ${i}`)) + expect((await layout.manifests.latest()).layers.length).toBeLessThan(12) + }) +}) + +describe('size-tiered index levels', () => { + it('merge small commits with each other, leaving a large run alone', async () => { + const layout = fLayout() + const entry = (i: number) => ({ key: ids[i]!, hash: `h.${i}`, size: 10 }) + const write = async (from: number, n: number) => { + const w = layout.writer() + for (let i = from; i < from + n; i++) w.add(entry(i), `line ${i}`) + await w.finish() + } + await write(0, 8_000) + const big = (await layout.manifests.latest()).layers[0]! + expect(big.level).toBe(levelFor(8_000)) + for (let k = 0; k < 4; k++) await write(10_000 + k * 5, 5) + await layout.maintain() + const m = await layout.manifests.latest() + // The large run is untouched; the four small ones are one run now. + expect(m.layers.some((l) => l.id === big.id)).toBe(true) + expect(m.layers.length).toBe(2) + const want = [...Array.from({ length: 8_000 }, (_, i) => i), 10_000, 10_019].map(entry) + expect(await layout.leafLines('leaf', want)).toEqual( + want.map((e) => `line ${ids.indexOf(e.key)}`), + ) + }) +}) + +describe('merges keep the newest place of every record', () => { + // Random commits (new records, and new versions of old ones), each followed by the + // merges it sets off, with replaced packs deleted at once: every record's newest + // version must read, whatever merged with what. + it('through random commits and merges', async () => { + const saved = layoutConfig.graceMs + layoutConfig.graceMs = 0 + // The head moves at most this often: still behind the manifests merges write. + headConfig.intervalMs = 25 + try { + const layout = fLayout() + let x = 12_345 + const rand = (n: number) => { + // xorshift32: exact in 32-bit integers. + x ^= x << 13 + x ^= x >>> 17 + x ^= x << 5 + return (x >>> 0) % n + } + const version = new Map() + const top = new Map() + for (let c = 0; c < 150; c++) { + // Mostly small commits, now and then a large one. + const size = rand(10) === 0 ? 400 + rand(800) : 1 + rand(40) + const touched = new Set() + while (touched.size < size) touched.add(rand(3_000)) + const w = layout.writer() + for (const i of [...touched].sort((a, b) => a - b)) { + // A third of changes go back to an earlier version: the same record, written again. + const now = version.get(i) ?? 0 + const v = now > 1 && rand(3) === 0 ? 1 + rand(now - 1) : (top.get(i) ?? 0) + 1 + if (v > (top.get(i) ?? 0)) top.set(i, v) + version.set(i, v) + w.add({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 }, `line ${v}.${i}`) + } + await w.finish() + // Several maintain jobs at once, as many commits' jobs would be: one does the work. + await Promise.all([layout.maintain(), layout.maintain(), layout.maintain()]) + } + await layout.maintain() + // However the commits came, the index stays a bounded number of runs. + const m = await layout.manifests.latest() + expect(new Set(m.layers.map((l) => l.run ?? l.id)).size).toBeLessThanOrEqual(13) + const live = [...version].sort((a, b) => a[0] - b[0]) + const want = live.map(([i, v]) => ({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 })) + expect(await layout.leafLines('leaf', want)).toEqual(live.map(([i, v]) => `line ${v}.${i}`)) + } finally { + layoutConfig.graceMs = saved + headConfig.intervalMs = 1000 + } + }) +}) + +describe('merges split into key-range jobs', () => { + it('keep every record readable, with commits landing while a split merge runs', async () => { + const saved = { ...layoutConfig } + layoutConfig.graceMs = 0 + layoutConfig.mergeJobEntries = 300 + try { + const layout = fLayout() + let x = 99 + const rand = (n: number) => { + x ^= x << 13 + x ^= x >>> 17 + x ^= x << 5 + return (x >>> 0) % n + } + const pending: MergePlan[] = [] + let splits = 0 + const split = async (plan: MergePlan) => { + pending.push(plan) + splits++ + } + // A split merge's range jobs run in any order, some time after it was planned. + const runOne = async () => { + const plan = pending.shift() + if (!plan) return + const order = plan.ranges.map((_, i) => i).sort(() => rand(3) - 1) + for (const i of order) if (await layout.mergeRange(plan, i)) await layout.mergeSwap(plan) + } + const version = new Map() + let most = 0 + for (let c = 0; c < 80; c++) { + const size = rand(8) === 0 ? 300 + rand(400) : 1 + rand(30) + const touched = new Set() + while (touched.size < size) touched.add(rand(3_000)) + const w = layout.writer() + for (const i of [...touched].sort((a, b) => a - b)) { + const now = version.get(i) ?? 0 + const v = now > 1 && rand(3) === 0 ? 1 + rand(now - 1) : now + 1 + version.set(i, v) + w.add({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 }, `line ${v}.${i}`) + } + await w.finish() + await layout.maintain({ split }) + most = Math.max(most, runs((await layout.manifests.latest()).layers).length) + // Range jobs lag far behind commits (a busy queue), then catch up. + if (rand(8) === 0) while (pending.length) await runOne() + } + while (pending.length) await runOne() + await layout.maintain({ split }) + while (pending.length) await runOne() + expect(splits).toBeGreaterThan(0) + // Runs no split merge holds keep merging while one waits: 19 at most here, + // 33 when every merge waited for it (round 12). + expect(most).toBeLessThanOrEqual(20) + const live = [...version].sort((a, b) => a[0] - b[0]) + const want = live.map(([i, v]) => ({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 })) + expect(await layout.leafLines('leaf', want)).toEqual(live.map(([i, v]) => `line ${v}.${i}`)) + } finally { + Object.assign(layoutConfig, saved) + } + }) +}) + +describe('a backlog of tiny commits', () => { + it('merges in a few passes, with pack merges among them', async () => { + // Many collections committing at once: hundreds of tiny runs and packs land + // while one maintain job works. Pack merges add mid-sized runs among them. + const store = memoryStore() + const layout = fLayout(store) + let x = 7 + const rand = (n: number) => { + x ^= x << 13 + x ^= x >>> 17 + x ^= x << 5 + return (x >>> 0) % n + } + const version = new Map() + let written = 0 + const put = store.put.bind(store) + store.put = (key, body, opts) => { + if (key.startsWith('runs/c1/layers/') && key.endsWith('.dir')) written++ + return put(key, body, opts) + } + for (let round = 0; round < 4; round++) { + for (let c = 0; c < 120; c++) { + const w = layout.writer() + const touched = new Set() + const n = 5 + rand(8) + while (touched.size < n) touched.add(rand(20_000)) + for (const i of [...touched].sort((a, b) => a - b)) { + const v = (version.get(i) ?? 0) + 1 + version.set(i, v) + w.add({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 }, `line ${v}.${i}`) + } + await w.finish() + } + written = 0 + await layout.maintain() + // A few merges catch up (18–33 layer files when four runs merged at a time). + expect(written).toBeLessThanOrEqual(6) + expect(runs((await layout.manifests.latest()).layers).length).toBeLessThanOrEqual(10) + } + const live = [...version].sort((a, b) => a[0] - b[0]) + const want = live.map(([i, v]) => ({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 })) + expect(await layout.leafLines('leaf', want)).toEqual(live.map(([i, v]) => `line ${v}.${i}`)) + }) +}) + +describe('pack merges', () => { + it('reach every tier while commits keep landing', async () => { + // One pass a job, with commits between: tier 0 always has enough (FANOUT) to merge. + const saved = { ...layoutConfig } + layoutConfig.maintainBudgetMs = 0 + try { + const layout = fLayout() + let next = 0 + let most = 0 + for (let job = 0; job < 120; job++) { + for (let c = 0; c < 10; c++) { + const w = layout.writer() + for (let i = 0; i < 6; i++, next++) + w.add({ key: ids[next % ids.length]!, hash: `h${next}`, size: 10 }, `line ${next}`) + await w.finish() + } + await layout.maintain() + most = Math.max(most, (await layout.manifests.latest()).packs.length) + } + // 20 at most; 120 and rising by one a job when only the lowest tier with enough merged. + expect(most).toBeLessThanOrEqual(30) + } finally { + Object.assign(layoutConfig, saved) + } + }) +}) + +describe('layout F', () => { + const rec = (i: number, v = 1) => ({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 }) + const line = (i: number, v = 1) => `line ${v}.${i}` + + it("writes no key twice when a push's units finish together", async () => { + // R2 refuses a second write to a key within a second: a push's units finishing + // at once must not each rewrite the collection's manifest. + const root = memoryStore() + const writes = new Map() + const put = root.put.bind(root) + root.put = (key, body, opts) => { + writes.set(key, (writes.get(key) ?? 0) + 1) + return put(key, body, opts) + } + const layout = fLayout(root) + await Promise.all( + Array.from({ length: 40 }, async (_, u) => { + const w = layout.writer() + for (let i = u * 50; i < u * 50 + 50; i++) w.add(rec(i), line(i)) + await w.finish() + }), + ) + expect([...writes].filter(([, n]) => n > 1)).toEqual([]) + // Readable before any merge folds them in: a reader that misses looks at pending runs. + const all = Array.from({ length: 2000 }, (_, i) => i) + expect( + await layout.leafLines( + 'leaf', + all.map((i) => rec(i)), + ), + ).toEqual(all.map((i) => line(i))) + await layout.maintain() + expect((await root.list('runs/c1/manifest/f/pending/')).keys).toEqual([]) + expect( + await layout.leafLines( + 'leaf', + all.map((i) => rec(i)), + ), + ).toEqual(all.map((i) => line(i))) + }) + + it('forks by reference: nothing copied, and each side keeps what the other reads', async () => { + const saved = layoutConfig.graceMs + layoutConfig.graceMs = 0 + headConfig.intervalMs = 25 + try { + const root = memoryStore() + const open = (id: string) => fLayout(root, id, `s/${id}`) + const parent = open('p') + const commit = async (l: KeyedLayout, from: number, to: number, v: number) => { + const w = l.writer() + for (let i = from; i < to; i++) w.add(rec(i, v), line(i, v)) + await w.finish() + await l.maintain() + } + for (let c = 0; c < 12; c++) await commit(parent, c * 100, c * 100 + 100, 1) + const bytes = async (prefix: string) => { + let n = 0 + for (const k of (await root.list(prefix)).keys) n += (await root.get(k))!.size + return n + } + const before = await bytes('runs/fork/') + expect(before).toBe(0) + const fork = open('fork') + const r = await parent.forkInto(fork) + // A list of what it reads where, and a marker in the parent: no pack or index copied. + expect(r.bytes).toBeLessThan(20_000) + expect((await root.list('runs/fork/packs/')).keys).toEqual([]) + const base = Array.from({ length: 1200 }, (_, i) => i) + expect( + await fork.leafLines( + 'leaf', + base.map((i) => rec(i)), + ), + ).toEqual(base.map((i) => line(i))) + // Both go on: the parent merges its runs and packs away, sweeping at once. + for (let c = 0; c < 10; c++) { + await commit(parent, c * 30, c * 30 + 30, 2) + await commit(fork, 600 + c * 30, 600 + c * 30 + 30, 3) + } + expect( + await fork.leafLines( + 'leaf', + base.map((i) => rec(i)), + ), + ).toEqual(base.map((i) => line(i))) + // The parent is deleted: what the fork reads stays, the rest goes. + const gone = new Set() + const deleted = async (id: string) => gone.has(id) + gone.add('p') + const dropped = await KeyedLayout.dropCollection(root, 'p', deleted) + expect(dropped.kept).toBeGreaterThan(0) + const forkView = open('fork') + expect( + await forkView.leafLines( + 'leaf', + base.map((i) => rec(i)), + ), + ).toEqual(base.map((i) => line(i))) + // Then the fork: nothing of either is left. + gone.add('fork') + await KeyedLayout.dropCollection(root, 'fork', deleted) + expect((await root.list('runs/')).keys).toEqual([]) + } finally { + layoutConfig.graceMs = saved + headConfig.intervalMs = 1000 + } + }) + + it("keeps a maintain job within a Worker's storage calls, queueing the rest", async () => { + // A 12M load's 354 units left one maintain job ~10,460 storage calls (round 19): + // Workers allow 10,000 subrequests an invocation. + const saved = { ...layoutConfig } + layoutConfig.graceMs = 0 + layoutConfig.maintainCalls = 400 + layoutConfig.mergeInputs = 32 + pendingConfig.deletesPerUpdate = 64 + headConfig.intervalMs = 25 + try { + const root = memoryStore() + let calls = 0 + const io = new Set(['get', 'head', 'put', 'list', 'delete']) + const counting = new Proxy(root, { + get(t, k) { + const v = Reflect.get(t, k, t) + if (typeof v !== 'function') return v + return io.has(String(k)) + ? (...a: unknown[]) => { + calls++ + return v.apply(t, a) + } + : v.bind(t) + }, + }) + const layout = fLayout(counting) + // A load's units all land before its first maintain job. + for (let u = 0; u < 300; u++) { + const w = layout.writer() + for (let i = u * 5; i < u * 5 + 5; i++) w.add(rec(i), line(i)) + await w.finish() + } + // Each job queues the next while work is left, as the server's handler does. + const jobs: number[] = [] + for (let more = true; more && jobs.length < 100;) { + const before = calls + more = (await layout.maintain()) === true + jobs.push(calls - before) + } + // Within the budget, but for the pass that crossed it. + expect(Math.max(...jobs)).toBeLessThanOrEqual(layoutConfig.maintainCalls * 2) + expect((await root.list('runs/c1/manifest/f/pending/')).keys).toEqual([]) + const all = Array.from({ length: 1500 }, (_, i) => i) + expect( + await layout.leafLines( + 'leaf', + all.map((i) => rec(i)), + ), + ).toEqual(all.map((i) => line(i))) + } finally { + Object.assign(layoutConfig, saved) + pendingConfig.deletesPerUpdate = 500 + headConfig.intervalMs = 1000 + } + }) +}) + +describe("F's manifest head", () => { + // Split merges' swaps, reclaim and forks settle the head from their own jobs, beside + // the maintain job: when it was last written must come from the head, not from an + // isolate's memory (round 17's quick suite: one head write R2 would refuse). + it('is written at most once an interval, whichever isolate writes it', async () => { + headConfig.intervalMs = 200 + try { + const store = memoryStore() + const writes: number[] = [] + const watched = new Proxy(store, { + get(t, k, r) { + const v = Reflect.get(t, k, r) + if (k === 'put') + return (key: string, ...rest: unknown[]) => { + if (key.endsWith('/head.json')) writes.push(Date.now()) + return v.call(t, key, ...rest) + } + return typeof v === 'function' ? v.bind(t) : v + }, + }) + const isolate = () => new ManifestStore(watched, 'f') + const a = isolate() + await a.update((m) => m) + // Another isolate, which has never read the head, changes the manifest at once. + const b = isolate() + await b.update((m) => m) + // Then several jobs end together, each bringing the head up to date. + await Promise.all([a.settle(), b.settle(), isolate().settle(), isolate().settle()]) + const gaps = writes.slice(1).map((t, i) => t - writes[i]!) + expect(gaps.every((g) => g >= headConfig.intervalMs - 2)).toBe(true) + // And the head ends at the newest manifest. + const latest = await isolate().latest() + expect((await isolate().current()).seq).toBe(latest.seq) + } finally { + headConfig.intervalMs = 1000 + } + }) +}) + +describe('a split merge whose inputs went', () => { + it('drops itself, its outputs too, instead of failing until abandoned', async () => { + const saved = { ...layoutConfig } + layoutConfig.graceMs = 0 + try { + const store = memoryStore() + const layout = fLayout(store) + for (let c = 0; c < 6; c++) { + const w = layout.writer() + for (let i = c * 10; i < c * 10 + 10; i++) + w.add({ key: ids[i]!, hash: `h${i}`, size: 10 }, `line ${i}`) + await w.finish() + } + const plan = await layout.mergePlan((await layout.manifests.latest()).layers, 20) + expect(plan.ranges.length).toBeGreaterThan(1) + expect(await layout.mergeRange(plan, 0)).toBe(false) + // Another maintain merges the same runs first (a lease two jobs both held). + await layout.maintain() + expect(await layout.mergeRange(plan, 1)).toBe(false) + expect(await layout.mergePlanById(plan.id)).toBeNull() + expect((await store.list('runs/c1/merges/')).keys).toEqual([]) + const m = await layout.manifests.latest() + const want = Array.from({ length: 60 }, (_, i) => ({ key: ids[i]!, hash: `h${i}`, size: 10 })) + expect(await layout.leafLines('leaf', want)).toEqual(want.map((_, i) => `line ${i}`)) + // No layer file is left but the manifest's and those it retired (the merge's inputs). + const id = (k: string) => k.slice('layers/'.length).split('.')[0]! + const files = new Set( + (await store.list('runs/c1/layers/')).keys.map((k) => id(k.slice('runs/c1/'.length))), + ) + const known = [ + ...m.layers.map((l) => l.id), + ...m.retired.filter((r) => r.key.startsWith('layers/')).map((r) => id(r.key)), + ] + expect([...files].sort()).toEqual([...new Set(known)].sort()) + } finally { + Object.assign(layoutConfig, saved) + } + }) +}) + +describe('layered index lookups', () => { + it('find what a range read finds, newest layer winning', async () => { + const store = memoryStore() + const manifests = new ManifestStore(store, 'f') + const layer = async (from: number, to: number, tag: string, level: number, seq: number) => { + const w = new RunWriter(store, level) + for (let i = from; i < to; i++) w.add(`${ids[i]}\u0000h${i % 7}`, `${tag}.${i}`) + const files = await w.finish(seq) + await manifests.update((m) => { + m.layers.push(...files) + return m + }) + } + await layer(0, 12_000, 'a', 1, 1) + await layer(8_000, 20_000, 'b', 0, 2) + await layer(5_000, 6_000, 'c', 0, 3) + const index = new LayeredIndex(store, repoLru(8 << 20), 's', manifests) + const m = await manifests.current() + for (const i of [0, 1, 4_999, 5_000, 5_500, 7_999, 8_000, 11_999, 12_000, 19_999]) { + const id = ids[i]! + const looked = await index.lookup(m, [id]) + const ranged = await index.range(m, `${id}\u0000`, `${id}\u0000￿`) + expect([...looked].sort()).toEqual([...ranged].sort()) + expect(looked.size).toBe(1) + } + const many = ids.slice(4_990, 5_010) + const looked = await index.lookup(m, many) + const ranged = await index.range(m, `${many[0]}\u0000`, `${many.at(-1)}\u0000￿`) + expect([...looked].sort()).toEqual([...ranged].sort()) + // The newest layer's value wins where layers overlap. + expect([...(await index.lookup(m, [ids[5_500]!])).values()][0]).toMatch(/^c\./) + }) +}) diff --git a/packages/protocol/test/stores/contract.test.ts b/packages/protocol/test/stores/contract.test.ts index 7cdd839f..b6f90cc3 100644 --- a/packages/protocol/test/stores/contract.test.ts +++ b/packages/protocol/test/stores/contract.test.ts @@ -123,6 +123,18 @@ for (const [name, make] of makers) { expect(await (await store.get('fresh'))!.text()).toBe('only') }) + it('lets one of several concurrent ifAbsent writers win, for good', async () => { + // A lease is an ifAbsent put read back: every writer must read the same winner. + const seen = await Promise.all( + Array.from({ length: 8 }, async (_, i) => { + await store.put('lease', `w${i}`, { ifAbsent: true }) + return (await store.get('lease'))!.text() + }), + ) + const won = await (await store.get('lease'))!.text() + expect(seen).toEqual(seen.map(() => won)) + }) + it('streams bodies', async () => { await store.put('stream', enc.encode('x'.repeat(100_000))) const obj = await store.get('stream') diff --git a/packages/protocol/test/stores/fs-get.test.ts b/packages/protocol/test/stores/fs-get.test.ts new file mode 100644 index 00000000..66a37ff4 --- /dev/null +++ b/packages/protocol/test/stores/fs-get.test.ts @@ -0,0 +1,30 @@ +/** + * An object the file store returned stays readable when deleted after the get, as + * on R2: a maintain job listing split merges read a plan the merge's last job had + * just deleted (round 20, ENOENT from the body's read). + */ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { afterEach, describe, expect, it } from 'vitest' + +import { fileStore } from '../../src/stores/fs.js' + +describe('fileStore get', () => { + const dirs: string[] = [] + afterEach(() => { + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) + }) + + it('reads a small object deleted after the get', async () => { + const root = mkdtempSync(join(tmpdir(), 'ul-fs-get-')) + dirs.push(root) + const store = fileStore(root) + await store.put('merges/m1/plan', '{"id":"m1"}') + const o = await store.get('merges/m1/plan') + await store.delete('merges/m1/plan') + expect(await o!.text()).toBe('{"id":"m1"}') + expect(await store.get('merges/m1/plan')).toBeNull() + }) +}) diff --git a/packages/protocol/test/stores/fs-list.test.ts b/packages/protocol/test/stores/fs-list.test.ts new file mode 100644 index 00000000..7562e680 --- /dev/null +++ b/packages/protocol/test/stores/fs-list.test.ts @@ -0,0 +1,50 @@ +/** + * The file store lists by prefix without walking the rest of the bucket: listing one + * collection's manifests walked every file (20,000 collections in the scale bench: + * most of each F maintain job's time, a cost R2's prefix listing doesn't have). + */ +import { mkdtempSync, rmSync } from 'node:fs' +import { tmpdir } from 'node:os' +import { join } from 'node:path' + +import { afterEach, describe, expect, it, vi } from 'vitest' + +import { fileStore } from '../../src/stores/fs.js' + +const fsp = process.getBuiltinModule('node:fs/promises') + +describe('fileStore list', () => { + const dirs: string[] = [] + afterEach(() => { + vi.restoreAllMocks() + for (const d of dirs.splice(0)) rmSync(d, { recursive: true, force: true }) + }) + + it('reads only the folders that can hold the prefix', async () => { + const root = mkdtempSync(join(tmpdir(), 'ul-fs-list-')) + dirs.push(root) + const store = fileStore(root) + const keys = [ + 'repo/runs/a/manifest/f/000000000001.json', + 'repo/runs/a/manifest/f/000000000002.json', + 'repo/runs/a/manifest/f/pending/1-x.json', + 'repo/runs/a/packs/p.pk', + 'repo/runs/b/manifest/f/000000000001.json', + 'repo/nodes/zz', + ] + for (const k of keys) await store.put(k, 'x') + const read = vi.spyOn(fsp, 'readdir') + expect((await store.list('repo/runs/a/manifest/f/0')).keys).toEqual(keys.slice(0, 2)) + const walked = read.mock.calls.map((c) => String(c[0])) + expect(walked.length).toBeGreaterThan(0) + expect(walked.every((d) => d === join(root, 'repo/runs/a/manifest/f'))).toBe(true) + // The same keys as before, whatever the prefix's shape. + expect((await store.list('repo/runs/a/')).keys).toEqual(keys.slice(0, 4)) + expect((await store.list('repo/run')).keys).toEqual(keys.slice(0, 5)) + expect((await store.list('')).keys).toEqual([...keys].sort()) + expect((await store.list('repo/runs/c/')).keys).toEqual([]) + expect( + (await store.list('repo/runs/a/', 'repo/runs/a/manifest/f/000000000002.json')).keys, + ).toEqual(keys.slice(2, 4)) + }) +}) diff --git a/packages/server/drizzle/0025_record_layout.sql b/packages/server/drizzle/0025_record_layout.sql new file mode 100644 index 00000000..b1f10425 --- /dev/null +++ b/packages/server/drizzle/0025_record_layout.sql @@ -0,0 +1,3 @@ +ALTER TABLE `collection_tombstones` ADD `record_layout` text DEFAULT 'today' NOT NULL;--> statement-breakpoint +ALTER TABLE `collection_tombstones` ADD `records_dropped_at` integer;--> statement-breakpoint +ALTER TABLE `collections` ADD `record_layout` text DEFAULT 'today' NOT NULL; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0025_snapshot.json b/packages/server/drizzle/meta/0025_snapshot.json new file mode 100644 index 00000000..0efc6caa --- /dev/null +++ b/packages/server/drizzle/meta/0025_snapshot.json @@ -0,0 +1,3583 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "42b4a563-0b06-417e-a83f-2216ffba5eb8", + "prevId": "8d26270d-9b7c-4470-bde3-4e4c122b30fb", + "tables": { + "abuse_reports": { + "name": "abuse_reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "contact": { + "name": "contact", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reporter_id": { + "name": "reporter_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "resolved_by": { + "name": "resolved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "abuse_reports_status_idx": { + "name": "abuse_reports_status_idx", + "columns": ["status", "created_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 86400000 + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 10 + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_request": { + "name": "last_request", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": ["key"], + "isUnique": false + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": ["reference_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_collections": { + "name": "ark_collections", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "ark_id": { + "name": "ark_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "custom_url": { + "name": "custom_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_collections_ark_id_unique": { + "name": "ark_collections_ark_id_unique", + "columns": ["ark_id"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_collections_collection_id_collections_id_fk": { + "name": "ark_collections_collection_id_collections_id_fk", + "tableFrom": "ark_collections", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_record_types": { + "name": "ark_record_types", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirect_url_field": { + "name": "redirect_url_field", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "ark_record_types_collection_id_collections_id_fk": { + "name": "ark_record_types_collection_id_collections_id_fk", + "tableFrom": "ark_record_types", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "ark_record_types_collection_id_record_type_pk": { + "columns": ["collection_id", "record_type"], + "name": "ark_record_types_collection_id_record_type_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_shoulders": { + "name": "ark_shoulders", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "shoulder": { + "name": "shoulder", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_shoulders_shoulder_unique": { + "name": "ark_shoulders_shoulder_unique", + "columns": ["shoulder"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_shoulders_organization_id_organization_id_fk": { + "name": "ark_shoulders_organization_id_organization_id_fk", + "tableFrom": "ark_shoulders", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cleanup_runs": { + "name": "cleanup_runs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "step": { + "name": "step", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dry_run": { + "name": "dry_run", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "requested_by": { + "name": "requested_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mark_run_id": { + "name": "mark_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lease": { + "name": "lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stats": { + "name": "stats", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "cleanup_runs_created_idx": { + "name": "cleanup_runs_created_idx", + "columns": ["created_at"], + "isUnique": false + }, + "cleanup_runs_step_idx": { + "name": "cleanup_runs_step_idx", + "columns": ["step", "status"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_tombstones": { + "name": "collection_tombstones", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "versions": { + "name": "versions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_by": { + "name": "deleted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_layout": { + "name": "record_layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'today'" + }, + "records_dropped_at": { + "name": "records_dropped_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_webhooks": { + "name": "collection_webhooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bump_filter": { + "name": "bump_filter", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_delivery_at": { + "name": "last_delivery_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "collection_webhooks_collection_idx": { + "name": "collection_webhooks_collection_idx", + "columns": ["collection_id"], + "isUnique": false + } + }, + "foreignKeys": { + "collection_webhooks_collection_id_collections_id_fk": { + "name": "collection_webhooks_collection_id_collections_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collections": { + "name": "collections", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "head_version_id": { + "name": "head_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_salt": { + "name": "private_salt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_files_root": { + "name": "public_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "version_count": { + "name": "version_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "history_bytes": { + "name": "history_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_push_at": { + "name": "last_push_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_started_at": { + "name": "reconcile_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_state": { + "name": "reconcile_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_seq": { + "name": "reconciled_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "reconciled_files_root": { + "name": "reconciled_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_layout": { + "name": "record_layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'today'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "collections_org_slug_uq": { + "name": "collections_org_slug_uq", + "columns": ["organization_id", "slug"], + "isUnique": true + }, + "collections_public_updated_idx": { + "name": "collections_public_updated_idx", + "columns": ["public", "updated_at"], + "isUnique": false + } + }, + "foreignKeys": { + "collections_organization_id_organization_id_fk": { + "name": "collections_organization_id_organization_id_fk", + "tableFrom": "collections", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "commit_units": { + "name": "commit_units", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "plan_id": { + "name": "plan_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ord": { + "name": "ord", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "after": { + "name": "after", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "through": { + "name": "through", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "gap": { + "name": "gap", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "slices_key": { + "name": "slices_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_key": { + "name": "output_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "queued_at": { + "name": "queued_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "commit_units_plan_idx": { + "name": "commit_units_plan_idx", + "columns": ["plan_id", "set", "type", "ord"], + "isUnique": false + }, + "commit_units_status_idx": { + "name": "commit_units_status_idx", + "columns": ["session_id", "status", "queued_at"], + "isUnique": false + } + }, + "foreignKeys": { + "commit_units_session_id_push_sessions_id_fk": { + "name": "commit_units_session_id_push_sessions_id_fk", + "tableFrom": "commit_units", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "denylist": { + "name": "denylist", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "file_uploads": { + "name": "file_uploads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "multipart_upload_id": { + "name": "multipart_upload_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "file_uploads_hash_idx": { + "name": "file_uploads_hash_idx", + "columns": ["hash"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "files": { + "name": "files", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "forks": { + "name": "forks", + "columns": { + "child_collection_id": { + "name": "child_collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "parent_collection_id": { + "name": "parent_collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_seq": { + "name": "parent_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'public'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "forks_child_collection_id_collections_id_fk": { + "name": "forks_child_collection_id_collections_id_fk", + "tableFrom": "forks", + "tableTo": "collections", + "columnsFrom": ["child_collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instance_settings": { + "name": "instance_settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invitation": { + "name": "invitation", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "jobs": { + "name": "jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "run_at": { + "name": "run_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locked_until": { + "name": "locked_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "jobs_ready_idx": { + "name": "jobs_ready_idx", + "columns": ["status", "run_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "member": { + "name": "member", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + }, + "member_user_id_idx": { + "name": "member_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organization": { + "name": "organization", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ark_naan": { + "name": "ark_naan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kf_org_id": { + "name": "kf_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "columns": ["slug"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "placements": { + "name": "placements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location_id": { + "name": "location_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "synced_seq": { + "name": "synced_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until": { + "name": "lease_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "placements_one_primary_uq": { + "name": "placements_one_primary_uq", + "columns": ["collection_id"], + "isUnique": true, + "where": "\"placements\".\"role\" = 'primary' AND \"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_collection_location_uq": { + "name": "placements_collection_location_uq", + "columns": ["collection_id", "location_id"], + "isUnique": true, + "where": "\"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_org_location_uq": { + "name": "placements_org_location_uq", + "columns": ["organization_id", "location_id"], + "isUnique": true, + "where": "\"placements\".\"organization_id\" IS NOT NULL" + }, + "placements_location_idx": { + "name": "placements_location_idx", + "columns": ["location_id"], + "isUnique": false + } + }, + "foreignKeys": { + "placements_collection_id_collections_id_fk": { + "name": "placements_collection_id_collections_id_fk", + "tableFrom": "placements", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_organization_id_organization_id_fk": { + "name": "placements_organization_id_organization_id_fk", + "tableFrom": "placements", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_location_id_storage_locations_id_fk": { + "name": "placements_location_id_storage_locations_id_fk", + "tableFrom": "placements", + "tableTo": "storage_locations", + "columnsFrom": ["location_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_runs": { + "name": "push_runs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "first_key": { + "name": "first_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_key": { + "name": "last_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "merging_into": { + "name": "merging_into", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "push_runs_tier_idx": { + "name": "push_runs_tier_idx", + "columns": ["session_id", "tier", "merging_into"], + "isUnique": false + } + }, + "foreignKeys": { + "push_runs_session_id_push_sessions_id_fk": { + "name": "push_runs_session_id_push_sessions_id_fk", + "tableFrom": "push_runs", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_runs_session_id_seq_pk": { + "columns": ["session_id", "seq"], + "name": "push_runs_session_id_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_sessions": { + "name": "push_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_version_id": { + "name": "base_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "strip_unknown_fields": { + "name": "strip_unknown_fields", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "records_received": { + "name": "records_received", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "runs": { + "name": "runs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "result": { + "name": "result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finalize_started_at": { + "name": "finalize_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "cleaned_at": { + "name": "cleaned_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_phase": { + "name": "commit_phase", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "progress_at": { + "name": "progress_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stalls": { + "name": "stalls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "commit_plan": { + "name": "commit_plan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "assembly_lease": { + "name": "assembly_lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "push_sessions_collection_idx": { + "name": "push_sessions_collection_idx", + "columns": ["collection_id"], + "isUnique": false + }, + "push_sessions_expires_idx": { + "name": "push_sessions_expires_idx", + "columns": ["status", "expires_at"], + "isUnique": false + }, + "push_sessions_user_idx": { + "name": "push_sessions_user_idx", + "columns": ["user_id", "status"], + "isUnique": false + } + }, + "foreignKeys": { + "push_sessions_collection_id_collections_id_fk": { + "name": "push_sessions_collection_id_collections_id_fk", + "tableFrom": "push_sessions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compaction_parts": { + "name": "ref_compaction_parts", + "columns": { + "compaction_id": { + "name": "compaction_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "part": { + "name": "part", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_compaction_parts_compaction_id_part_pk": { + "columns": ["compaction_id", "part"], + "name": "ref_compaction_parts_compaction_id_part_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compactions": { + "name": "ref_compactions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_runs": { + "name": "input_runs", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output_run": { + "name": "output_run", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parts": { + "name": "parts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix_length": { + "name": "prefix_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'running'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_index_units": { + "name": "ref_index_units", + "columns": { + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unit": { + "name": "unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "events": { + "name": "events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_index_units_version_id_unit_pk": { + "columns": ["version_id", "unit"], + "name": "ref_index_units_version_id_unit_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_segments": { + "name": "ref_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_hash": { + "name": "first_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_hash": { + "name": "last_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'live'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ref_segments_range_idx": { + "name": "ref_segments_range_idx", + "columns": ["state", "first_hash", "last_hash"], + "isUnique": false + }, + "ref_segments_run_idx": { + "name": "ref_segments_run_idx", + "columns": ["run_id"], + "isUnique": false + }, + "ref_segments_tier_idx": { + "name": "ref_segments_tier_idx", + "columns": ["tier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_labels": { + "name": "schema_labels", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "schema_labels_label_idx": { + "name": "schema_labels_label_idx", + "columns": ["label"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "schema_labels_schema_hash_label_pk": { + "columns": ["schema_hash", "label"], + "name": "schema_labels_schema_hash_label_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_usage": { + "name": "schema_usage", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type_slug": { + "name": "type_slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_seq": { + "name": "from_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "to_seq": { + "name": "to_seq", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "schema_usage_hash_idx": { + "name": "schema_usage_hash_idx", + "columns": ["schema_hash"], + "isUnique": false + } + }, + "foreignKeys": { + "schema_usage_collection_id_collections_id_fk": { + "name": "schema_usage_collection_id_collections_id_fk", + "tableFrom": "schema_usage", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "schema_usage_collection_id_type_slug_set_from_seq_pk": { + "columns": ["collection_id", "type_slug", "set", "from_seq"], + "name": "schema_usage_collection_id_type_slug_set_from_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schemas": { + "name": "schemas", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": ["token"], + "isUnique": true + }, + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_fence": { + "name": "storage_fence", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "epoch": { + "name": "epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "window_until": { + "name": "window_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "window_run_id": { + "name": "window_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_locations": { + "name": "storage_locations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucket": { + "name": "bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "credentials": { + "name": "credentials", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unverified'" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checked_at": { + "name": "checked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "storage_locations_organization_id_organization_id_fk": { + "name": "storage_locations_organization_id_organization_id_fk", + "tableFrom": "storage_locations", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "usage_rollups": { + "name": "usage_rollups", + "columns": { + "day": { + "name": "day", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metric": { + "name": "metric", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "amount": { + "name": "amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "usage_rollups_account_idx": { + "name": "usage_rollups_account_idx", + "columns": ["account_id", "day"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "usage_rollups_day_account_id_collection_id_metric_pk": { + "columns": ["day", "account_id", "collection_id", "metric"], + "name": "usage_rollups_day_account_id_collection_id_metric_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": ["email"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": ["identifier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "versions": { + "name": "versions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "major": { + "name": "major", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "minor": { + "name": "minor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "patch": { + "name": "patch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pushed_by": { + "name": "pushed_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_record_count": { + "name": "public_record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_file_count": { + "name": "public_file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "public_total_bytes": { + "name": "public_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "type_counts": { + "name": "type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_type_counts": { + "name": "public_type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "has_private": { + "name": "has_private", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "public_refs_root": { + "name": "public_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_refs_root": { + "name": "private_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refs_indexed": { + "name": "refs_indexed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "changes": { + "name": "changes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "push_session_id": { + "name": "push_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "versions_collection_seq_uq": { + "name": "versions_collection_seq_uq", + "columns": ["collection_id", "seq"], + "isUnique": true + }, + "versions_collection_semver_uq": { + "name": "versions_collection_semver_uq", + "columns": ["collection_id", "semver"], + "isUnique": true + }, + "versions_hash_idx": { + "name": "versions_hash_idx", + "columns": ["hash"], + "isUnique": false + }, + "versions_published_idx": { + "name": "versions_published_idx", + "columns": ["published_at"], + "isUnique": false + }, + "versions_push_session_idx": { + "name": "versions_push_session_idx", + "columns": ["push_session_id"], + "isUnique": false + } + }, + "foreignKeys": { + "versions_collection_id_collections_id_fk": { + "name": "versions_collection_id_collections_id_fk", + "tableFrom": "versions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhook_deliveries": { + "name": "webhook_deliveries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "webhook_id": { + "name": "webhook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bump_type": { + "name": "bump_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'version.created'" + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "delivered_at": { + "name": "delivered_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhook_deliveries_webhook_idx": { + "name": "webhook_deliveries_webhook_idx", + "columns": ["webhook_id", "created_at"], + "isUnique": false + }, + "webhook_deliveries_pending_idx": { + "name": "webhook_deliveries_pending_idx", + "columns": ["status", "next_attempt_at"], + "isUnique": false + } + }, + "foreignKeys": { + "webhook_deliveries_webhook_id_collection_webhooks_id_fk": { + "name": "webhook_deliveries_webhook_id_collection_webhooks_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collection_webhooks", + "columnsFrom": ["webhook_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index 2aa3310a..f8fd9100 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -176,6 +176,13 @@ "when": 1792066046645, "tag": "0024_stream_push", "breakpoints": true + }, + { + "idx": 25, + "version": "6", + "when": 1792506002064, + "tag": "0025_record_layout", + "breakpoints": true } ] } diff --git a/packages/server/src/api/manage.ts b/packages/server/src/api/manage.ts index 80cf0a0c..946a58dc 100644 --- a/packages/server/src/api/manage.ts +++ b/packages/server/src/api/manage.ts @@ -166,6 +166,7 @@ export function manageRoutes() { versions: totals?.n ?? 0, totalBytes: Number(totals?.bytes ?? 0), deletedBy: c.var.principal?.userId ?? null, + recordLayout: col.recordLayout, }) .onConflictDoUpdate({ target: schema.collectionTombstones.collectionId, @@ -174,6 +175,8 @@ export function manageRoutes() { refEvents: col.refEvents, refBytes: col.refBytes, deletedAt: new Date(), + recordLayout: col.recordLayout, + recordsDroppedAt: null, }, }), db.delete(schema.collections).where(eq(schema.collections.id, col.id)), diff --git a/packages/server/src/api/versions.ts b/packages/server/src/api/versions.ts index e7e4dab3..5c563340 100644 --- a/packages/server/src/api/versions.ts +++ b/packages/server/src/api/versions.ts @@ -383,7 +383,8 @@ export function versionRoutes() { c.var.meter.logicalBytes = (c.var.meter.logicalBytes ?? 0) + entries.reduce((n, e) => (view.withheld.has(e.hash) ? n : n + e.size + 1), 0) - if (plain) { + // F keeps no leaf bodies: its records come through the leaf stream. + if (plain && !view.repo.records) { yield await view.repo.rawBody(leaf.hash) continue } diff --git a/packages/server/src/cleanup/layout-reclaim.ts b/packages/server/src/cleanup/layout-reclaim.ts new file mode 100644 index 00000000..630bc8df --- /dev/null +++ b/packages/server/src/cleanup/layout-reclaim.ts @@ -0,0 +1,98 @@ +/** + * Layout F's reclaim as bounded jobs (record-storage-layout.md, "Reclaim needs a + * mark again"): drops records no retained version of a collection holds. + * + * plan snapshot the index and packs, cut the key space into ranges + * range one job a range: mark the range's live records over every retained + * version, keep the index entries they need; the last range done + * swaps the new index in, drops dead packs, queues the rewrites + * rewrite a few mostly-dead packs a job: copy their live records out + * + * Nothing starts it yet: records only go dead when versions are thinned (a + * deleted collection's files go whole, cleanup/sweep.ts), and nothing may commit to + * the collection while a reclaim runs, so it needs the write fence first + * (cleanup/fence.ts). The scale bench ran it on F through `startLayoutReclaim`. + */ +import { KeyedLayout, markRange, type RecordSet, type Repo, recordRoots } from '@underlay/protocol' +import { eq } from 'drizzle-orm' + +import * as schema from '../db/schema.js' +import { registerJob } from '../jobs.js' +import type { JobMessage, Ports } from '../ports.js' + +/** Mostly-dead packs a rewrite job copies. */ +const REWRITE_PACKS = 4 + +interface Target { + collectionId: string + set: RecordSet +} + +const target = (job: JobMessage): Target => ({ + collectionId: String(job.collectionId), + set: job.set === 'private' ? 'private' : 'public', +}) + +async function open(ports: Ports, t: Target): Promise<{ repo: Repo; layout: KeyedLayout }> { + const repo = await ports.stores.forCollection(t.collectionId) + const layout = repo.records?.[t.set] + if (!(layout instanceof KeyedLayout)) + throw new Error('layout.reclaim: the collection is not on layout F') + return { repo, layout } +} + +/** Every retained version of the collection. */ +async function retained(ports: Ports, t: Target): Promise { + const rows = await ports.db + .select({ hash: schema.versions.hash }) + .from(schema.versions) + .where(eq(schema.versions.collectionId, t.collectionId)) + return rows.map((r) => r.hash) +} + +export async function startLayoutReclaim(ports: Ports, t: Target): Promise { + await ports.jobs.enqueue({ type: 'layout.reclaim', step: 'plan', ...t }) +} + +registerJob('layout.reclaim', async (job, ports) => { + const t = target(job) + const { repo, layout } = await open(ports, t) + switch (job.step) { + case 'plan': { + const plan = await layout.reclaimPlan() + await ports.jobs.enqueueBatch( + plan.ranges.map((_, i) => ({ + type: 'layout.reclaim', + step: 'range', + ...t, + plan: plan.id, + range: i, + })), + ) + return + } + case 'range': { + const plan = await layout.reclaimPlanById(String(job.plan)) + const r = plan.ranges[Number(job.range)]! + const { live } = await markRange(repo, await recordRoots(repo, await retained(ports, t)), r) + if (!(await layout.reclaimRange(plan, Number(job.range), live))) return + const swapped = await layout.reclaimSwap(plan) + if (!swapped) return + const jobs: JobMessage[] = [] + for (let i = 0; i < swapped.rewrite.length; i += REWRITE_PACKS) + jobs.push({ + type: 'layout.reclaim', + step: 'rewrite', + ...t, + packs: swapped.rewrite.slice(i, i + REWRITE_PACKS).join(','), + }) + await ports.jobs.enqueueBatch(jobs) + return + } + case 'rewrite': + await layout.reclaimRewrite(String(job.packs).split(',')) + return + default: + throw new Error(`layout.reclaim: unknown step ${String(job.step)}`) + } +}) diff --git a/packages/server/src/cleanup/marks.ts b/packages/server/src/cleanup/marks.ts index 4c285771..203fc4fc 100644 --- a/packages/server/src/cleanup/marks.ts +++ b/packages/server/src/cleanup/marks.ts @@ -167,6 +167,12 @@ export class Marker { if (this.spent) return false const node = await this.#read(() => this.repo.decoded(spec, root)) await this.onRead?.() + // A leaf stored as a delta needs the full node it was diffed against. (The read + // just now noted the base in memory, so this costs nothing for full nodes.) + if (node.kind === 'leaf') { + const base = await this.repo.deltaBase(root) + if (base) this.marks.add('n', base) + } if (node.kind === 'leaf') leaf?.(node.entries) else { // A slice of children at a time keeps the walk close to depth first. diff --git a/packages/server/src/cleanup/sweep.ts b/packages/server/src/cleanup/sweep.ts index 67b38395..bc1cec47 100644 --- a/packages/server/src/cleanup/sweep.ts +++ b/packages/server/src/cleanup/sweep.ts @@ -10,6 +10,9 @@ * roots/.json, private/.json: unmarked * files/: unmarked, with its `files` row * collections//…: no collection row, no tombstone in grace + * runs//…: layout F's files of a collection whose tombstone's + * grace is over, once the areas are done, but what a live fork + * still reads (KeyedLayout.dropCollection) * internal sessions//…: no push session row, past the orphan age * (reference-log scratch, sessions/refs-…, is step 1's) * uploads/: no upload row, or one that's done, past the orphan age @@ -24,8 +27,8 @@ * inside it), delete `files` rows, then objects, checking the window's deadline * as it goes, and close it. */ -import type { Store } from '@underlay/protocol' -import { and, asc, eq, gt, gte, inArray, or } from 'drizzle-orm' +import { KeyedLayout, setHome, type Store } from '@underlay/protocol' +import { and, asc, eq, gt, gte, inArray, isNull, lte, or } from 'drizzle-orm' import { chunks } from '../db/chunks.js' import * as schema from '../db/schema.js' @@ -422,5 +425,57 @@ export async function sweepStep( if (deleted >= cleanupConfig.sweepDeletes) break } if ((await flush()) === 'wait') return { state: start, stats, outcome: 'wait' } + if (s.area >= AREAS.length && (await dropDeletedRuns(ports, repo.blobs, stats, dryRun))) + return { state: s, stats, outcome: 'more' } return { state: s, stats, outcome: s.area >= AREAS.length ? 'done' : 'more' } } + +/** Deleted collections whose layout F files one sweep job drops. */ +const DROPS_PER_JOB = 20 + +/** + * Layout F's files of collections deleted past their grace, a few collections a + * job. A fork reads its parent's files where they are, so what a live fork lists + * stays, and goes with the last fork (KeyedLayout.dropCollection). Nothing writes + * to a deleted collection, so no fence is needed. True when more are due. + */ +async function dropDeletedRuns( + ports: Ports, + repo: Store, + stats: schema.CleanupStats, + dryRun: boolean, +): Promise { + const T = schema.collectionTombstones + const due = await ports.db + .select({ id: T.collectionId }) + .from(T) + .where( + and(eq(T.recordLayout, 'f'), isNull(T.recordsDroppedAt), lte(T.deletedAt, graceCutoff())), + ) + .orderBy(asc(T.collectionId)) + .limit(DROPS_PER_JOB) + if (dryRun) return false + // A parent is gone once it has no row and its tombstone's grace is over. Homes are + // a collection's sets (`/`). + const deleted = async (home: string) => { + const id = home.split('/')[0]! + const [row] = await ports.db + .select({ id: schema.collections.id }) + .from(schema.collections) + .where(eq(schema.collections.id, id)) + if (row) return false + const [t] = await ports.db + .select({ id: T.collectionId }) + .from(T) + .where(and(eq(T.collectionId, id), lte(T.deletedAt, graceCutoff()))) + return !!t + } + for (const { id } of due) { + for (const set of ['public', 'private'] as const) { + const r = await KeyedLayout.dropCollection(repo, setHome(id, set), deleted) + count(stats, 'runs', 0, r.objects) + } + await ports.db.update(T).set({ recordsDroppedAt: new Date() }).where(eq(T.collectionId, id)) + } + return due.length === DROPS_PER_JOB +} diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index 7d169741..095f757a 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -271,6 +271,15 @@ export const collections = sqliteTable( */ reconciledSeq: integer('reconciled_seq').notNull().default(0), reconciledFilesRoot: text('reconciled_files_root'), + /** + * Where its record bytes live (record-storage-layout.md): `today`, inside leaf + * bodies, or `f`, in packs found through the collection's own index under + * `runs//`. Set at creation (forks take their parent's); existing + * collections are `today` until converted. + */ + recordLayout: text('record_layout', { enum: ['today', 'f'] }) + .notNull() + .default('today'), deletedAt: ts('deleted_at'), createdAt: createdAt(), updatedAt: ts('updated_at') @@ -733,6 +742,12 @@ export const collectionTombstones = sqliteTable('collection_tombstones', { deletedAt: ts('deleted_at') .notNull() .$defaultFn(() => new Date()), + /** The collection's record layout: layout F's files (`runs//`) go once its grace is over. */ + recordLayout: text('record_layout', { enum: ['today', 'f'] }) + .notNull() + .default('today'), + /** When cleanup dropped its layout F files (null: not yet, or none). */ + recordsDroppedAt: ts('records_dropped_at'), }) // --- Reference log (provenance; edge-redesign.md "Provenance: the reference log") -------- diff --git a/packages/server/src/handlers.ts b/packages/server/src/handlers.ts index a87ecb5f..3f95cd58 100644 --- a/packages/server/src/handlers.ts +++ b/packages/server/src/handlers.ts @@ -20,6 +20,7 @@ import { type FsckCursor, type FsckReport, fsckStep, + KeyedLayout, readCollectionInfo, readHead, } from '@underlay/protocol' @@ -28,6 +29,7 @@ import { and, asc, eq, gt } from 'drizzle-orm' import './files/files.js' import { reconcileDue } from './billing/reconcile.js' import { cleanupTick } from './cleanup/runs.js' +import './cleanup/layout-reclaim.js' import * as schema from './db/schema.js' import './push/compact.js' import './push/parallel.js' @@ -35,7 +37,7 @@ import './refs/log.js' import { registerJob } from './jobs.js' import { recheckLocations } from './locations/locations.js' import { laggingPlacements, queueMirrors } from './locations/mirror.js' -import type { Ports } from './ports.js' +import type { JobMessage, Ports } from './ports.js' import { expireSessions } from './push/finalize.js' import { recoverCommits } from './push/recover.js' import { queueInfoBackfill } from './versions/collection-info.js' @@ -50,6 +52,56 @@ registerJob('version.published', async (job, ports) => { .from(schema.versions) .where(eq(schema.versions.id, String(job.versionId))) if (v) await queueMirrors(ports, v.collectionId) + // Layout F merges a collection's packs and index runs after a commit, a set a job + // (a set with nothing due leaves after a few reads). + if (v && (await ports.stores.forCollection(v.collectionId)).records) + await ports.jobs.enqueueBatch( + RECORD_SETS.map((set) => ({ type: 'layout.maintain', collectionId: v.collectionId, set })), + ) +}) + +const RECORD_SETS = ['public', 'private'] as const + +/** A job's record set, and the collection's layout F store for it (null: not on F). */ +async function setLayout(ports: Ports, job: JobMessage): Promise { + const repo = await ports.stores.forCollection(String(job.collectionId)) + const records = repo.records?.[job.set === 'private' ? 'private' : 'public'] + return records instanceof KeyedLayout ? records : null +} + +registerJob('layout.maintain', async (job, ports) => { + const layout = await setLayout(ports, job) + if (!layout) return + // A merge too large for one job runs as key-range jobs, swapped in by the last. + const more = await layout.maintain({ + split: (plan) => + ports.jobs.enqueueBatch( + plan.ranges.map((_, i) => ({ + type: 'layout.merge', + collectionId: String(job.collectionId), + set: String(job.set ?? 'public'), + plan: plan.id, + range: i, + })), + ), + }) + // Stopped by its time or storage-call budget: the next job carries on (a load's + // last commit queues no other). + if (more) + await ports.jobs.enqueue({ + type: 'layout.maintain', + collectionId: String(job.collectionId), + set: String(job.set ?? 'public'), + }) +}) + +registerJob('layout.merge', async (job, ports) => { + const layout = await setLayout(ports, job) + if (!layout) return + const plan = await layout.mergePlanById(String(job.plan)) + // Gone: swapped in, or abandoned. + if (!plan) return + if (await layout.mergeRange(plan, Number(job.range))) await layout.mergeSwap(plan) }) registerJob('maintenance.sweep', async (_job, ports) => { diff --git a/packages/server/src/jobs.ts b/packages/server/src/jobs.ts index 8f6cf6d7..53947425 100644 --- a/packages/server/src/jobs.ts +++ b/packages/server/src/jobs.ts @@ -50,6 +50,11 @@ export const BULK_JOBS: ReadonlySet = new Set([ 'cleanup.internal', 'cleanup.mark', 'cleanup.sweep', + // Layout F's merges after a commit. + 'layout.maintain', + // Its reclaim, a key range a job (cleanup/layout-reclaim.ts), and merges too large for one job. + 'layout.reclaim', + 'layout.merge', ]) export const isBulk = (type: string) => BULK_JOBS.has(type) diff --git a/packages/server/src/locations/mirror.ts b/packages/server/src/locations/mirror.ts index 5454ce2b..2709f35e 100644 --- a/packages/server/src/locations/mirror.ts +++ b/packages/server/src/locations/mirror.ts @@ -10,7 +10,11 @@ * reaches that the previous version doesn't, then the file bytes it added, the * private set object (public+private placements only), the root, then * collection.json, the signed log entry and head.json, in the repository's - * write order. Backfill is the same loop from version 1. A copy stops after a + * write order. Layout F keeps records out of leaf bodies, in files of the + * collection's own, a set's apart: after the nodes, the job copies those of the + * sets the placement holds that it hasn't yet, as they are, and the manifest + * that lists them (KeyedLayout.mirrorFiles). + * Backfill is the same loop from version 1. A copy stops after a * budget and resumes from a cursor (the tree and last key done), so no single * job copies more than a few thousand objects. Copies are tracked in the * placement row, never found by HEAD requests: a write-only mirror can't read. @@ -23,6 +27,8 @@ import { type FileEntry, fileTree, jcs, + KeyedLayout, + type RecordSet, keys, type LogEntry, putFromParts, @@ -50,6 +56,61 @@ export const mirrorConfig = { fileBytesPerJob: 256 * 1024 * 1024, /** Files are read and written in parts of this size (S3 joins parts under 5 MiB). */ partBytes: 8 * 1024 * 1024, + /** A packed layout's record files one job copies before it queues the rest. */ + recordFilesPerJob: 500, +} + +/** A set's record files (F) a mirror placement holds: copied once, and dropped once no manifest lists them. */ +const copiedKey = (placementId: string, set: RecordSet) => + `mirrors/${placementId}/record-files${set === 'public' ? '' : `-${set}`}.json` + +/** + * Copy a packed layout's record files a placement lacks, then the manifest naming + * them. False when the job's budget ran out first (the next job carries on: what + * was copied is remembered). + */ +async function copyRecordFiles( + ports: Ports, + repo: { blobs: Store; records: KeyedLayout }, + dest: Store, + placementId: string, + set: RecordSet, +): Promise { + const { files, manifest, manifestKeys } = await repo.records.mirrorFiles() + const saved = await ports.stores.internal.get(copiedKey(placementId, set)) + const copied = new Set(saved ? (JSON.parse(await saved.text()) as string[]) : []) + const remember = () => + ports.stores.internal.put(copiedKey(placementId, set), JSON.stringify([...copied])) + let n = 0 + for (const f of files) { + if (copied.has(f.key)) continue + const head = await repo.blobs.head(f.key) + if (!head) { + if (f.optional) continue + // Swept since the listing (a merge replaced it): the next job lists again. Still + // listed and still missing is a broken collection, not something to wait out. + const again = await repo.records.mirrorFiles() + if (again.files.some((g) => g.key === f.key) && !(await repo.blobs.head(f.key))) + throw new Error(`Record file ${f.key} is listed but missing`) + return remember().then(() => false) + } + await putFromParts(dest, f.key, readParts(repo.blobs, f.key, head.size, mirrorConfig.partBytes)) + copied.add(f.key) + if (++n >= mirrorConfig.recordFilesPerJob) return remember().then(() => false) + } + await dest.put(manifestKeys.numbered, JSON.stringify(manifest), { + contentType: 'application/json', + }) + await dest.put(manifestKeys.head, JSON.stringify(manifest), { contentType: 'application/json' }) + // What no manifest lists any more (merged away) goes from the mirror too. + const listed = new Set(files.map((f) => f.key)) + for (const k of [...copied]) + if (!listed.has(k)) { + await dest.delete(k).catch(() => undefined) + copied.delete(k) + } + await remember() + return true } type Placement = typeof schema.placements.$inferSelect @@ -190,6 +251,10 @@ export async function mirrorStep(ports: Ports, placementId: string): Promise { const obj = await repo.blobs.get(key) @@ -228,6 +293,15 @@ export async function mirrorStep(ports: Ports, placementId: string): Promise + /** The record layout a new collection gets (a fork takes its parent's). */ + newCollections: LayoutName /** The repository at a storage location. */ forLocation(locationId: string): Promise /** diff --git a/packages/server/src/push/compact.ts b/packages/server/src/push/compact.ts index 5e1aa4d3..7dee47e3 100644 --- a/packages/server/src/push/compact.ts +++ b/packages/server/src/push/compact.ts @@ -336,7 +336,9 @@ async function joinPieces(ports: Ports, sessionId: string, seq: number, plan: Co parts.push(...(JSON.parse(await obj.text()) as RunPart[])) } const index: RunIndex = { seq, tier: plan.tier, parts } - await writeRunIndex(store, sessionId, index) + // The last two pieces can finish together and both join: the index is the same, + // written once (a second write of a key within a second is one R2 refuses). + await writeRunIndex(store, sessionId, index, { ifAbsent: true }) await finishCompaction(ports, sessionId, seq, index) } diff --git a/packages/server/src/push/parallel.ts b/packages/server/src/push/parallel.ts index fcde4e14..6a9be61a 100644 --- a/packages/server/src/push/parallel.ts +++ b/packages/server/src/push/parallel.ts @@ -47,6 +47,7 @@ import { mergeTree, type NodeDesc, type BodyRead, + type HeldRun, type RecordEntry, recordPayloadBytes, recordTree, @@ -187,6 +188,11 @@ interface UnitOutput { stats: { added: number; removed: number; updated: number } /** File reference count deltas: [fileHash, delta]. */ refs: [string, number][] + /** + * Layout F: the index runs and packs it wrote, held back from the manifest; the + * assembly names a tree's together (one pending object a tree, not one a unit). + */ + records?: HeldRun[] } type UnitRow = typeof schema.commitUnits.$inferSelect @@ -444,7 +450,9 @@ interface TypeMode { * type (so moving them is O(files)): public → private with nothing in the * private set yet, and private → public with no private upsert of the type in * this push. Anything else that changes privacy, and a type removed from a base - * without per-type counts, stays serial at the cost of the type. + * without per-type counts, stays serial at the cost of the type. In layout F a + * moved type's records move to the other set's packs (build.ts copyRecords), so + * every privacy change is serial there, at the cost of the type. */ async function typeModes( repo: Repo, @@ -472,11 +480,11 @@ async function typeModes( const pubRoot = pubBase?.root ?? null const privRoot = privBase?.root ?? null if (isPrivateSchema(s) && pubRoot) { - if (!privRoot && tracked.public) mode.move = 'toPrivate' + if (!privRoot && tracked.public && !repo.records) mode.move = 'toPrivate' else mode.serialOnly = bytes } else if (!isPrivateSchema(s) && !pubBase && privBase) { const privateUpserts = runs.some((ix) => partsInRange(ix, { type: slug }).some((p) => p.p)) - if (tracked.private && !privateUpserts) mode.move = 'toPublic' + if (tracked.private && !privateUpserts && !repo.records) mode.move = 'toPublic' else mode.serialOnly = bytes } modes[slug] = mode @@ -897,7 +905,12 @@ async function runPendingUnit(ports: Ports, unit: UnitRow, session: SessionRow): const set = unit.set as SetName const repo = await ports.stores.forCollection(session.collectionId) // Writes in flight hold compressed leaves; halved from the default so two units fit an isolate. - const sink = new RepoSink(repo, { bodyOf: bodyOfRecord, ...unitSink }) + const sink = new RepoSink(repo, { + bodyOf: bodyOfRecord, + ...unitSink, + set, + holdRecords: true, + }) const range = { after: unit.after, through: unit.through } // Progress in the Worker's log, so a unit that dies (out of memory, say) shows // how far it got, the largest things it met, and what else its isolate ran. @@ -920,6 +933,16 @@ async function runPendingUnit(ports: Ports, unit: UnitRow, session: SessionRow): ) const leaves: NodeDesc[] = [] const refs = new FileRefDelta() + // A base with no file references in this set: old records can't hold any, so a + // packed layout needn't read them (the scale bench, round 3). + const [baseRefs] = session.baseVersionId + ? await ports.db + .select({ pub: schema.versions.publicRefsRoot, priv: schema.versions.privateRefsRoot }) + .from(schema.versions) + .where(eq(schema.versions.id, session.baseVersionId)) + : [] + if (!baseRefs || (set === 'public' ? baseRefs.pub : baseRefs.priv) === null) + refs.noOldRefs.add(set) // A privacy change merges into the moved tree (`bases`). const into = (t.bases ?? t)[set].root // Base bodies by how they were read, and their stored bytes where known. @@ -933,7 +956,7 @@ async function runPendingUnit(ports: Ports, unit: UnitRow, session: SessionRow): // The most members compressing at a leaf's end: what a leaf's write holds raw. let mostCompressing = 0 const merged = await mergeTree( - new RepoSource(recordTree, repo, { onBodyRead, pointers: true }), + new RepoSource(recordTree, repo, { onBodyRead, pointers: true, set }), sink, into, changes, @@ -951,6 +974,7 @@ async function runPendingUnit(ports: Ports, unit: UnitRow, session: SessionRow): ) }, onChange: (before, after) => refs.record(set, before, after, unit.type), + oldPayloads: !refs.noOldRefs.has(set), }, ) log( @@ -966,6 +990,7 @@ async function runPendingUnit(ports: Ports, unit: UnitRow, session: SessionRow): leaves, stats: { added, removed, updated }, refs: [...refs.refs[set]].filter(([, n]) => n !== 0), + ...(sink.held.length ? { records: sink.held } : {}), } const outputKey = `${planDir(session.id, unit.planId)}/${unit.id}.out.json` await putJson(ports, outputKey, output) @@ -1112,9 +1137,16 @@ export async function assembleParallel(ports: Ports, sessionId: string): Promise const plan = await getJson(ports, `${planDir(sessionId, planId)}/plan.json`) const repo = await ports.stores.forCollection(session.collectionId) - // Base leaves keep large records as pointers, as in units (runPendingUnit). - const source = new RepoSource(recordTree, repo, { pointers: true }) - const sink = new RepoSink(repo, { bodyOf: bodyOfRecord }) + // Base leaves keep large records as pointers, as in units (runPendingUnit). A source + // and a sink a set: layout F reads and writes each set's records apart. + const sources = { + public: new RepoSource(recordTree, repo, { pointers: true, set: 'public' }), + private: new RepoSource(recordTree, repo, { pointers: true, set: 'private' }), + } + const sinks = { + public: new RepoSink(repo, { bodyOf: bodyOfRecord, set: 'public' }), + private: new RepoSink(repo, { bodyOf: bodyOfRecord, set: 'private' }), + } const built: Record = {} for (const [slug, t] of Object.entries(plan.types)) built[slug] = { ...(t.bases ?? t) } const refs = new FileRefDelta() @@ -1150,6 +1182,7 @@ export async function assembleParallel(ports: Ports, sessionId: string): Promise const countStats = !(plan.types[type]!.move === 'toPublic' && set === 'public') const treeStats = { added: 0, removed: 0, updated: 0 } const treeRefs = new Map() + const treeRuns: HeldRun[] = [] // Each segment's output is read once, when assembly reaches it. const segments: Segment[] = units.map((u) => ({ after: u.after, @@ -1163,11 +1196,12 @@ export async function assembleParallel(ports: Ports, sessionId: string): Promise } // A unit covers one type, so its deltas are that type's too. for (const [h, n] of out.refs) treeRefs.set(h, (treeRefs.get(h) ?? 0) + n) + if (out.records) treeRuns.push(...out.records) return out.leaves }, })) const treeStarted = Date.now() - const result = await assembleTree(source, sink, built[type]![set].root, segments) + const result = await assembleTree(sources[set], sinks[set], built[type]![set].root, segments) console.log( `[assemble] ${isolateTag()} ${sessionId} ${type}/${set}: ${segments.length} segments ` + `in ${Math.round((Date.now() - treeStarted) / 1000)}s, ${JSON.stringify(result.stats)}`, @@ -1184,14 +1218,17 @@ export async function assembleParallel(ports: Ports, sessionId: string): Promise stats: treeStats, refs: [...treeRefs], } - // The tree's nodes are written before it counts as done. - await sink.flush() + // The tree's nodes are written before it counts as done, and in layout F its units' + // records named in the set's index (a retry writing them again folds in once). + await sinks[set].flush() + await repo.records?.[set].appendRuns?.(treeRuns) await putJson(ports, treeKey, tree) take(tree) builtHere++ await touch(ports, sessionId) } - await sink.flush() + await sinks.public.flush() + await sinks.private.flush() const assembled: Assembled = { built, refs: refsOut(refs), stats } if (Object.values(plan.types).some((t) => t.validate)) { @@ -1415,7 +1452,7 @@ export async function runValidation( const inputs = await loadInputs(ports, sessionId) const validate = compileSchema(inputs.schemas[t.type]!) const repo = await ports.stores.forCollection(session.collectionId) - const source = new RepoSource(recordTree, repo) + const source = new RepoSource(recordTree, repo, { set: t.set }) const result: ValidationResult = { total: 0, errors: [] } for await (const e of entriesInRange(source, t.root, t.after, t.through)) { const errs = validate((JSON.parse(e.body!) as { data: unknown }).data) diff --git a/packages/server/src/push/runs.ts b/packages/server/src/push/runs.ts index 6ac5a1a1..657a6e22 100644 --- a/packages/server/src/push/runs.ts +++ b/packages/server/src/push/runs.ts @@ -398,9 +398,16 @@ export class RunWriter { } } -export async function writeRunIndex(store: Store, sessionId: string, index: RunIndex) { +export async function writeRunIndex( + store: Store, + sessionId: string, + index: RunIndex, + /** Keep an index already there (it's the same: a compaction's pieces joined twice). */ + opts: { ifAbsent?: boolean } = {}, +) { await store.put(indexKey(sessionId, index.seq), JSON.stringify({ v: 2, ...index }), { contentType: 'application/json', + ...(opts.ifAbsent ? { ifAbsent: true } : {}), }) } diff --git a/packages/server/src/stores.ts b/packages/server/src/stores.ts index 8851496e..c3911825 100644 --- a/packages/server/src/stores.ts +++ b/packages/server/src/stores.ts @@ -7,6 +7,7 @@ * credential encryption lands with bucket mirrors (phase 11). */ import { + type LayoutName, type Cache, PrefixedStore, type PresigningStore, @@ -34,11 +35,18 @@ export interface PlatformStorage { internalPrefix: string /** The isolate's memory cache for repository objects (default 32 MB; the bulk Worker's is smaller). */ memoryBytes?: number + /** + * The record layout new collections get (record-storage-layout.md): `today` + * (default for now), leaf bodies, or `f`, records in packs per collection. Each collection keeps + * the layout it was created with (`collections.record_layout`); forks take their + * parent's. + */ + newCollections?: LayoutName } -/** Primary placements change rarely; remember them briefly per isolate. */ +/** Primary placements change rarely, a collection's layout only by a conversion; remember them briefly per isolate. */ const PRIMARY_TTL_MS = 60_000 -const primaryCache = new Map() +const primaryCache = new Map() export function createStores(db: Db, cache: Cache, platform: PlatformStorage): Stores { const repos = new Map() @@ -67,14 +75,45 @@ export function createStores(db: Db, cache: Cache, platform: PlatformStorage): S return repo } + /** + * Layout F keeps records per collection: one repository per collection over the + * location's objects (nodes are shared), its packs, index and manifest under + * `runs//`, and rewritten leaves stored as node deltas. + */ + const forCollectionRuns = async (locationId: string, collectionId: string): Promise => { + const k = `${locationId}/${collectionId}` + const known = repos.get(k) + if (known) return known + const loc = await forLocation(locationId) + const repo = new Repo(loc.blobs, { + cache, + scope: loc.scope, + trusted: true, + lru: loc.lru, + layout: 'f', + recordScope: collectionId, + nodes: 'delta', + }) + repos.set(k, repo) + return repo + } + const located = (locationId: string, collectionId: string, layout: LayoutName) => + layout === 'f' ? forCollectionRuns(locationId, collectionId) : forLocation(locationId) + return { forLocation, + newCollections: platform.newCollections ?? 'today', async forCollection(collectionId: string): Promise { const hit = primaryCache.get(collectionId) - if (hit && Date.now() - hit.at < PRIMARY_TTL_MS) return forLocation(hit.locationId) + if (hit && Date.now() - hit.at < PRIMARY_TTL_MS) + return located(hit.locationId, collectionId, hit.layout) const [row] = await db - .select({ locationId: schema.placements.locationId }) + .select({ + locationId: schema.placements.locationId, + layout: schema.collections.recordLayout, + }) .from(schema.placements) + .innerJoin(schema.collections, eq(schema.collections.id, schema.placements.collectionId)) .where( and( eq(schema.placements.collectionId, collectionId), @@ -83,8 +122,12 @@ export function createStores(db: Db, cache: Cache, platform: PlatformStorage): S ) .limit(1) if (!row) throw new Error(`Collection ${collectionId} has no primary placement`) - primaryCache.set(collectionId, { locationId: row.locationId, at: Date.now() }) - return forLocation(row.locationId) + primaryCache.set(collectionId, { + locationId: row.locationId, + layout: row.layout, + at: Date.now(), + }) + return located(row.locationId, collectionId, row.layout) }, internal: new PrefixedStore(platform.bucket, platform.internalPrefix), fileBytes: files, diff --git a/packages/server/src/versions/fork.ts b/packages/server/src/versions/fork.ts index e7a9e933..e359f2af 100644 --- a/packages/server/src/versions/fork.ts +++ b/packages/server/src/versions/fork.ts @@ -9,7 +9,7 @@ * the public set only. Both collections must share a primary location, since a * repository never references another location. */ -import { makeRoot, newSalt } from '@underlay/protocol' +import { type LayoutName, makeRoot, newSalt } from '@underlay/protocol' import { eq } from 'drizzle-orm' import { FenceError, fenced } from '../cleanup/fence.js' @@ -28,6 +28,8 @@ export async function createCollectionRows( privateSalt?: string /** Until the first version's metadata replaces it. */ summary?: schema.CollectionSummary + /** Its record layout: the platform's for new collections (`Stores.newCollections`). */ + recordLayout?: LayoutName }, ): Promise { const id = crypto.randomUUID() @@ -40,6 +42,7 @@ export async function createCollectionRows( public: c.public, privateSalt: c.privateSalt ?? newSalt(), summary: c.summary ?? null, + recordLayout: c.recordLayout ?? ports.stores.newCollections, }), ports.db.insert(schema.placements).values({ collectionId: id, @@ -74,9 +77,17 @@ export async function forkCollection( const collection = await createCollectionRows(ports, { ...target, ...(priv ? { privateSalt: source.collection.privateSalt } : {}), + // Layout F forks by reference: the fork reads its parent's packs and index. + recordLayout: source.collection.recordLayout, }) const targetRepo = await ports.stores.forCollection(collection.id) - if (targetRepo !== repo) throw new Error('Forks across storage locations are not supported yet') + if (repo.records && targetRepo.records) { + // Layout F: the fork lists the parent's files and reads them where they are, a set + // at a time; a fork of the public set only never references the private set's. + for (const set of priv ? (['public', 'private'] as const) : (['public'] as const)) + await repo.records[set].forkInto!(targetRepo.records[set]) + } else if (targetRepo !== repo) + throw new Error('Forks across storage locations are not supported yet') const newRoot = makeRoot(root.metadata, root.public, priv) const v = source.version diff --git a/packages/server/src/versions/view.ts b/packages/server/src/versions/view.ts index 80909062..ade8d6e3 100644 --- a/packages/server/src/versions/view.ts +++ b/packages/server/src/versions/view.ts @@ -154,13 +154,14 @@ export async function* typeRecords( if (opts.offset >= type.count) return after = await keyBeforeOffset(source, pubRoot, privRoot, opts.offset) } - const it = (root: string | null) => - iterate(source, root, { + // Each set's records from its own source: layout F keeps them apart. + const it = (root: string | null, set: 'public' | 'private') => + iterate(new RepoSource(recordTree, view.repo, { set }), root, { payloads: opts.bodies ?? false, stream: true, ...(after !== undefined ? { after } : {}), }) - for await (const { e, set } of mergeById(it(pubRoot), it(privRoot))) { + for await (const { e, set } of mergeById(it(pubRoot, 'public'), it(privRoot, 'private'))) { if (view.withheld.has(e.hash)) continue yield { ...e, type: type.slug, set } } @@ -220,12 +221,12 @@ export async function getRecord( type: TypeView, id: string, ): Promise { - const source = new RepoSource(recordTree, view.repo) for (const [set, tree] of [ ['public', type.public], ['private', view.owner ? type.private : null], ] as const) { if (!tree?.root) continue + const source = new RepoSource(recordTree, view.repo, { set }) const hit = await getEntry(source, tree.root, id) if (!hit) continue if (view.withheld.has(hit.hash)) return null @@ -240,6 +241,14 @@ async function bodyOf(source: RepoSource, root: string, key: string for (;;) { const node = await source.node(hash) if (node.kind === 'leaf') { + if (source.repo.records) { + // A packed layout reads the one record. + const i = node.entries.findIndex((e) => e.key === key) + if (i < 0) throw new Error(`Record ${key} is not in leaf ${hash}`) + return ( + await source.repo.linesAt({ hash, entries: node.entries }, [i], source.opts.set) + )[0]! + } // Streamed, stopping at the record: a leaf of large records is never held whole. for await (const e of source.leafStream(hash)) if (e.key === key) return e.body! throw new Error(`Record ${key} is not in leaf ${hash}`) @@ -261,6 +270,8 @@ export async function recordBodies( ): Promise> { const source = new RepoSource(recordTree, view.repo) const byLeaf = new Map>() + // Each leaf's set: layout F reads a record from its set's index. + const leafSet = new Map() for (const w of wanted) { const t = view.types.find((x) => x.slug === w.type) let hash = (w.set === 'public' ? t?.public : t?.private)?.root ?? null @@ -273,6 +284,7 @@ export async function recordBodies( const keys = byLeaf.get(hash) ?? new Set() keys.add(`${w.type}\u0000${w.key}`) byLeaf.set(hash, keys) + leafSet.set(hash, w.set) } const out = new Map() const leaves = [...byLeaf] @@ -281,6 +293,17 @@ export async function recordBodies( while (next < leaves.length) { const [hash, keys] = leaves[next++]! const type = [...keys][0]!.split('\u0000')[0]! + if (source.repo.records) { + // A packed layout reads only the records wanted. + const node = await source.node(hash) + const entries = node.kind === 'leaf' ? node.entries : [] + const at = entries.flatMap((e, i) => + keys.has(`${type}\u0000${e.key}`) && !view.withheld.has(e.hash) ? [i] : [], + ) + const lines = await source.repo.linesAt({ hash, entries }, at, leafSet.get(hash)) + at.forEach((i, j) => out.set(`${type}\u0000${entries[i]!.key}`, lines[j]!)) + continue + } let left = keys.size for await (const e of source.leafStream(hash)) { const k = `${type}\u0000${e.key}` @@ -290,6 +313,8 @@ export async function recordBodies( } } } - await Promise.all(Array.from({ length: Math.min(concurrency, leaves.length) }, worker)) + // A packed layout reads a few records a leaf: more leaves at once. + const n = source.repo.records ? Math.max(concurrency, 16) : concurrency + await Promise.all(Array.from({ length: Math.min(n, leaves.length) }, worker)) return out } diff --git a/packages/server/test/cleanup.test.ts b/packages/server/test/cleanup.test.ts index 56d8c9cf..610fd705 100644 --- a/packages/server/test/cleanup.test.ts +++ b/packages/server/test/cleanup.test.ts @@ -1,6 +1,6 @@ import { createHash } from 'node:crypto' -import { fsck, newSalt } from '@underlay/protocol' +import { fsck, newSalt, openRepo } from '@underlay/protocol' import { eq, sql } from 'drizzle-orm' import { afterAll, beforeEach, describe, expect, it } from 'vitest' @@ -208,7 +208,12 @@ describe('storage cleanup, steps 2 and 3: mark and sweep', () => { // aren't kept: they're bookkeeping for new commits.) const platform = await h.ports.stores.forLocation(schema.PLATFORM_LOCATION_ID) for (const id of [a.id, cRow!.id]) { - const report = await fsck(platform, { + // Its row is gone: opened in the layout it was made in (layout F: its own files). + const repo = + h.ports.stores.newCollections === 'f' + ? openRepo(platform.blobs, { layout: 'f', recordScope: id, nodes: 'delta' }) + : platform + const report = await fsck(repo, { collectionId: id, trustedKeys: [h.signer.publicKey], fileBytes: true, diff --git a/packages/server/test/commit.test.ts b/packages/server/test/commit.test.ts index 7d82ff54..5f35390c 100644 --- a/packages/server/test/commit.test.ts +++ b/packages/server/test/commit.test.ts @@ -406,7 +406,8 @@ describe('commitVersion', () => { }) it('falls back to reading records on a count tree without per-type counts', async () => { - const h = await harness() + // It watches leaf bodies being read: today's layout's. + const h = await harness({ newCollections: 'today' }) const c = await h.collection() const repo = await h.ports.stores.forCollection(c.id) await h.ports.db diff --git a/packages/server/test/fsck.test.ts b/packages/server/test/fsck.test.ts index 7c8166f7..f24e28b4 100644 --- a/packages/server/test/fsck.test.ts +++ b/packages/server/test/fsck.test.ts @@ -88,10 +88,12 @@ describe('fsck', () => { ]) await repo.blobs.put(headKey, head) - // Corrupt a file, drop a body, and sign nothing it trusts. + // Corrupt a file, drop a body (in layout F, a pack of records), and sign nothing it trusts. await repo.blobs.put(keys.file(sha(pdf)), 'other bytes!') - const bodies = [...h.bucket.objects.keys()].filter((k) => k.includes('/bodies/')) - h.bucket.objects.delete(bodies[0]!) + const held = [...h.bucket.objects.keys()].filter((k) => + repo.records ? /\/packs\/[^/]+\.pk$/.test(k) : k.includes('/bodies/'), + ) + h.bucket.objects.delete(held[0]!) const bad = await fsck(repo, { collectionId: c.id, trustedKeys: [], fileBytes: true }) expect(bad.ok).toBe(false) const text = bad.errors.join('\n') diff --git a/packages/server/test/harness.ts b/packages/server/test/harness.ts index 9a97fdc8..e9dad084 100644 --- a/packages/server/test/harness.ts +++ b/packages/server/test/harness.ts @@ -10,6 +10,7 @@ import { memoryStore, newSalt, type Signer, + type LayoutName, } from '@underlay/protocol' import '../src/handlers.js' @@ -23,6 +24,9 @@ import { createStores } from '../src/stores.js' const dirs: string[] = [] +/** The layout this test project runs collections on (vitest.config.ts). */ +const testLayout = (process.env.UNDERLAY_TEST_LAYOUT ?? 'today') as LayoutName + /** Outbound requests made by the app (webhooks), and the responder tests set. */ export const outboundCalls: { url: string; init: RequestInit }[] = [] export let outboundResponder: (url: string) => Response = () => new Response('ok') @@ -56,10 +60,18 @@ export interface Harness { /** SQL statements sent so far (D1 runs at most 1,000 per invocation). */ statements(): number /** An org and a collection with a primary placement on the platform location. */ - collection(slug?: string): Promise + /** A collection in the org, in the project's layout unless `recordLayout` says. */ + collection( + slug?: string, + opts?: { recordLayout?: LayoutName }, + ): Promise } -export async function harness(): Promise { +/** + * The record layout new collections get: the test project's (vitest.config.ts runs + * the suite once a layout), unless a test needs one. + */ +export async function harness(opts: { newCollections?: LayoutName } = {}): Promise { const dir = await mkdtemp(join(tmpdir(), 'ul-it-')) dirs.push(dir) let statements = 0 @@ -72,7 +84,12 @@ export async function harness(): Promise { const signer = await ed25519Signer(await generateSigningKey()) const ports: Ports = { db, - stores: createStores(db, cache, { bucket, repoPrefix: 'repo', internalPrefix: 'internal' }), + stores: createStores(db, cache, { + bucket, + repoPrefix: 'repo', + internalPrefix: 'internal', + newCollections: opts.newCollections ?? testLayout, + }), cache, signer: async () => signer, jobs: new SqliteJobs(db), @@ -132,11 +149,17 @@ export async function harness(): Promise { }), ) }, - async collection(slug = 'c') { + async collection(slug = 'c', opts: { recordLayout?: LayoutName } = {}) { await ensureOrg() const [c] = await db .insert(schema.collections) - .values({ organizationId: 'org1', slug, name: slug, privateSalt: newSalt() }) + .values({ + organizationId: 'org1', + slug, + name: slug, + privateSalt: newSalt(), + recordLayout: opts.recordLayout ?? ports.stores.newCollections, + }) .returning() await db.insert(schema.placements).values({ collectionId: c!.id, diff --git a/packages/server/test/layouts.test.ts b/packages/server/test/layouts.test.ts new file mode 100644 index 00000000..abc9b347 --- /dev/null +++ b/packages/server/test/layouts.test.ts @@ -0,0 +1,166 @@ +/** + * Today's record layout (leaf bodies) and layout F (packs per collection and set) + * side by side in one location: the same pushes give the same versions, every + * record reads the same from both, and both pass fsck. Both share the location's + * node files, so each also reads nodes the other wrote: F's as deltas, today's in full. + */ +import { createHash } from 'node:crypto' + +import { fsck, iterate, type RecordSet, RepoSource, recordTree } from '@underlay/protocol' +import { eq } from 'drizzle-orm' +import { afterAll, afterEach, describe, expect, it } from 'vitest' + +import * as schema from '../src/db/schema.js' +import { parallelConfig } from '../src/push/parallel.js' +import { cleanup, type Harness, harness } from './harness.js' + +afterAll(cleanup) + +const defaults = { ...parallelConfig } +afterEach(() => { + Object.assign(parallelConfig, defaults) +}) + +const Author = { type: 'object', properties: { name: { type: 'string' } } } +const rec = (id: string, name: string, priv = false) => ({ + id, + type: 'Author', + data: { name }, + ...(priv ? { private: true } : {}), +}) +const id = (i: number) => `a${String(i).padStart(5, '0')}` + +async function push( + h: Harness, + user: string, + path: string, + open: Record, + upserts: unknown[], + deletes: string[], + parallel: boolean, +) { + const res = await h.request(`${path}/push`, { method: 'POST', user, json: open }) + expect(res.status).toBe(200) + const sid = ((await res.json()) as { session_id: string }).session_id + if (upserts.length) { + const r = await h.request(`${path}/push/${sid}/records`, { + method: 'POST', + user, + ndjson: upserts, + }) + expect(r.status).toBe(200) + } + if (deletes.length) { + const r = await h.request(`${path}/push/${sid}/deletes`, { + method: 'POST', + user, + ndjson: deletes.map((d) => ({ type: 'Author', id: d })), + }) + expect(r.status).toBe(200) + } + Object.assign( + parallelConfig, + parallel ? { aboveBytes: 0, unitBytes: 32_000, baseLevel: 0 } : { aboveBytes: Infinity }, + ) + const commit = await h.request(`${path}/push/${sid}/commit`, { method: 'POST', user }) + expect(commit.status).toBe(parallel ? 202 : 201) + await h.drain() +} + +/** Each version's hash, by seq. */ +async function versions(h: Harness, collectionId: string) { + const rows = await h.ports.db + .select({ seq: schema.versions.seq, hash: schema.versions.hash }) + .from(schema.versions) + .where(eq(schema.versions.collectionId, collectionId)) + return rows.sort((a, b) => a.seq - b.seq).map((r) => r.hash) +} + +/** Every record of the head version, with its body, read from its set alone. */ +async function records(h: Harness, collectionId: string) { + const [c] = await h.ports.db + .select() + .from(schema.collections) + .where(eq(schema.collections.id, collectionId)) + const [v] = await h.ports.db + .select() + .from(schema.versions) + .where(eq(schema.versions.id, c!.headVersionId!)) + const repo = await h.ports.stores.forCollection(collectionId) + const root = await repo.root(v!.hash) + const out: string[] = [] + const sets: [RecordSet, string | null][] = [['public', root.public.types.Author?.root ?? null]] + if (root.private) + sets.push(['private', (await repo.privateSet(root.private)).types.Author?.root ?? null]) + for (const [set, tree] of sets) + for await (const e of iterate(new RepoSource(recordTree, repo, { set }), tree, { + payloads: true, + })) + out.push(`${set} ${e.key} ${e.body}`) + return { repo, out } +} + +describe("today's layout and layout F side by side", () => { + it('give the same versions, read the same records, and both pass fsck', async () => { + const h = await harness() + const user = await h.member() + const today = await h.collection('today', { recordLayout: 'today' }) + const f = await h.collection('f', { recordLayout: 'f' }) + // The private set's commitment is salted per collection: the same salt, the same hashes. + await h.ports.db + .update(schema.collections) + .set({ privateSalt: today.privateSalt }) + .where(eq(schema.collections.id, f.id)) + const paths = { today: '/api/collections/org/today', f: '/api/collections/org/f' } + + const first = [ + ...Array.from({ length: 3000 }, (_, i) => rec(id(i), `n${i}`)), + ...Array.from({ length: 20 }, (_, i) => rec(id(5000 + i), `p${i}`, true)), + ] + const second = Array.from({ length: 200 }, (_, i) => rec(id(i * 7), `again${i}`)) + const gone = Array.from({ length: 50 }, (_, i) => id(i * 11 + 1)) + const third = Array.from({ length: 400 }, (_, i) => rec(id(i * 5 + 2), `third${i}`)) + // Interleaved, so each layout reads nodes the other wrote first. + for (const [upserts, deletes, parallel] of [ + [first, [], false], + [second, gone, false], + [third, [], true], + ] as const) { + for (const which of ['today', 'f'] as const) { + const open = upserts === first ? { schemas: { Author } } : {} + await push(h, user, paths[which], open, [...upserts], [...deletes], parallel) + } + expect(await versions(h, f.id)).toEqual(await versions(h, today.id)) + } + expect(await versions(h, f.id)).toHaveLength(3) + + const a = await records(h, today.id) + const b = await records(h, f.id) + expect(b.out).toEqual(a.out) + // The records the pushes leave: a push's deletes, sent after its upserts, win over + // them, and the third push brings some deleted ones back. + const live = new Set(first.map((r) => r.id)) + for (const d of gone) live.delete(d) + for (const r of third) live.add(r.id) + expect(a.out.length).toBe(live.size) + expect(b.repo.records).not.toBeNull() + expect(a.repo.records).toBeNull() + + for (const c of [today, f]) { + const repo = await h.ports.stores.forCollection(c.id) + const report = await fsck(repo, { + collectionId: c.id, + trustedKeys: [h.signer.publicKey], + fileBytes: true, + }) + expect(report.errors).toEqual([]) + } + // F's records are its own files, a set apart; today's are leaf bodies. + const keys = [...h.bucket.objects.keys()] + expect(keys.some((k) => k.startsWith(`repo/runs/${f.id}/public/packs/`))).toBe(true) + expect(keys.some((k) => k.startsWith(`repo/runs/${f.id}/private/packs/`))).toBe(true) + expect(keys.some((k) => k.startsWith(`repo/runs/${today.id}/`))).toBe(false) + const sha = (s: string) => createHash('sha256').update(s).digest('hex') + expect(sha(a.out.join('\n'))).toBe(sha(b.out.join('\n'))) + }, 60_000) +}) diff --git a/packages/server/test/manage.test.ts b/packages/server/test/manage.test.ts index 43d532ef..06f36ca3 100644 --- a/packages/server/test/manage.test.ts +++ b/packages/server/test/manage.test.ts @@ -135,7 +135,11 @@ describe('collection management', () => { forkedFrom: { version: 'v1.0.2' }, version: { recordCount: 1 }, }) - expect(h.bucket.puts - putsBefore).toBeLessThanOrEqual(4) // root, log entry, head.json + // Root, log entry, head.json; layout F adds the fork's list of its parent's files, a + // marker in the parent, and the fork's manifest and its head (the public set's alone). + expect(h.bucket.puts - putsBefore).toBeLessThanOrEqual( + h.ports.stores.newCollections === 'f' ? 8 : 4, + ) const [child] = await h.ports.db .select() .from(schema.collections) diff --git a/packages/server/test/mirrors.test.ts b/packages/server/test/mirrors.test.ts index f905a611..5d1920df 100644 --- a/packages/server/test/mirrors.test.ts +++ b/packages/server/test/mirrors.test.ts @@ -1,6 +1,7 @@ import { createHash } from 'node:crypto' import { + headConfig, iterate, openRepo, PrefixedStore, @@ -14,7 +15,7 @@ import { verifyTree, } from '@underlay/protocol' import { eq } from 'drizzle-orm' -import { afterAll, afterEach, describe, expect, it } from 'vitest' +import { afterAll, afterEach, beforeEach, describe, expect, it } from 'vitest' import { type FakeS3, startFakeS3 } from '../../protocol/test/stores/fake-s3.js' import * as schema from '../src/db/schema.js' @@ -97,14 +98,22 @@ async function addLocation( } /** The mirror as a reader sees it: a repository in the bucket, verified. */ -function mirrorRepo(f: FakeS3, prefix: string): Repo { +/** A mirror's bucket as a third party opens it: in the collection's record layout, when given. */ +function mirrorRepo( + f: FakeS3, + prefix: string, + c?: { id: string; recordLayout: 'today' | 'f' }, +): Repo { const store = s3Store({ endpoint: f.url, bucket: 'bucket', accessKeyId: 'AKID', secretAccessKey: 'secret', }) - return openRepo(new PrefixedStore(store, prefix)) + return openRepo( + new PrefixedStore(store, prefix), + c?.recordLayout === 'f' ? { layout: 'f', recordScope: c.id, nodes: 'delta' } : {}, + ) } async function placements(h: Harness, user: string, path: string) { @@ -124,8 +133,8 @@ async function placements(h: Harness, user: string, path: string) { } } -async function setup() { - const h = await harness() +async function setup(storage: Parameters[0] = {}) { + const h = await harness(storage) const user = await h.member() const c = await h.collection('books') const path = '/api/collections/org/books' @@ -151,6 +160,103 @@ async function setup() { return { h, user, c, path, cover, v1, v2 } } +describe('bucket mirrors of layout F (records in packs of the collection)', () => { + // F moves a manifest's head at most once a second; the tests' jobs needn't wait. + beforeEach(() => void (headConfig.intervalMs = 25)) + afterEach(() => void (headConfig.intervalMs = 1000)) + + it('copies the record files as written, and the bucket reads with no database', async () => { + const { h, user, c, path, v2 } = await setup({ newCollections: 'f' }) + const f = await fake() + const loc = await addLocation(h, user, f, 'mirror') + const created = await h.request(`${path}/placements`, { + method: 'POST', + user, + json: { locationId: loc.body.location.id, sets: 'public+private' }, + }) + expect(created.status).toBe(201) + await h.drain() + const mirror = (await placements(h, user, path)).placements.find((p) => p.role === 'mirror')! + expect(mirror).toMatchObject({ syncedSeq: 2, state: 'active', lastError: null }) + + // Opened as layout F over the bucket alone: every record reads, every tree verifies. + const store = s3Store({ + endpoint: f.url, + bucket: 'bucket', + accessKeyId: 'AKID', + secretAccessKey: 'secret', + }) + const repo = openRepo(new PrefixedStore(store, 'mirror'), { + layout: 'f', + recordScope: c.id, + nodes: 'delta', + }) + expect(await readHead(repo, c.id)).toMatchObject({ seq: 2, versionHash: v2.hash }) + const root = await repo.root(v2.hash) + const priv = await repo.privateSet(root.private!) + for (const [set, t] of [ + ['public', root.public.types.Book!], + ['private', priv.types.Book!], + ] as const) { + const src = new RepoSource(recordTree, repo, { set }) + expect((await verifyTree(src, t.root)).errors).toEqual([]) + let n = 0 + for await (const e of iterate(src, t.root, { payloads: true })) if (e.body) n++ + expect(n).toBe(t.count) + } + // No leaf bodies: the records are the collection's packs, a set's apart, copied as they are. + const put = (part: string) => f.requests.some((r) => r.method === 'PUT' && r.url.includes(part)) + expect(put('/bodies/')).toBe(false) + expect(put(`/runs/${c.id}/public/packs/`)).toBe(true) + expect(put(`/runs/${c.id}/private/packs/`)).toBe(true) + + // The next version copies only what changed. + const before = f.requests.length + await push(h, user, path, {}, [book('b5', 'again')]) + await h.drain() + expect( + (await placements(h, user, path)).placements.find((p) => p.role === 'mirror')!.syncedSeq, + ).toBe(3) + expect(f.requests.slice(before).filter((r) => r.method === 'PUT').length).toBeLessThan(30) + }) + + it('copies the public set alone to a public-only mirror of a collection with private records', async () => { + // Someone who doesn't own a collection may mirror it: they get its public records only. + const { h, user, c, path, v2 } = await setup({ newCollections: 'f' }) + const f = await fake() + const loc = await addLocation(h, user, f, 'mirror') + await h.request(`${path}/placements`, { + method: 'POST', + user, + json: { locationId: loc.body.location.id, sets: 'public' }, + }) + await h.drain() + const mirror = (await placements(h, user, path)).placements.find((p) => p.role === 'mirror')! + expect(mirror).toMatchObject({ syncedSeq: 2, state: 'active', lastError: null }) + const put = (part: string) => f.requests.some((r) => r.method === 'PUT' && r.url.includes(part)) + expect(put(`/runs/${c.id}/public/packs/`)).toBe(true) + expect(put(`/runs/${c.id}/private/`)).toBe(false) + // The public set reads from the bucket alone. + const store = s3Store({ + endpoint: f.url, + bucket: 'bucket', + accessKeyId: 'AKID', + secretAccessKey: 'secret', + }) + const repo = openRepo(new PrefixedStore(store, 'mirror'), { + layout: 'f', + recordScope: c.id, + nodes: 'delta', + }) + const t = (await repo.root(v2.hash)).public.types.Book! + const src = new RepoSource(recordTree, repo, { set: 'public' }) + expect((await verifyTree(src, t.root)).errors).toEqual([]) + let n = 0 + for await (const e of iterate(src, t.root, { payloads: true })) if (e.body) n++ + expect(n).toBe(t.count) + }) +}) + describe('bucket mirrors', () => { it('adds a location, mirrors the history, and the bucket reads as a repository', async () => { const { h, user, c, path, cover, v2 } = await setup() @@ -179,7 +285,7 @@ describe('bucket mirrors', () => { }) // A third party with read access sees a complete, verifiable repository. - const repo = mirrorRepo(f, 'mirror') + const repo = mirrorRepo(f, 'mirror', c) const head = await readHead(repo, c.id) expect(head).toMatchObject({ seq: 2, versionHash: v2.hash }) const info = await readCollectionInfo(repo, c.id) @@ -187,8 +293,11 @@ describe('bucket mirrors', () => { expect(entries).toHaveLength(2) const root = await repo.root(v2.hash) const priv = await repo.privateSet(root.private!) - const src = new RepoSource(recordTree, repo) - for (const t of [root.public.types.Book!, priv.types.Book!]) { + for (const [set, t] of [ + ['public', root.public.types.Book!], + ['private', priv.types.Book!], + ] as const) { + const src = new RepoSource(recordTree, repo, { set }) expect((await verifyTree(src, t.root)).errors).toEqual([]) let n = 0 for await (const e of iterate(src, t.root, { payloads: true })) if (e.body) n++ diff --git a/packages/server/test/parallel.test.ts b/packages/server/test/parallel.test.ts index cf4c5cf5..7d86220b 100644 --- a/packages/server/test/parallel.test.ts +++ b/packages/server/test/parallel.test.ts @@ -3,6 +3,7 @@ import { gunzipSync } from 'node:zlib' import { boundaryBytes, + iterate, recordTree, RepoSource, trailingZeros, @@ -169,6 +170,52 @@ describe('parallel commit', () => { expect(status.status).toBe('committed') }) + it("names a commit's records in layout F's index once a tree, not once a unit", async () => { + // A unit's runs went in a pending object each: a 30M push left ~3,300, and a cold + // isolate's first manifest read reads them all. + const h = await harness({ newCollections: 'f' }) + const user = await h.member() + const c = await h.collection('fh') + const path = '/api/collections/org/fh' + const id = (i: number) => `r${String(i).padStart(5, '0')}` + const pending: string[] = [] + const put = h.bucket.put.bind(h.bucket) + h.bucket.put = (key, body, opts) => { + if (key.startsWith(`repo/runs/${c.id}/`) && key.includes('/manifest/f/pending/')) + pending.push(key) + return put(key, body, opts) + } + // Marks are where units can be cut: a dozen units. + const m = markIds('fm', 12) + const records = [ + ...Array.from({ length: 6000 }, (_, i) => rec(id(i), `n${i}`)), + ...m.map((k) => rec(k, k)), + ] + const { sid } = await push(h, user, path, { schemas: { Author } }, [records], [], { + unitBytes: 1, + }) + h.bucket.put = put + const units = await h.ports.db + .select() + .from(schema.commitUnits) + .where(eq(schema.commitUnits.sessionId, sid)) + expect(units.filter((u) => !u.gap).length).toBeGreaterThan(4) + // One type in one set: one object, however many units. + expect(pending.length).toBe(1) + const t = await trees(h, c.id) + expect(t.public.count).toBe(6012) + let n = 0 + for await (const e of iterate( + new RepoSource(recordTree, t.repo, { set: 'public' }), + t.public.root, + { + payloads: true, + }, + )) + if (e.body) n++ + expect(n).toBe(6012) + }) + it('builds the same trees as the serial commit', async () => { const h = await harness() const user = await h.member() @@ -307,7 +354,8 @@ describe('parallel commit', () => { }, 60_000) it('keeps large base records as pointers when units rewrite their leaves', async () => { - const h = await harness() + // Pointers in leaf bodies: today's layout's (F keeps records out of leaves). + const h = await harness({ newCollections: 'today' }) const user = await h.member() const a = await h.collection('ka') const b = await h.collection('kb') diff --git a/packages/server/test/recover.test.ts b/packages/server/test/recover.test.ts index 8f79a2be..dcd46866 100644 --- a/packages/server/test/recover.test.ts +++ b/packages/server/test/recover.test.ts @@ -328,6 +328,53 @@ describe('recovering stalled commits', () => { expect(res.status).toBe(201) }) + it("writes a compaction's run index once when its last pieces finish together", async () => { + // Each of the last two pieces sees every piece done and joins them: the index was + // written twice within a second, which R2 refuses (the scale bench, 2026-10-19). + const h = await harness() + const user = await h.member() + await h.collection('join') + const path = '/api/collections/org/join' + Object.assign(compactConfig, { pieceBytes: 1 }) + const all = ids(MERGE_FAN_IN * 5) + const batches = Array.from({ length: MERGE_FAN_IN }, (_, i) => + all.slice(i * 5, i * 5 + 5).map(rec), + ) + const sid = await upload(h, user, path, batches) + const jobs = (type: string) => + h.ports.db.select().from(schema.jobs).where(eq(schema.jobs.type, type)) + while ((await jobs('push.compactPiece')).length === 0) + expect(await drainSqliteJobs(h.ports, { max: 1 })).toBe(1) + const pieces = (await jobs('push.compactPiece')).map( + (j) => (j.payload as unknown as { piece: number }).piece, + ) + expect(pieces.length).toBeGreaterThan(1) + const [run] = await h.ports.db + .select() + .from(schema.pushRuns) + .where(eq(schema.pushRuns.sessionId, sid)) + const seq = run!.mergingInto! + const internal = h.ports.stores.internal + const put = internal.put.bind(internal) + const writes: { ifAbsent: boolean }[] = [] + internal.put = (key, body, opts) => { + if (key.endsWith(`/runs/${seq}/index.json`)) writes.push({ ifAbsent: !!opts?.ifAbsent }) + return put(key, body, opts) + } + // Every piece at once: the last ones finish together. + await Promise.all(pieces.map((p) => compactPiece(h.ports, sid, seq, p))) + internal.put = put + expect(writes.length).toBeGreaterThan(0) + expect(writes.every((w) => w.ifAbsent)).toBe(true) + await dropJobs(h, 'push.compactPiece') + await h.drain() + const runs = await h.ports.db + .select() + .from(schema.pushRuns) + .where(eq(schema.pushRuns.sessionId, sid)) + expect(runs.map((r) => [r.tier, r.mergingInto])).toEqual([[1, null]]) + }) + it('leaves a long compaction alone while its pieces keep finishing', async () => { const h = await harness() const user = await h.member() diff --git a/packages/server/test/schema-parallel.test.ts b/packages/server/test/schema-parallel.test.ts index 5ecb36cb..17a6caa2 100644 --- a/packages/server/test/schema-parallel.test.ts +++ b/packages/server/test/schema-parallel.test.ts @@ -6,6 +6,7 @@ */ import { createHash } from 'node:crypto' +import { iterate, type RecordSet, RepoSource, recordTree } from '@underlay/protocol' import { eq } from 'drizzle-orm' import { afterAll, afterEach, describe, expect, it } from 'vitest' @@ -118,6 +119,17 @@ async function planKeys(h: Harness, sid: string) { return out } +/** Records of a tree read with their bodies from one set alone (layout F keeps them apart). */ +async function readsFrom(h: Harness, collectionId: string, set: RecordSet, root: string | null) { + const repo = await h.ports.stores.forCollection(collectionId) + let n = 0 + for await (const e of iterate(new RepoSource(recordTree, repo, { set }), root, { + payloads: true, + })) + if (e.body) n++ + return n +} + async function planOf(h: Harness, sid: string) { const key = (await planKeys(h, sid)).find((k) => k.endsWith('/plan.json'))! return JSON.parse(await (await h.ports.stores.internal.get(key))!.text()) as { @@ -224,13 +236,18 @@ describe('schema and privacy changes in a parallel commit', () => { const pa = await push(h, user, paths.a, { schemas: { Doc: PrivateDoc } }, changes, 'parallel') const pb = await push(h, user, paths.b, { schemas: { Doc: PrivateDoc } }, changes, 'serial') expect(pa.status.status).toBe('committed') - expect(pa.parallel).toBe(true) - expect((await planOf(h, pa.sid)).types.Doc!.move).toBe('toPrivate') + // Layout F copies a moved type's records into the other set's packs, in the serial build. + if (a.recordLayout === 'today') { + expect(pa.parallel).toBe(true) + expect((await planOf(h, pa.sid)).types.Doc!.move).toBe('toPrivate') + } else expect(pa.parallel).toBe(false) expect(pb.status.status).toBe('committed') const ha = await head(h, a.id) expect(ha).toEqual(await head(h, b.id)) expect(ha.public.types.Doc).toBeUndefined() expect(ha.private?.types.Doc?.count).toBe(300) + // Every record of the moved type reads from its new set alone. + expect(await readsFrom(h, a.id, 'private', ha.private!.types.Doc!.root)).toBe(300) }, 30_000) it('makes a private type public by moving its tree, as the serial commit does', async () => { @@ -243,11 +260,16 @@ describe('schema and privacy changes in a parallel commit', () => { const pa = await push(h, user, paths.a, { schemas: { Doc } }, changes, 'parallel') const pb = await push(h, user, paths.b, { schemas: { Doc } }, changes, 'serial') expect(pa.status.status).toBe('committed') - expect(pa.parallel).toBe(true) - expect((await planOf(h, pa.sid)).types.Doc!.move).toBe('toPublic') + if (a.recordLayout === 'today') { + expect(pa.parallel).toBe(true) + expect((await planOf(h, pa.sid)).types.Doc!.move).toBe('toPublic') + } else expect(pa.parallel).toBe(false) expect(pb.status.status).toBe('committed') expect(pa.status.result.changes).toEqual(pb.status.result.changes) const ha = await head(h, a.id) + expect(await readsFrom(h, a.id, 'public', ha.public.types.Doc!.root)).toBe( + ha.public.types.Doc!.count, + ) expect(ha).toEqual(await head(h, b.id)) expect(ha.public.types.Doc?.count).toBe(340) }, 30_000) diff --git a/packages/server/vitest.config.ts b/packages/server/vitest.config.ts index 3e427974..ad15914c 100644 --- a/packages/server/vitest.config.ts +++ b/packages/server/vitest.config.ts @@ -1,7 +1,22 @@ import { defineConfig } from 'vitest/config' +// The suite runs once a record layout: new collections on today's (leaf bodies), then +// on layout F (packs per collection). Both are live while collections convert. export default defineConfig({ test: { - include: ['test/**/*.test.ts'], + projects: [ + { + extends: true, + test: { + name: 'today', + include: ['test/**/*.test.ts'], + env: { UNDERLAY_TEST_LAYOUT: 'today' }, + }, + }, + { + extends: true, + test: { name: 'f', include: ['test/**/*.test.ts'], env: { UNDERLAY_TEST_LAYOUT: 'f' } }, + }, + ], }, }) From f61babd1c1f192157e938af55f6c81b851755e8b Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Tue, 20 Oct 2026 12:19:05 -0400 Subject: [PATCH 2/3] Layout F: no private-set maintain for a collection without private records; every record read names its set --- packages/protocol/src/repo/build.ts | 13 ++++--- packages/protocol/src/repo/file-sets.ts | 3 +- packages/server/src/api/versions.ts | 10 +++-- packages/server/src/handlers.ts | 18 +++++---- packages/server/src/refs/log.ts | 8 +++- packages/server/test/layouts.test.ts | 49 +++++++++++++++++++++++++ 6 files changed, 83 insertions(+), 18 deletions(-) diff --git a/packages/protocol/src/repo/build.ts b/packages/protocol/src/repo/build.ts index 83384a1d..d2ceeb7e 100644 --- a/packages/protocol/src/repo/build.ts +++ b/packages/protocol/src/repo/build.ts @@ -167,12 +167,15 @@ async function* overlay( } } -/** Every entry of a tree (with bodies) as upserts. */ +/** Every entry of a tree (with bodies) as upserts, read from the tree's set. */ async function* treeAsUpserts( repo: Repo, root: string | null, + set: SetName, ): AsyncGenerator> { - for await (const e of iterate(new RepoSource(recordTree, repo), root, { payloads: true })) { + for await (const e of iterate(new RepoSource(recordTree, repo, { set }), root, { + payloads: true, + })) { yield { key: e.key, entry: e } } } @@ -286,7 +289,7 @@ export async function buildVersion(repo: Repo, input: BuildInput): Promise t.public?.root && (!want || t.slug === want)) if (want && types.length === 0) return jsonError(c, 404, `No public records of type ${want}`) - const source = new RepoSource(recordTree, view.repo) + const source = new RepoSource(recordTree, view.repo, { set: 'public' }) const members = async function* () { for (const t of types) { for await (const leaf of leaves(source, t.public!.root)) { @@ -785,7 +785,11 @@ async function* diffAll( cursor: { t: string; k: string } | null, { moves = false }: { moves?: boolean } = {}, ): AsyncGenerator { - const source = new RepoSource(recordTree, to.repo) + // A set's records from its own source: layout F keeps them apart. + const sources = { + public: new RepoSource(recordTree, to.repo, { set: 'public' }), + private: new RepoSource(recordTree, to.repo, { set: 'private' }), + } // A change to a blocked record (the denylist) isn't served. const shown = (d: TypedDiff) => !(d.after && to.withheld.has(d.after.hash)) const slugs = new Set([...(from?.types.map((t) => t.slug) ?? []), ...to.types.map((t) => t.slug)]) @@ -799,7 +803,7 @@ async function* diffAll( const side = (set: 'public' | 'private') => set === 'private' && !to.owner ? null - : diffTrees(source, a?.[set]?.root ?? null, b?.[set]?.root ?? null, after)[ + : diffTrees(sources[set], a?.[set]?.root ?? null, b?.[set]?.root ?? null, after)[ Symbol.asyncIterator ]() const pub = side('public') diff --git a/packages/server/src/handlers.ts b/packages/server/src/handlers.ts index 3f95cd58..1c20c0b6 100644 --- a/packages/server/src/handlers.ts +++ b/packages/server/src/handlers.ts @@ -48,20 +48,24 @@ registerJob('version.published', async (job, ports) => { await enqueueDeliveries(ports, String(job.versionId), job.bump as BumpType) await ports.jobs.enqueue({ type: 'refs.index', versionId: String(job.versionId) }) const [v] = await ports.db - .select({ collectionId: schema.versions.collectionId }) + .select({ collectionId: schema.versions.collectionId, hash: schema.versions.hash }) .from(schema.versions) .where(eq(schema.versions.id, String(job.versionId))) if (v) await queueMirrors(ports, v.collectionId) - // Layout F merges a collection's packs and index runs after a commit, a set a job - // (a set with nothing due leaves after a few reads). - if (v && (await ports.stores.forCollection(v.collectionId)).records) + // Layout F merges a collection's packs and index runs after a commit, a set a job. The + // private set's only when the version has one: an idle job is a bulk slot and a dozen + // storage calls, and a collection with no private records would pay them every commit. + const repo = v && (await ports.stores.forCollection(v.collectionId)) + if (v && repo?.records) { + const sets = (await repo.root(v.hash)).private + ? (['public', 'private'] as const) + : (['public'] as const) await ports.jobs.enqueueBatch( - RECORD_SETS.map((set) => ({ type: 'layout.maintain', collectionId: v.collectionId, set })), + sets.map((set) => ({ type: 'layout.maintain', collectionId: v.collectionId, set })), ) + } }) -const RECORD_SETS = ['public', 'private'] as const - /** A job's record set, and the collection's layout F store for it (null: not on F). */ async function setLayout(ports: Ports, job: JobMessage): Promise { const repo = await ports.stores.forCollection(String(job.collectionId)) diff --git a/packages/server/src/refs/log.ts b/packages/server/src/refs/log.ts index 371b7347..4d2a6c9f 100644 --- a/packages/server/src/refs/log.ts +++ b/packages/server/src/refs/log.ts @@ -86,7 +86,11 @@ export async function* versionEvents( ): AsyncGenerator { const repo = await ports.stores.forCollection(version.collectionId) const { now, before } = await viewsOf(ports, version) - const records = new RepoSource(recordTree, repo) + // A set's records from its own source: layout F keeps them apart. + const sources = { + public: new RepoSource(recordTree, repo, { set: 'public' }), + private: new RepoSource(recordTree, repo, { set: 'private' }), + } const files = new RepoSource(fileTree, repo) const c = version.collectionId const seq = version.seq @@ -103,7 +107,7 @@ export async function* versionEvents( if (scope && scope.slug !== slug) continue const a = before?.types.find((t) => t.slug === slug)?.[set]?.root ?? null const b = now.types.find((t) => t.slug === slug)?.[set]?.root ?? null - for await (const d of diffTrees(records, a, b, { after })) { + for await (const d of diffTrees(sources[set], a, b, { after })) { if (past(d.key)) break if (d.before) yield [d.before.hash, 'r', c, set, seq, '-', slug, d.key] if (d.after) yield [d.after.hash, 'r', c, set, seq, '+', slug, d.key] diff --git a/packages/server/test/layouts.test.ts b/packages/server/test/layouts.test.ts index abc9b347..8a51dfb7 100644 --- a/packages/server/test/layouts.test.ts +++ b/packages/server/test/layouts.test.ts @@ -100,6 +100,55 @@ async function records(h: Harness, collectionId: string) { return { repo, out } } +describe('layout F maintenance', () => { + it("queues no private set's maintain for a collection with no private records", async () => { + // An idle maintain is a bulk slot and a dozen storage calls a commit: with thousands + // of collections committing, the queue they share is what loads wait on. + const h = await harness() + const user = await h.member() + await h.collection('pub', { recordLayout: 'f' }) + await h.collection('mixed', { recordLayout: 'f' }) + const queued: { set: unknown; collectionId: unknown }[] = [] + const { enqueue, enqueueBatch } = h.ports.jobs + h.ports.jobs.enqueue = (job, o) => { + if (job.type === 'layout.maintain') + queued.push({ set: job.set, collectionId: job.collectionId }) + return enqueue.call(h.ports.jobs, job, o) + } + h.ports.jobs.enqueueBatch = (jobs) => { + for (const job of jobs) + if (job.type === 'layout.maintain') + queued.push({ set: job.set, collectionId: job.collectionId }) + return enqueueBatch.call(h.ports.jobs, jobs) + } + const all = Array.from({ length: 300 }, (_, i) => rec(id(i), `n${i}`)) + await push(h, user, '/api/collections/org/pub', { schemas: { Author } }, all, [], false) + await push( + h, + user, + '/api/collections/org/mixed', + { schemas: { Author } }, + [...all, rec(id(9000), 'p', true)], + [], + false, + ) + const [pub, mixed] = await Promise.all( + ['pub', 'mixed'].map(async (slug) => { + const [c] = await h.ports.db + .select() + .from(schema.collections) + .where(eq(schema.collections.slug, slug)) + return c!.id + }), + ) + const sets = (c: string) => [ + ...new Set(queued.filter((q) => q.collectionId === c).map((q) => q.set)), + ] + expect(sets(pub!)).toEqual(['public']) + expect(sets(mixed!).sort()).toEqual(['private', 'public']) + }) +}) + describe("today's layout and layout F side by side", () => { it('give the same versions, read the same records, and both pass fsck', async () => { const h = await harness() From 827c76a115773583826514f4cd067712894f2f43 Mon Sep 17 00:00:00 2001 From: Travis Rich Date: Tue, 20 Oct 2026 12:42:15 -0400 Subject: [PATCH 3/3] Tests: room for a slow CI runner (a shorter F head interval in tests, a 20 s default timeout for the doubled server suite) --- packages/protocol/test/repo/layouts.test.ts | 8 +++++--- packages/server/test/setup.ts | 10 ++++++++++ packages/server/vitest.config.ts | 3 +++ 3 files changed, 18 insertions(+), 3 deletions(-) create mode 100644 packages/server/test/setup.ts diff --git a/packages/protocol/test/repo/layouts.test.ts b/packages/protocol/test/repo/layouts.test.ts index 3131e83e..0ec1becd 100644 --- a/packages/protocol/test/repo/layouts.test.ts +++ b/packages/protocol/test/repo/layouts.test.ts @@ -213,7 +213,7 @@ describe('merges keep the newest place of every record', () => { layoutConfig.graceMs = saved headConfig.intervalMs = 1000 } - }) + }, 30_000) }) describe('merges split into key-range jobs', () => { @@ -275,7 +275,7 @@ describe('merges split into key-range jobs', () => { } finally { Object.assign(layoutConfig, saved) } - }) + }, 30_000) }) describe('a backlog of tiny commits', () => { @@ -326,6 +326,8 @@ describe('a backlog of tiny commits', () => { describe('pack merges', () => { it('reach every tier while commits keep landing', async () => { // One pass a job, with commits between: tier 0 always has enough (FANOUT) to merge. + // Not a test of the head: its interval only makes the 120 jobs wait. + headConfig.intervalMs = 1 const saved = { ...layoutConfig } layoutConfig.maintainBudgetMs = 0 try { @@ -347,7 +349,7 @@ describe('pack merges', () => { } finally { Object.assign(layoutConfig, saved) } - }) + }, 30_000) }) describe('layout F', () => { diff --git a/packages/server/test/setup.ts b/packages/server/test/setup.ts new file mode 100644 index 00000000..b347aa26 --- /dev/null +++ b/packages/server/test/setup.ts @@ -0,0 +1,10 @@ +/** + * Every test file's setup. Layout F writes a manifest's head at most once a second + * (R2's limit a key), and each maintain job waits that out: the tests wait a fortieth. + */ +import { headConfig } from '@underlay/protocol' +import { beforeEach } from 'vitest' + +beforeEach(() => { + headConfig.intervalMs = 25 +}) diff --git a/packages/server/vitest.config.ts b/packages/server/vitest.config.ts index ad15914c..6e1f7725 100644 --- a/packages/server/vitest.config.ts +++ b/packages/server/vitest.config.ts @@ -4,6 +4,9 @@ import { defineConfig } from 'vitest/config' // on layout F (packs per collection). Both are live while collections convert. export default defineConfig({ test: { + setupFiles: ['test/setup.ts'], + // The suite runs twice and F's jobs do more per commit: room for a slow CI runner. + testTimeout: 20_000, projects: [ { extends: true,