diff --git a/packages/protocol/src/index.ts b/packages/protocol/src/index.ts index 9b78817..6e62201 100644 --- a/packages/protocol/src/index.ts +++ b/packages/protocol/src/index.ts @@ -21,13 +21,17 @@ export { type MergePlan, type PackInfo, MissingRecordError, + RECLAIM_MAX_RUNS, type ReclaimPlan, + recordIdKey, recordKey, type RecordSet, type RecordSets, setHome, + TooManyRunsError, + type TypedLeaf, } from './repo/layouts/index.js' -export { LiveHashes, markRange, recordRoots } from './repo/layouts/reclaim.js' +export { LiveHashes, markRange, recordRoots, type TypedRoot } from './repo/layouts/reclaim.js' export { fsck, type FsckCursor, diff --git a/packages/protocol/src/repo/build.ts b/packages/protocol/src/repo/build.ts index 5208cb2..bcac509 100644 --- a/packages/protocol/src/repo/build.ts +++ b/packages/protocol/src/repo/build.ts @@ -167,13 +167,14 @@ async function* overlay( } } -/** Every entry of a tree (with bodies) as upserts, read from the tree's set. */ +/** Every entry of a type's tree (with bodies) as upserts, read from the tree's set. */ async function* treeAsUpserts( repo: Repo, root: string | null, set: SetName, + type: string, ): AsyncGenerator> { - for await (const e of iterate(new RepoSource(recordTree, repo, { set }), root, { + for await (const e of iterate(new RepoSource(recordTree, repo, { set, type }), root, { payloads: true, })) { yield { key: e.key, entry: e } @@ -186,17 +187,23 @@ async function* treeAsUpserts( * (a public mirror copies them as they are). O(type size), as a rebuild is; the * old set's copies go when its reclaim finds them unreferenced. */ -async function copyRecords(repo: Repo, root: string | null, from: SetName, to: SetName) { +async function copyRecords( + repo: Repo, + root: string | null, + type: string, + from: SetName, + to: SetName, +) { if (!repo.records || root === null) return - const source = new RepoSource(recordTree, repo, { set: from }) + const source = new RepoSource(recordTree, repo, { set: from, type }) const writer = repo.records[to].writer() for await (const leaf of leaves(source, root)) { const node = await source.node(leaf.hash) const entries = (node.kind === 'leaf' ? node.entries : []) as RecordEntry[] for (let i = 0; i < entries.length; i += 256) { const at = Array.from({ length: Math.min(256, entries.length - i) }, (_, j) => i + j) - const lines = await repo.records[from].linesAt(leaf.hash, entries, at) - at.forEach((k, j) => writer.add(entries[k]!, lines[j]!)) + const lines = await repo.records[from].linesAt({ hash: leaf.hash, type, entries }, at) + at.forEach((k, j) => writer.add(type, entries[k]!, lines[j]!)) await writer.drain?.() } } @@ -214,11 +221,11 @@ export async function buildVersion(repo: Repo, input: BuildInput): Promise new RepoSource(recordTree, repo, { set, type }) + // Nodes alone: any source reads them. + const nodes = new RepoSource(recordTree, repo) const sinks = { public: new RepoSink(repo, { bodyOf: bodyOfRecord, set: 'public' }), private: new RepoSink(repo, { bodyOf: bodyOfRecord, set: 'private' }), @@ -242,7 +249,8 @@ export async function buildVersion(repo: Repo, input: BuildInput): Promise>, ) => { - const result = await mergeTree(sources[set], sinks[set], baseTree, changes, { + sinks[set].type = type + const result = await mergeTree(source(set, type), sinks[set], baseTree, changes, { ...builderOpts, oldPayloads: !refs.noOldRefs.has(set), onChange: (before, after) => { @@ -280,7 +288,7 @@ export async function buildVersion(repo: Repo, input: BuildInput): Promise { + if (!repo.records) { + await repo.bodyLines(leaf, set) + return + } + for (let i = 0; i < leaf.entries.length; i += 256) { + const at = Array.from({ length: Math.min(256, leaf.entries.length - i) }, (_, j) => i + j) + await repo.linesAt(leaf, at, set) + } +} export interface FsckOptions { /** Version hashes to check; with a collection id and none given, the log's. */ @@ -149,7 +165,11 @@ export async function fsck(given: Repo, opts: FsckOptions = {}): Promise records.node(leaf.hash)) if (!node || node.kind !== 'leaf') continue await attempt(`body of ${leaf.hash}`, () => - repo.bodyLines({ hash: leaf.hash, entries: node.entries as RecordEntry[] }, rs), + checkLeaf( + repo, + { hash: leaf.hash, type: slug, entries: node.entries as RecordEntry[] }, + rs, + ), ) } } @@ -372,7 +392,7 @@ export async function fsckStep( const node = await attempt(`leaf ${n.hash}`, () => records.node(n.hash)) if (!node || node.kind !== 'leaf') continue await attempt(`body of ${n.hash}`, () => - repo.bodyLines({ hash: n.hash, entries: node.entries as RecordEntry[] }, rs), + checkLeaf(repo, { hash: n.hash, type: slug, entries: node.entries as RecordEntry[] }, rs), ) } } diff --git a/packages/protocol/src/repo/layouts/index.ts b/packages/protocol/src/repo/layouts/index.ts index d6e153d..10de27d 100644 --- a/packages/protocol/src/repo/layouts/index.ts +++ b/packages/protocol/src/repo/layouts/index.ts @@ -11,7 +11,9 @@ export { layoutConfig, type MergePlan, MissingRecordError, + RECLAIM_MAX_RUNS, type ReclaimPlan, + TooManyRunsError, } from './keyed.js' export { headConfig, type LayerInfo, type PackInfo, pendingConfig } from './kit.js' export * from './types.js' diff --git a/packages/protocol/src/repo/layouts/keyed.ts b/packages/protocol/src/repo/layouts/keyed.ts index 61b71e0..8545578 100644 --- a/packages/protocol/src/repo/layouts/keyed.ts +++ b/packages/protocol/src/repo/layouts/keyed.ts @@ -1,7 +1,7 @@ import type { RecordEntry } from '../../tree/node.js' /** * Layout F (record-storage-layout.md): records in packs, found through an index - * keyed by (id, record hash), with no map per leaf. Everything is per collection: + * keyed by (type, id, record hash), with no map per leaf. Everything is per collection: * its packs, index and manifest live under its own prefix (`runs//`), * the manifest as objects (no database), so a collection is maintained on its own * and a mirror copies its files as written. A fork reads its parent's files where @@ -12,8 +12,9 @@ import type { RecordEntry } from '../../tree/node.js' * into one, and small packs of a tier into one, re-pointing their records in a * new layer. Reclaiming keeps only what a mark found live. * - * The record type isn't part of the key (readers of a leaf don't know it), so a - * leaf's index range also holds other types' records with ids in that range. + * Every read and merge holds a bounded amount however large the collection: an + * index read takes a few layers at a time and keeps only the keys it wants, a + * merge's reads share a fixed budget (`mergeRead`), and writers write as they go. */ import { compareUtf8 } from '../../utf8.js' import type { Lru } from '../lru.js' @@ -31,6 +32,7 @@ import { runs, type Manifest, ManifestStore, + mergeRead, mergeSorted, PackRegistry, newId, @@ -47,11 +49,26 @@ import { filterKey, } from './kit.js' import { type KeyRange, LiveHashes } from './reclaim.js' -import { type LiveSet, recordKey, type RecordStore, type RecordWriter } from './types.js' +import { + type LiveSet, + recordKey, + type RecordStore, + type RecordWriter, + type TypedLeaf, +} from './types.js' /** Index entries a reclaim range job works on: its live set is 8 bytes an entry. */ export const RECLAIM_RANGE_ENTRIES = 250_000 +/** + * Index runs past which a reclaim waits for merges: each range job reads every run + * at once (a share of `mergeRead`'s budget each, and each one's directory). + */ +export const RECLAIM_MAX_RUNS = 32 + +/** A reclaim asked for while the index has more than RECLAIM_MAX_RUNS runs: merge, then ask again. */ +export class TooManyRunsError extends Error {} + /** A reclaim's snapshot of the index and packs, and the key ranges its jobs work in. */ export interface ReclaimPlan { id: string @@ -210,64 +227,50 @@ export class KeyedLayout implements RecordStore { return { keys: r.keys, packs: r.packs.filter((p) => !listed.has(p.id)) } } - async #find(m: Manifest, entries: readonly RecordEntry[]): Promise<(string | undefined)[]> { - // A few records: each looked up through the layers' filters, so only files - // that may hold it are read. A run of a leaf: one range over the layers. - const found = - entries.length <= 64 - ? await this.index.lookup(m, [...new Set(entries.map((e) => e.key))]) - : await this.index.range( - m, - `${entries[0]!.key}\u0000`, - `${entries.at(-1)!.key}\u0000\uffff`, - ) - return entries.map((e) => found.get(recordKey(e.key, e.hash))) + /** Each entry's place: a few keys through the filters, a leaf's as one range of its type's. */ + async #find( + m: Manifest, + type: string, + entries: readonly RecordEntry[], + ): Promise<(string | undefined)[]> { + const keys = entries.map((e) => recordKey(type, e.key, e.hash)) + const found = await this.index.values(m, keys) + return keys.map((k) => found.get(k)) } - async leafLines(leafHash: string, entries: readonly RecordEntry[]): Promise { - if (!entries.length) return [] + async leafLines(leaf: TypedLeaf): Promise { + if (!leaf.entries.length) return [] try { - return await this.#leafLines(leafHash, entries, await this.manifests.current()) + return await this.#leafLines(leaf, await this.manifests.current()) } catch (err) { // A head behind the newest manifest (F moves it at most once a second) can name // files a merge has since swept: read again from the newest. if (!(err instanceof Error) || !/^Missing (pack|layer)/.test(err.message)) throw err - return this.#leafLines(leafHash, entries, await this.manifests.latest()) + return this.#leafLines(leaf, await this.manifests.latest()) } } - async #leafLines( - leafHash: string, - entries: readonly RecordEntry[], - m: Manifest, - ): Promise { - let found = await this.#find(m, entries) + async #leafLines(leaf: TypedLeaf, m: Manifest): Promise { + let found = await this.#find(m, leaf.type, leaf.entries) // Another isolate may have committed since this one read the manifest. if (found.some((x) => x === undefined)) - found = await this.#find(await this.manifests.latest(), entries) + found = await this.#find(await this.manifests.latest(), leaf.type, leaf.entries) const missing = found.findIndex((x) => x === undefined) if (missing >= 0) { - const e = entries[missing]! + const e = leaf.entries[missing]! throw new MissingRecordError( - `Layout ${this.name}: leaf ${leafHash} has no stored record ${e.key} (${e.hash})`, + `Layout ${this.name}: leaf ${leaf.hash} has no stored ${leaf.type} record ${e.key} (${e.hash})`, ) } return this.packs.lines(found as string[]) } - linesAt( - leafHash: string, - entries: readonly RecordEntry[], - at: readonly number[], - ): Promise { - return this.leafLines( - leafHash, - at.map((i) => entries[i]!), - ) + linesAt(leaf: TypedLeaf, at: readonly number[]): Promise { + return this.leafLines({ ...leaf, entries: at.map((i) => leaf.entries[i]!) }) } - async entryLine(leafHash: string, entries: readonly RecordEntry[], i: number): Promise { - return (await this.leafLines(leafHash, [entries[i]!]))[0]! + async entryLine(leaf: TypedLeaf, i: number): Promise { + return (await this.linesAt(leaf, [i]))[0]! } writer(opts: { hold?: boolean } = {}): RecordWriter { @@ -432,7 +435,10 @@ export class KeyedLayout implements RecordStore { const out = new RunWriter(this.store, Math.max(...inputs.map((r) => r.level))) let n = 0 // Entries of dropped packs never reach a merge: reclaim rewrites the index as it drops them. - for await (const [k, v] of mergeSorted(inputs.map((r) => runEntries(this.layers, r.files)))) { + const read = mergeRead(inputs.length) + for await (const [k, v] of mergeSorted( + inputs.map((r) => runEntries(this.layers, r.files, undefined, undefined, read)), + )) { out.add(k, v) if (++n % 1024 === 0) await out.ready() } @@ -475,9 +481,11 @@ export class KeyedLayout implements RecordStore { stored += p.bytes } const reader = this.packs + const read = mergeRead(inputs.length) const streams = inputs.map((p) => (async function* () { - for await (const { key, line } of reader.all(p.id)) yield [key, line] as [string, string] + for await (const { key, line } of reader.all(p.id, read)) + yield [key, line] as [string, string] })(), ) const pw = new PackWriter(this.store, { tier: tier + 1 }) @@ -589,7 +597,7 @@ export class KeyedLayout implements RecordStore { async reclaim(live: LiveSet): Promise> { const h = new LiveHashes() for (const k of live.records) { - const z = k.indexOf('\u0000') + const z = k.lastIndexOf('\u0000') h.add(k.slice(0, z), k.slice(z + 1)) } const plan = await this.reclaimPlan() @@ -669,9 +677,12 @@ export class KeyedLayout implements RecordStore { const r = plan.ranges[i]! const out = new RunWriter(this.store, plan.level) const inputs = runs(plan.layers).sort((a, b) => b.seq - a.seq) + const read = mergeRead(inputs.length) let n = 0 for await (const [k, v] of mergeSorted( - inputs.map((x) => runEntries(this.layers, x.files, r.lo ?? undefined, r.hi ?? undefined)), + inputs.map((x) => + runEntries(this.layers, x.files, r.lo ?? undefined, r.hi ?? undefined, read), + ), )) { out.add(k, v) if (++n % 1024 === 0) await out.ready() @@ -755,6 +766,9 @@ export class KeyedLayout implements RecordStore { async reclaimPlan(rangeEntries = RECLAIM_RANGE_ENTRIES): Promise { const m = await this.manifests.latest() + const n = runs(m.layers).length + if (n > RECLAIM_MAX_RUNS) + throw new TooManyRunsError(`Reclaim: ${n} index runs, more than ${RECLAIM_MAX_RUNS}`) const ranges = await this.#cutRanges(m.layers, rangeEntries) const registry = await this.#registered(m) const plan: ReclaimPlan = { @@ -792,8 +806,9 @@ export class KeyedLayout implements RecordStore { let dropped = 0 const out = new RunWriter(this.store, plan.level) const inputs = runs(plan.layers).sort((a, b) => b.seq - a.seq) + const read = mergeRead(inputs.length) for await (const [k, v] of mergeSorted( - inputs.map((x) => runEntries(this.layers, x.files, from, to)), + inputs.map((x) => runEntries(this.layers, x.files, from, to, read)), )) { const pack = parseLoc(v).pack const z = k.lastIndexOf('\u0000') @@ -925,35 +940,36 @@ export class KeyedLayout implements RecordStore { const ids = packIds.filter((id) => !retired.has(packKey(id))) if (!ids.length) return {} const pw = new PackWriter(this.store, { tier: 0 }) - const moved: [string, string][] = [] + const lw = new RunWriter(this.store, 0) let kept = 0 let dropped = 0 - for (const id of ids) { - let batch: { key: string; line: string; place: string }[] = [] - const flush = async () => { - const found = await this.index.lookup(m, [...new Set(batch.map((b) => filterKey(b.key)))]) - for (const b of batch) { - const v = found.get(b.key) - if (v === undefined || v !== b.place) { - dropped++ - continue - } - moved.push([b.key, pw.append(b.key, b.line)]) - kept++ + // The packs' records in key order (each pack is sorted), so the new packs and + // the layer pointing at them are written as they go. A record in two of them + // comes twice; the index points at one copy, the one kept. + const read = mergeRead(ids.length) + let batch: { key: string; line: string; place: string }[] = [] + const flush = async () => { + const found = await this.index.values( + m, + batch.map((b) => b.key), + ) + for (const b of batch) { + if (found.get(b.key) !== b.place) { + dropped++ + continue } - batch = [] - await pw.ready() - } - for await (const x of this.packs.all(id)) { - batch.push(x) - if (batch.length === 1024) await flush() + lw.add(b.key, pw.append(b.key, b.line)) + kept++ } - await flush() + batch = [] + await Promise.all([pw.ready(), lw.ready()]) } + for await (const x of mergeAll(ids.map((id) => this.packs.all(id, read)))) { + batch.push(x) + if (batch.length === 1024) await flush() + } + await flush() const packs = await pw.finish() - moved.sort((a, b) => compareUtf8(a[0], b[0])) - const lw = new RunWriter(this.store, 0) - for (const [k, v] of moved) lw.add(k, v) const files = await lw.finish() await this.#swap({ oldPacks: ids.filter((id) => listed.has(id)), @@ -1116,6 +1132,25 @@ export class KeyedLayout implements RecordStore { } } +/** Pack records from several sorted streams, in key order, every one kept (mergeSorted keeps one a key). */ +async function* mergeAll( + streams: AsyncIterable[], +): AsyncGenerator { + const its = streams.map((s) => s[Symbol.asyncIterator]()) + const heads = await Promise.all(its.map((it) => it.next())) + for (;;) { + let best = -1 + for (let i = 0; i < heads.length; i++) { + const h = heads[i]! + if (h.done) continue + if (best < 0 || compareUtf8(h.value.key, (heads[best]!.value as T).key) < 0) best = i + } + if (best < 0) return + yield heads[best]!.value as T + heads[best] = await its[best]!.next() + } +} + /** F: the files a fork read from other collections at fork time, by id: where each lives. */ export interface Inherited { packs: Record @@ -1229,8 +1264,8 @@ class KeyedWriter implements RecordWriter { this.#packs = new PackWriter(layout.store) } - add(entry: RecordEntry, line: string): string { - const k = recordKey(entry.key, entry.hash) + add(type: string, entry: RecordEntry, line: string): string { + const k = recordKey(type, entry.key, entry.hash) const at = this.#packs.append(k, line) this.#entries.push([k, at]) return at diff --git a/packages/protocol/src/repo/layouts/kit.ts b/packages/protocol/src/repo/layouts/kit.ts index bb9ae83..f7c7e82 100644 --- a/packages/protocol/src/repo/layouts/kit.ts +++ b/packages/protocol/src/repo/layouts/kit.ts @@ -39,8 +39,21 @@ export const levelFor = (entries: number) => : Math.ceil(Math.log(entries / LEVEL_BASE) / Math.log(LAYER_FANOUT) - 1e-9) /** Raw bytes of one file of a merged run: a merge holds one file's pages, never the level's. */ export const RUN_FILE_RAW = 16 * 1024 * 1024 -/** Bytes a merge reads from each input at a time. */ +/** Bytes a merge reads from each input at a time, at most. */ const MERGE_READ = 1024 * 1024 +/** Bytes a merge reads at a time over all its inputs: each input's share, at least MERGE_READ_MIN. */ +export const MERGE_READ_BUDGET = 16 * 1024 * 1024 +const MERGE_READ_MIN = 64 * 1024 +/** Each input's read size when a merge takes `inputs` streams at once. */ +export const mergeRead = (inputs: number) => + Math.max( + MERGE_READ_MIN, + Math.min(MERGE_READ, Math.floor(MERGE_READ_BUDGET / Math.max(1, inputs))), + ) +/** Blocks of one pack a read takes in one request, at most (neighbours, gaps of up to two). */ +const READ_BLOCKS = 16 +/** Layers an index read works through at once, newest first. */ +const READ_LAYERS = 4 export const newId = () => crypto.randomUUID().replace(/-/g, '') @@ -199,22 +212,35 @@ export class PackReader { return v } - /** Lines at many places, read a block once each, `concurrency` blocks at a time. */ /** * Lines at many places. Blocks are read in runs: neighbouring blocks of one pack - * (gaps of up to two) come in one range request, as a sorted pack's leaf does. + * (gaps of up to two, READ_BLOCKS at most) come in one range request, as a sorted + * pack's leaf does. Each block's lines are taken as it's decoded and the block + * let go (the memory cache may keep it): a leaf whose records are spread over + * many packs never holds every block it touches. */ - async lines(places: readonly string[], concurrency = 16): Promise { + async lines(places: readonly string[], concurrency = 8): Promise { const parsed = places.map(parseLoc) - const got = new Map() + const out: (string | undefined)[] = Array.from({ length: parsed.length }) + // Where each block's lines go: (line in block, index in `out`). + const wants = new Map() const need = new Map>() - for (const p of parsed) { + parsed.forEach((p, i) => { const k = `${p.pack}.${p.block}` - if (got.has(k)) continue const hit = this.lru.get(`${this.scope}/block:${k}`) - if (hit) got.set(k, (hit as { v: string[] }).v) - else need.set(p.pack, (need.get(p.pack) ?? new Set()).add(p.block)) - } + if (hit) { + out[i] = (hit as { v: string[] }).v[p.line] + if (out[i] === undefined) + throw new Error(`No line ${p.line} in block ${p.block} of pack ${p.pack}`) + return + } + const w = wants.get(k) + if (w) w.push([p.line, i]) + else { + wants.set(k, [[p.line, i]]) + need.set(p.pack, (need.get(p.pack) ?? new Set()).add(p.block)) + } + }) const runs: { pack: string; from: number; to: number }[] = [] for (const [pack, set] of need) { const blocks = [...set].sort((x, y) => x - y) @@ -223,25 +249,32 @@ export class PackReader { while ( j + 1 < blocks.length && blocks[j + 1]! - blocks[j]! <= 3 && - blocks[j + 1]! - blocks[i]! < 256 + blocks[j + 1]! - blocks[i]! < READ_BLOCKS ) j++ runs.push({ pack, from: blocks[i]!, to: blocks[j]! }) i = j + 1 } } - for (let i = 0; i < runs.length; i += concurrency) - await Promise.all(runs.slice(i, i + concurrency).map((r) => this.#run(r, got))) - return parsed.map((p) => { - const line = got.get(`${p.pack}.${p.block}`)?.[p.line] - if (line === undefined) - throw new Error(`No line ${p.line} in block ${p.block} of pack ${p.pack}`) - return line - }) + const take = (pack: string, b: number, v: string[]) => { + for (const [line, i] of wants.get(`${pack}.${b}`) ?? []) { + out[i] = v[line] + if (out[i] === undefined) throw new Error(`No line ${line} in block ${b} of pack ${pack}`) + } + } + let next = 0 + const worker = async () => { + while (next < runs.length) await this.#run(runs[next++]!, take) + } + await Promise.all(Array.from({ length: Math.min(concurrency, runs.length) }, worker)) + return out as string[] } - /** Blocks `from` through `to` of a pack, in one range read. */ - async #run(r: { pack: string; from: number; to: number }, got: Map) { + /** Blocks `from` through `to` of a pack, in one range read, each handed to `take` as decoded. */ + async #run( + r: { pack: string; from: number; to: number }, + take: (pack: string, block: number, lines: string[]) => void, + ) { const dir = await this.#dir(r.pack) const start = dir[r.from]! const o = await this.store.get(packKey(r.pack), { @@ -255,17 +288,20 @@ export class PackReader { const v = text.split('\n') v.pop() for (let i = 0; i < v.length; i++) v[i] = v[i]!.slice(v[i]!.indexOf('\t') + 1) - got.set(`${r.pack}.${b}`, v) + take(r.pack, b, v) this.lru.set(`${this.scope}/block:${r.pack}.${b}`, { v, approxBytes: text.length * 2 }) } } - /** A whole pack's records in order, with their keys, read 8 MB at a time (merges and reclaims). */ - async *all(pack: string): AsyncGenerator<{ key: string; line: string; place: string }> { + /** A whole pack's records in order, with their keys, read `readBytes` at a time (merges and reclaims). */ + async *all( + pack: string, + readBytes = MERGE_READ, + ): AsyncGenerator<{ key: string; line: string; place: string }> { const dir = await this.#dir(pack) for (let b = 0; b < dir.length - 1;) { let e = b - while (e + 1 < dir.length - 1 && dir[e + 2]! - dir[b]! <= MERGE_READ) e++ + while (e + 1 < dir.length - 1 && dir[e + 2]! - dir[b]! <= readBytes) e++ const o = await this.store.get(packKey(pack), { offset: dir[b]!, length: dir[e + 1]! - dir[b]!, @@ -326,9 +362,9 @@ export const layerKey = (id: string) => `layers/${id}.ly` export const layerDirKey = (id: string) => `layers/${id}.dir` export const layerFilterKey = (id: string) => `layers/${id}.bf` -/** What a layer's filter holds for a key: its id part (before \u0000), or the whole key. */ +/** What a layer's filter holds for a key: all but its hash (after the last \u0000), or the whole key. */ export const filterKey = (k: string) => { - const z = k.indexOf('\u0000') + const z = k.lastIndexOf('\u0000') return z < 0 ? k : k.slice(0, z) } @@ -619,51 +655,44 @@ export class LayerReader { } /** - * Entries whose filter key is one of `ids` (records of those ids), reading only - * pages the layer's filter says may hold them. + * The entries of `keys` (sorted, exact) this layer holds. Only keys its filter + * may hold are looked for. A few keys read the one page each could be in; more + * (a leaf's) read the pages from the first such key's to the last's, one at a + * time, keeping only the keys wanted: a range also holds older versions of the + * leaf's records, and memory stays a page however many. */ - async lookup(layer: LayerInfo, ids: readonly string[]): Promise<[string, string][]> { - const inRange = ids.filter( - (id) => compareUtf8(layer.max, id) >= 0 && compareUtf8(layer.min, `${id}\u0000\uffff`) <= 0, + async pick(layer: LayerInfo, keys: readonly string[]): Promise<[string, string][]> { + const inRange = keys.filter( + (k) => compareUtf8(layer.min, k) <= 0 && compareUtf8(layer.max, k) >= 0, ) if (!inRange.length) return [] const filter = await this.#filter(layer.id) - const maybe = filter ? inRange.filter((id) => filterHas(filter, id)) : inRange + const maybe = filter ? inRange.filter((k) => filterHas(filter, filterKey(k))) : inRange if (!maybe.length) return [] const dir = await this.#dir(layer.id) - const pages = new Set() - for (const id of maybe) { - // Pages whose range can hold id's keys: from the last page starting at or before id. - const lo = pageAt(dir, id) - pages.add(lo) - for ( - let p = lo + 1; - p < dir.first.length && compareUtf8(dir.first[p]!, `${id}\u0000\uffff`) <= 0; - p++ - ) - pages.add(p) - } const want = new Set(maybe) const out: [string, string][] = [] - const read = await Promise.all([...pages].map((p) => this.#page(layer.id, dir, p))) - for (const page of read) for (const e of page) if (want.has(filterKey(e[0]))) out.push(e) - return out - } - - /** Entries with keys in [from, to], in key order. */ - async range(layer: LayerInfo, from: string, to: string): Promise<[string, string][]> { - if (compareUtf8(layer.max, from) < 0 || compareUtf8(layer.min, to) > 0) return [] - const dir = await this.#dir(layer.id) - // The last page whose first key is ≤ from, through the last whose first key is ≤ to. - const lo = pageAt(dir, from) - const hi = Math.max(lo, pageAt(dir, to)) - const out: [string, string][] = [] - const pages = await Promise.all( - Array.from({ length: hi - lo + 1 }, (_, j) => this.#page(layer.id, dir, lo + j)), - ) - for (const page of pages) - for (const e of page) - if (compareUtf8(e[0], from) >= 0 && compareUtf8(e[0], to) <= 0) out.push(e) + const keep = (page: [string, string][]) => { + for (const e of page) if (want.has(e[0])) out.push(e) + } + if (maybe.length <= 64) { + const pages = [...new Set(maybe.map((k) => pageAt(dir, k)))] + for (let i = 0; i < pages.length; i += 8) + for (const page of await Promise.all( + pages.slice(i, i + 8).map((p) => this.#page(layer.id, dir, p)), + )) + keep(page) + return out + } + const lo = pageAt(dir, maybe[0]!) + const hi = Math.max(lo, pageAt(dir, maybe.at(-1)!)) + // One page ahead: the next page's read overlaps this one's scan. + let ahead = this.#page(layer.id, dir, lo) + for (let p = lo; p <= hi; p++) { + const page = await ahead + if (p < hi) ahead = this.#page(layer.id, dir, p + 1) + keep(page) + } return out } @@ -674,10 +703,15 @@ export class LayerReader { } /** - * Every entry in order (keys in [from, to) when given), read 2 MB at a time, + * Every entry in order (keys in [from, to) when given), read `readBytes` at a time, * bypassing the cache (merges, reclaim's key ranges). */ - async *all(layer: LayerInfo, from?: string, to?: string): AsyncGenerator<[string, string]> { + async *all( + layer: LayerInfo, + from?: string, + to?: string, + readBytes = MERGE_READ, + ): AsyncGenerator<[string, string]> { if (from !== undefined && compareUtf8(layer.max, from) < 0) return if (to !== undefined && compareUtf8(layer.min, to) >= 0) return const dir = await this.#dir(layer.id) @@ -685,7 +719,7 @@ export class LayerReader { for (let p = from === undefined ? 0 : pageAt(dir, from); p < dir.first.length;) { if (to !== undefined && compareUtf8(dir.first[p]!, to) >= 0) return let e = p - while (e + 1 < dir.first.length && off[e + 2]! - off[p]! <= MERGE_READ) e++ + while (e + 1 < dir.first.length && off[e + 2]! - off[p]! <= readBytes) e++ const o = await this.store.get(layerKey(layer.id), { offset: off[p]!, length: off[e + 1]! - off[p]!, @@ -709,14 +743,18 @@ export class LayerReader { } } -/** A run's entries in order (keys in [from, to) when given): its files one after another. */ +/** + * A run's entries in order (keys in [from, to) when given): its files one after + * another, read `readBytes` at a time (`mergeRead` of the merge's input count). + */ export async function* runEntries( reader: LayerReader, files: readonly LayerInfo[], from?: string, to?: string, + readBytes = MERGE_READ, ): AsyncGenerator<[string, string]> { - for (const f of files) yield* reader.all(f, from, to) + for (const f of files) yield* reader.all(f, from, to, readBytes) } /** Merge sorted streams, newest first winning a repeated key. */ @@ -1160,21 +1198,23 @@ export class LayeredIndex { this.reader = new LayerReader(store, lru, scope) } - /** Values for the keys of `ids` (by filter key), newest layer winning, from manifest `m`. */ - async lookup(m: Manifest, ids: readonly string[]): Promise> { - const layers = m.layers.filter(this.filter).sort((a, b) => b.seq - a.seq) - const results = await Promise.all(layers.map((l) => this.reader.lookup(l, ids))) + /** + * Values of `keys` (exact), newest layer winning, from manifest `m`: READ_LAYERS + * layers at a time, newest first, until every key is found. Memory is the keys' + * values and a few layers' pages, however many layers there are (a large push + * leaves hundreds until merges catch up). + */ + async values(m: Manifest, keys: Iterable): Promise> { + const sorted = [...new Set(keys)].sort(compareUtf8) const out = new Map() - for (const r of results) for (const [k, v] of r) if (!out.has(k)) out.set(k, v) - return out - } - - /** Values for keys in [from, to], newest layer winning, from manifest `m`. */ - async range(m: Manifest, from: string, to: string): Promise> { const layers = m.layers.filter(this.filter).sort((a, b) => b.seq - a.seq) - const results = await Promise.all(layers.map((l) => this.reader.range(l, from, to))) - const out = new Map() - for (const r of results) for (const [k, v] of r) if (!out.has(k)) out.set(k, v) + for (let i = 0; i < layers.length && out.size < sorted.length; i += READ_LAYERS) { + const left = out.size ? sorted.filter((k) => !out.has(k)) : sorted + const got = await Promise.all( + layers.slice(i, i + READ_LAYERS).map((l) => this.reader.pick(l, left)), + ) + for (const r of got) for (const [k, v] of r) if (!out.has(k)) out.set(k, v) + } return out } } diff --git a/packages/protocol/src/repo/layouts/reclaim.ts b/packages/protocol/src/repo/layouts/reclaim.ts index 36bfa63..d5030f5 100644 --- a/packages/protocol/src/repo/layouts/reclaim.ts +++ b/packages/protocol/src/repo/layouts/reclaim.ts @@ -11,8 +11,9 @@ import { type RecordEntry, recordTree } from '../../tree/node.js' */ import { compareUtf8 } from '../../utf8.js' import type { Repo } from '../repo.js' +import { recordIdKey } from './types.js' -/** Live records of one key range, as sorted 52-bit hashes of (id, record hash). */ +/** Live records of one key range, as sorted 52-bit hashes of (type and id, record hash). */ export class LiveHashes { #h = new Float64Array(4096) size = 0 @@ -61,7 +62,7 @@ function hash52(id: string, hash: string): number { return (a >>> 0) * 1_048_576 + ((b >>> 0) >>> 12) } -/** A key range: ids in [lo, hi); null is open. */ +/** A key range: index keys' type and id (`recordIdKey`) in [lo, hi); null is open. */ export interface KeyRange { lo: string | null hi: string | null @@ -70,51 +71,70 @@ export interface KeyRange { export const inRange = (r: KeyRange, k: string) => (r.lo === null || compareUtf8(k, r.lo) >= 0) && (r.hi === null || compareUtf8(k, r.hi) < 0) +/** A record tree's root, with its type: index keys start with the type. */ +export interface TypedRoot { + type: string + root: string +} + /** * Mark the records of `range` that the given record tree roots hold. Each node is * read once however many versions share it, and only nodes whose keys overlap - * the range are read. + * the range are read (a type's tree wholly outside it, not at all). */ export async function markRange( repo: Repo, - treeRoots: Iterable, + treeRoots: Iterable, range: KeyRange, ): Promise<{ live: LiveHashes; nodesRead: number }> { const live = new LiveHashes() const seen = new Set() let nodesRead = 0 - const walk = async (hash: string): Promise => { + const walk = async (type: string, hash: string): Promise => { if (seen.has(hash)) return seen.add(hash) nodesRead++ const node = await repo.decoded(recordTree, hash) if (node.kind === 'leaf') { - for (const e of node.entries as RecordEntry[]) - if (inRange(range, e.key)) live.add(e.key, e.hash) + for (const e of node.entries as RecordEntry[]) { + const k = recordIdKey(type, e.key) + if (inRange(range, k)) live.add(k, e.hash) + } return } // A child holds keys after the one before it's last, through its own last. const due: string[] = [] for (let i = 0; i < node.children.length; i++) { const c = node.children[i]! - const prev = i > 0 ? node.children[i - 1]!.lastKey : null + const prev = i > 0 ? recordIdKey(type, node.children[i - 1]!.lastKey) : null if (range.hi !== null && prev !== null && compareUtf8(prev, range.hi) >= 0) break - if (range.lo !== null && compareUtf8(c.lastKey, range.lo) < 0) continue + if (range.lo !== null && compareUtf8(recordIdKey(type, c.lastKey), range.lo) < 0) continue due.push(c.hash) } - for (let i = 0; i < due.length; i += 16) await Promise.all(due.slice(i, i + 16).map(walk)) + for (let i = 0; i < due.length; i += 16) + await Promise.all(due.slice(i, i + 16).map((h) => walk(type, h))) + } + for (const { type, root } of treeRoots) { + // Every key of the type sorts after `${type}\u0000` and before `${type}\u0001`. + if (range.hi !== null && compareUtf8(`${type}\u0000`, range.hi) >= 0) continue + if (range.lo !== null && compareUtf8(`${type}\u0001`, range.lo) <= 0) continue + await walk(type, root) } - for (const root of treeRoots) await walk(root) return { live, nodesRead } } /** Every record tree root of the given versions (public and private sets, every type). */ -export async function recordRoots(repo: Repo, versionHashes: Iterable): Promise { - const out = new Set() +export async function recordRoots( + repo: Repo, + versionHashes: Iterable, +): Promise { + const out = new Map() for (const v of versionHashes) { const root = await repo.root(v) const sets = [root.public, ...(root.private ? [await repo.privateSet(root.private)] : [])] - for (const s of sets) for (const t of Object.values(s.types)) if (t.root) out.add(t.root) + for (const s of sets) + for (const [type, t] of Object.entries(s.types)) + if (t.root) out.set(t.root, { type, root: t.root }) } - return [...out] + return [...out.values()] } diff --git a/packages/protocol/src/repo/layouts/types.ts b/packages/protocol/src/repo/layouts/types.ts index 428e139..783457e 100644 --- a/packages/protocol/src/repo/layouts/types.ts +++ b/packages/protocol/src/repo/layouts/types.ts @@ -15,7 +15,7 @@ export type Placed = RecordEntry & { loc?: string } /** Records still in use, from a mark over every retained version (storage reclaim). */ export interface LiveSet { - /** `${id}\u0000${hash}` of every record in a retained version. */ + /** `recordKey(type, id, hash)` of every record in a retained version. */ records: Set } @@ -32,18 +32,21 @@ export interface RecordSets { readonly private: RecordStore } +/** A leaf of a type's record tree: its records are found by type, id and hash. */ +export interface TypedLeaf { + hash: string + type: string + entries: readonly RecordEntry[] +} + export interface RecordStore { readonly name: LayoutName /** The stored lines of a leaf's entries, in entry order (out-of-line pointers unresolved). */ - leafLines(leafHash: string, entries: readonly RecordEntry[]): Promise + leafLines(leaf: TypedLeaf): Promise /** One entry's stored line: a changed record's old body, which file references need. */ - entryLine(leafHash: string, entries: readonly RecordEntry[], i: number): Promise + entryLine(leaf: TypedLeaf, i: number): Promise /** The stored lines of some of a leaf's entries (by index), reading only those records. */ - linesAt( - leafHash: string, - entries: readonly RecordEntry[], - at: readonly number[], - ): Promise + linesAt(leaf: TypedLeaf, at: readonly number[]): Promise /** A writer for one commit unit or build; with `hold`, its runs are returned, not named. */ writer(opts?: { hold?: boolean }): RecordWriter /** @@ -60,8 +63,8 @@ export interface RecordStore { } export interface RecordWriter { - /** Store one record line (a spilled or new entry): its place. */ - add(entry: RecordEntry, line: string): string + /** Store one record line (a spilled or new entry) of type `type`: its place. */ + add(type: string, entry: RecordEntry, line: string): string /** Write a run out if enough is held (called as a build goes): memory stays bounded. */ drain?(): Promise /** @@ -74,4 +77,13 @@ export interface RecordWriter { /** A run a commit unit wrote and held back from the manifest (layout F). */ export type HeldRun = PendingRun -export const recordKey = (id: string, hash: string) => `${id}\u0000${hash}` +/** + * A record's index key: its type, id and hash. The type comes first so a type's + * records are together in the index, as its tree's leaves are: a leaf's records are + * one short range, never the other types' records whose ids fall between its ids. + */ +export const recordKey = (type: string, id: string, hash: string) => + `${type}\u0000${id}\u0000${hash}` + +/** The key a record's versions share (`recordKey` without the hash): what a filter holds. */ +export const recordIdKey = (type: string, id: string) => `${type}\u0000${id}` diff --git a/packages/protocol/src/repo/repo.ts b/packages/protocol/src/repo/repo.ts index 7ab6680..6140226 100644 --- a/packages/protocol/src/repo/repo.ts +++ b/packages/protocol/src/repo/repo.ts @@ -50,6 +50,7 @@ import { type Placed, type RecordSet, type RecordSets, + type TypedLeaf, recordStore, type RecordStore, type RecordWriter, @@ -368,7 +369,8 @@ export class Repo { * untrusted location, each line is checked against the leaf's record hashes. */ async bodyLines( - leaf: { hash: string; entries: readonly RecordEntry[] }, + /** Layout F: with its tree's type, which its records are found by. */ + leaf: Leaf, /** Layout F: the set the leaf is in (else public, then private). */ set?: RecordSet, ): Promise { @@ -395,14 +397,8 @@ export class Repo { } } - async #packedLines( - leaf: { hash: string; entries: readonly RecordEntry[] }, - key: string, - set: RecordSet | undefined, - ): Promise { - const lines = await this.#resolve( - await this.#inSet(set, (r) => r.leafLines(leaf.hash, leaf.entries)), - ) + async #packedLines(leaf: Leaf, key: string, set: RecordSet | undefined): Promise { + const lines = await this.#resolve(await this.#inSet(set, (r) => r.leafLines(typed(leaf)))) this.#checkPacked(leaf, lines) this.lru.set(this.#ck(key), { lines, approxBytes: lines.reduce((n, l) => n + l.length * 2, 0) }) return lines @@ -433,30 +429,20 @@ export class Repo { * Some of a leaf's records (by entry index), resolved. A packed layout reads only * those; leaf bodies are read whole (or from memory) and picked from. */ - async linesAt( - leaf: { hash: string; entries: readonly RecordEntry[] }, - at: readonly number[], - set?: RecordSet, - ): Promise { + async linesAt(leaf: Leaf, at: readonly number[], set?: RecordSet): Promise { if (!this.records) { const lines = await this.bodyLines(leaf) return at.map((i) => lines[i]!) } - const lines = await this.#resolve( - await this.#inSet(set, (r) => r.linesAt(leaf.hash, leaf.entries, at)), - ) + const lines = await this.#resolve(await this.#inSet(set, (r) => r.linesAt(typed(leaf), at))) this.#checkPacked(leaf, lines, at) return lines } /** One entry's record line (resolved): a changed record's old body, in a packed layout. */ - async entryLine( - leaf: { hash: string; entries: readonly RecordEntry[] }, - i: number, - set?: RecordSet, - ): Promise { + async entryLine(leaf: Leaf, i: number, set?: RecordSet): Promise { if (!this.records) return (await this.bodyLines(leaf))[i]! - const stored = await this.#inSet(set, (r) => r.entryLine(leaf.hash, leaf.entries, i)) + const stored = await this.#inSet(set, (r) => r.entryLine(typed(leaf), i)) const line = stored.startsWith(REF_PREFIX) ? await this.#outOfLine(stored) : stored this.#checkPacked(leaf, [line], [i]) return line @@ -503,10 +489,7 @@ export class Repo { * resolving each was a read for nothing, and it wrote the record inline. */ async *bodyLineStream( - leaf: { - hash: string - entries: readonly RecordEntry[] - }, + leaf: Leaf, onRead?: (r: BodyRead) => void, opts: { pointers?: boolean; set?: RecordSet } = {}, ): AsyncGenerator { @@ -686,6 +669,19 @@ export interface BodyRead { storedBytes: number | null } +/** A record tree's leaf: layout F also needs the tree's type, which its records are found by. */ +export interface Leaf { + hash: string + entries: readonly RecordEntry[] + type?: string | undefined +} + +function typed(leaf: Leaf): TypedLeaf { + if (leaf.type === undefined) + throw new Error(`Leaf ${leaf.hash}: layout F reads records by their tree's type; pass it`) + return leaf as TypedLeaf +} + /** NodeSource over a repository. Record trees get bodies through `leafEntries`. */ export class RepoSource implements NodeSource { constructor( @@ -697,6 +693,8 @@ export class RepoSource implements NodeSource { pointers?: boolean /** Layout F: the record set this tree is in, so a read looks in its index alone. */ set?: RecordSet + /** Layout F: the tree's type, which its records are found by (needed to read any). */ + type?: string } = {}, ) {} @@ -709,7 +707,7 @@ export class RepoSource implements NodeSource { if (node.kind !== 'leaf') throw new Error(`Node ${hash} is not a leaf`) if (this.spec !== (recordTree as TreeSpec)) return node.entries.slice() const entries = node.entries as RecordEntry[] - const lines = await this.repo.bodyLines({ hash, entries }, this.opts.set) + const lines = await this.repo.bodyLines({ hash, entries, type: this.opts.type }, this.opts.set) return entries.map((e, i) => ({ ...e, body: lines[i]! })) as E[] } @@ -735,7 +733,7 @@ export class RepoSource implements NodeSource { if (!this.repo.records || this.spec !== (recordTree as TreeSpec)) return undefined return async (leafHash, entries, at) => { const lines = await this.repo.linesAt( - { hash: leafHash, entries: entries as unknown as RecordEntry[] }, + { hash: leafHash, entries: entries as unknown as RecordEntry[], type: this.opts.type }, at, this.opts.set, ) @@ -752,7 +750,11 @@ export class RepoSource implements NodeSource { const i = entries.findIndex((x) => x.key === r.key) return { ...r, - body: await this.repo.entryLine({ hash: leafHash, entries }, i, this.opts.set), + body: await this.repo.entryLine( + { hash: leafHash, entries, type: this.opts.type }, + i, + this.opts.set, + ), } as E } @@ -766,10 +768,14 @@ export class RepoSource implements NodeSource { } const entries = node.entries as RecordEntry[] let i = 0 - for await (const body of this.repo.bodyLineStream({ hash, entries }, this.opts.onBodyRead, { - pointers: this.opts.pointers ?? false, - ...(this.opts.set ? { set: this.opts.set } : {}), - })) { + for await (const body of this.repo.bodyLineStream( + { hash, entries, type: this.opts.type }, + this.opts.onBodyRead, + { + pointers: this.opts.pointers ?? false, + ...(this.opts.set ? { set: this.opts.set } : {}), + }, + )) { yield { ...entries[i++]!, body } as E } } @@ -819,14 +825,24 @@ export class RepoSink implements TreeSink { compressingBytes?: number /** Layout F: the record set this sink's records go to (needed to write any). */ set?: RecordSet + /** Layout F: the type of the tree being built (needed to write any; see `type`). */ + type?: string /** Layout F: hold each run back from the manifest (`held`), for the commit to name. */ holdRecords?: boolean } = {}, - ) {} + ) { + this.type = opts.type + } /** Runs held back (`holdRecords`), in the order written. */ readonly held: HeldRun[] = [] + /** + * Layout F: the type of the tree being built, which its records are stored under. + * A sink that builds several types' trees, one after another, is told each. + */ + type: string | undefined + /** Layout F's writer for this sink's set, made on first use; null for leaf bodies. */ get #writer(): RecordWriter | null { if (!this.repo.records) return null @@ -881,7 +897,9 @@ export class RepoSink implements TreeSink { #place(e: E) { const p = e as unknown as Placed if (p.loc !== undefined || p.body === undefined) return - p.loc = this.#writer!.add(p, p.body) + if (this.type === undefined) + throw new Error("RepoSink: layout F stores records by type; set the tree's `type`") + p.loc = this.#writer!.add(this.type, p, p.body) } leaf(desc: NodeDesc, json: string, entries: readonly E[], spilled: number): void { diff --git a/packages/protocol/src/repo/sync.ts b/packages/protocol/src/repo/sync.ts index 4532ca2..2b845d7 100644 --- a/packages/protocol/src/repo/sync.ts +++ b/packages/protocol/src/repo/sync.ts @@ -231,9 +231,16 @@ export async function* treeObjects( yield { ...node, resumeKey: n.lastKey } continue } - const lines = await repo.records[tree.set].leafLines(n.hash, entries) - const body = await gzip(lines.join('\n') + '\n') - yield { key: keys.body(n.hash), bytes: body, resumeKey: n.lastKey } + // Built a member per 256 records, so the leaf is never held raw: as stored, a + // body is gzip members, and only its compressed size is held (as a stored one is). + const store = repo.records[tree.set] + const leafOf = { hash: n.hash, type: tree.slug!, entries } + const members: Uint8Array[] = [] + for (let i = 0; i < entries.length; i += 256) { + const at = Array.from({ length: Math.min(256, entries.length - i) }, (_, j) => i + j) + members.push(await gzip((await store.linesAt(leafOf, at)).join('\n') + '\n')) + } + yield { key: keys.body(n.hash), bytes: concatBytes(members), resumeKey: n.lastKey } continue } yield node @@ -246,6 +253,16 @@ export async function* treeObjects( } } +function concatBytes(parts: readonly Uint8Array[]): Uint8Array { + const out = new Uint8Array(parts.reduce((n, p) => n + p.byteLength, 0)) + let at = 0 + for (const p of parts) { + out.set(p, at) + at += p.byteLength + } + return out +} + /** The objects version `target` reaches that `opts.base` doesn't. */ export async function* packVersion( repo: Repo, diff --git a/packages/protocol/test/repo/layouts.test.ts b/packages/protocol/test/repo/layouts.test.ts index 65bc67f..1052308 100644 --- a/packages/protocol/test/repo/layouts.test.ts +++ b/packages/protocol/test/repo/layouts.test.ts @@ -27,9 +27,13 @@ import { openRepo, type RecordEntry, recordCanonical, + RECLAIM_MAX_RUNS, + recordIdKey, RepoSink, recordTree, sha256Hex, + type Store, + TooManyRunsError, treeObjects, utf8ByteLength, } from '../../src/index.js' @@ -40,6 +44,8 @@ import { levelFor, LayeredIndex, ManifestStore, + MERGE_READ_BUDGET, + mergeRead, pendingConfig, runs, RunWriter, @@ -55,7 +61,7 @@ afterEach(() => { }) /** A collection's layout F over `store`: its files under `runs//`. */ -const fLayout = (store = memoryStore(), home = 'c1', scope = 's') => +const fLayout = (store: Store = memoryStore(), home = 'c1', scope = 's') => new KeyedLayout(new HomeStore(store, home), repoLru(8 << 20), scope) const ids = Array.from( @@ -99,7 +105,7 @@ describe('reclaim a key range at a time', () => { const line = (i: number, v: number) => `line ${v}.${i}${big ? ` ${noise(i * 7 + v)}` : ''}` const write = async (from: number, to: number, v: number) => { const w = layout.writer() - for (let i = from; i < to; i++) w.add(entry(i, v), line(i, v)) + for (let i = from; i < to; i++) w.add('T', entry(i, v), line(i, v)) await w.finish() } await write(0, n, 1) @@ -113,9 +119,9 @@ describe('reclaim a key range at a time', () => { const live = new LiveHashes() const want = Array.from({ length: n }, (_, i) => entry(i, i < 2_000 ? 2 : 1)) - for (const e of want) live.add(e.key, e.hash) - expect(live.has(want[0]!.key, 'h1.0')).toBe(false) - expect(live.has(want[0]!.key, want[0]!.hash)).toBe(true) + for (const e of want) live.add(recordIdKey('T', e.key), e.hash) + expect(live.has(recordIdKey('T', want[0]!.key), 'h1.0')).toBe(false) + expect(live.has(recordIdKey('T', want[0]!.key), want[0]!.hash)).toBe(true) const plan = await layout.reclaimPlan(500) expect(plan.ranges.length).toBeGreaterThan(3) @@ -139,7 +145,7 @@ describe('reclaim a key range at a time', () => { const packs = await layout.allPacks() expect(packs.some((p) => swapped!.rewrite.includes(p.id))).toBe(false) expect(packs.reduce((s, p) => s + p.records, 0)).toBe(n) - const lines = await layout.leafLines('leaf', want) + const lines = await layout.leafLines({ hash: 'leaf', type: 'T', entries: want }) expect(lines).toEqual(want.map((_, i) => line(i, i < 2_000 ? 2 : 1))) }) }) @@ -151,14 +157,16 @@ describe('concurrent commits', () => { await Promise.all( Array.from({ length: 12 }, async (_, k) => { const w = layout.writer() - for (let i = k * 100; i < k * 100 + 100; i++) w.add(entry(i), `line ${i}`) + for (let i = k * 100; i < k * 100 + 100; i++) w.add('T', entry(i), `line ${i}`) await w.finish() }), ) expect((await layout.manifests.latest()).layers.length).toBe(12) await layout.maintain() const want = Array.from({ length: 1_200 }, (_, i) => entry(i)) - expect(await layout.leafLines('leaf', want)).toEqual(want.map((_, i) => `line ${i}`)) + expect(await layout.leafLines({ hash: 'leaf', type: 'T', entries: want })).toEqual( + want.map((_, i) => `line ${i}`), + ) expect((await layout.manifests.latest()).layers.length).toBeLessThan(12) }) }) @@ -169,7 +177,7 @@ describe('size-tiered index levels', () => { const entry = (i: number) => ({ key: ids[i]!, hash: `h.${i}`, size: 10 }) const write = async (from: number, n: number) => { const w = layout.writer() - for (let i = from; i < from + n; i++) w.add(entry(i), `line ${i}`) + for (let i = from; i < from + n; i++) w.add('T', entry(i), `line ${i}`) await w.finish() } await write(0, 8_000) @@ -182,7 +190,7 @@ describe('size-tiered index levels', () => { expect(m.layers.some((l) => l.id === big.id)).toBe(true) expect(m.layers.length).toBe(2) const want = [...Array.from({ length: 8_000 }, (_, i) => i), 10_000, 10_019].map(entry) - expect(await layout.leafLines('leaf', want)).toEqual( + expect(await layout.leafLines({ hash: 'leaf', type: 'T', entries: want })).toEqual( want.map((e) => `line ${ids.indexOf(e.key)}`), ) }) @@ -221,7 +229,7 @@ describe('merges keep the newest place of every record', () => { const v = now > 1 && rand(3) === 0 ? 1 + rand(now - 1) : (top.get(i) ?? 0) + 1 if (v > (top.get(i) ?? 0)) top.set(i, v) version.set(i, v) - w.add({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 }, `line ${v}.${i}`) + w.add('T', { key: ids[i]!, hash: `h${v}.${i}`, size: 10 }, `line ${v}.${i}`) } await w.finish() // Several maintain jobs at once, as many commits' jobs would be: one does the work. @@ -233,7 +241,9 @@ describe('merges keep the newest place of every record', () => { expect(new Set(m.layers.map((l) => l.run ?? l.id)).size).toBeLessThanOrEqual(13) const live = [...version].sort((a, b) => a[0] - b[0]) const want = live.map(([i, v]) => ({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 })) - expect(await layout.leafLines('leaf', want)).toEqual(live.map(([i, v]) => `line ${v}.${i}`)) + expect(await layout.leafLines({ hash: 'leaf', type: 'T', entries: want })).toEqual( + live.map(([i, v]) => `line ${v}.${i}`), + ) } finally { layoutConfig.graceMs = saved headConfig.intervalMs = 1000 @@ -279,7 +289,7 @@ describe('merges split into key-range jobs', () => { const now = version.get(i) ?? 0 const v = now > 1 && rand(3) === 0 ? 1 + rand(now - 1) : now + 1 version.set(i, v) - w.add({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 }, `line ${v}.${i}`) + w.add('T', { key: ids[i]!, hash: `h${v}.${i}`, size: 10 }, `line ${v}.${i}`) } await w.finish() await layout.maintain({ split }) @@ -296,7 +306,9 @@ describe('merges split into key-range jobs', () => { expect(most).toBeLessThanOrEqual(20) const live = [...version].sort((a, b) => a[0] - b[0]) const want = live.map(([i, v]) => ({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 })) - expect(await layout.leafLines('leaf', want)).toEqual(live.map(([i, v]) => `line ${v}.${i}`)) + expect(await layout.leafLines({ hash: 'leaf', type: 'T', entries: want })).toEqual( + live.map(([i, v]) => `line ${v}.${i}`), + ) } finally { Object.assign(layoutConfig, saved) } @@ -332,7 +344,7 @@ describe('a backlog of tiny commits', () => { for (const i of [...touched].sort((a, b) => a - b)) { const v = (version.get(i) ?? 0) + 1 version.set(i, v) - w.add({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 }, `line ${v}.${i}`) + w.add('T', { key: ids[i]!, hash: `h${v}.${i}`, size: 10 }, `line ${v}.${i}`) } await w.finish() } @@ -344,7 +356,9 @@ describe('a backlog of tiny commits', () => { } const live = [...version].sort((a, b) => a[0] - b[0]) const want = live.map(([i, v]) => ({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 })) - expect(await layout.leafLines('leaf', want)).toEqual(live.map(([i, v]) => `line ${v}.${i}`)) + expect(await layout.leafLines({ hash: 'leaf', type: 'T', entries: want })).toEqual( + live.map(([i, v]) => `line ${v}.${i}`), + ) }) }) @@ -363,7 +377,7 @@ describe('pack merges', () => { for (let c = 0; c < 10; c++) { const w = layout.writer() for (let i = 0; i < 6; i++, next++) - w.add({ key: ids[next % ids.length]!, hash: `h${next}`, size: 10 }, `line ${next}`) + w.add('T', { key: ids[next % ids.length]!, hash: `h${next}`, size: 10 }, `line ${next}`) await w.finish() } await layout.maintain() @@ -395,7 +409,7 @@ describe('layout F', () => { await Promise.all( Array.from({ length: 40 }, async (_, u) => { const w = layout.writer() - for (let i = u * 50; i < u * 50 + 50; i++) w.add(rec(i), line(i)) + for (let i = u * 50; i < u * 50 + 50; i++) w.add('T', rec(i), line(i)) await w.finish() }), ) @@ -403,18 +417,12 @@ describe('layout F', () => { // Readable before any merge folds them in: a reader that misses looks at pending runs. const all = Array.from({ length: 2000 }, (_, i) => i) expect( - await layout.leafLines( - 'leaf', - all.map((i) => rec(i)), - ), + await layout.leafLines({ hash: 'leaf', type: 'T', entries: all.map((i) => rec(i)) }), ).toEqual(all.map((i) => line(i))) await layout.maintain() expect((await root.list('runs/c1/manifest/f/pending/')).keys).toEqual([]) expect( - await layout.leafLines( - 'leaf', - all.map((i) => rec(i)), - ), + await layout.leafLines({ hash: 'leaf', type: 'T', entries: all.map((i) => rec(i)) }), ).toEqual(all.map((i) => line(i))) }) @@ -428,7 +436,7 @@ describe('layout F', () => { const parent = open('p') const commit = async (l: KeyedLayout, from: number, to: number, v: number) => { const w = l.writer() - for (let i = from; i < to; i++) w.add(rec(i, v), line(i, v)) + for (let i = from; i < to; i++) w.add('T', rec(i, v), line(i, v)) await w.finish() await l.maintain() } @@ -447,10 +455,7 @@ describe('layout F', () => { expect((await root.list('runs/fork/packs/')).keys).toEqual([]) const base = Array.from({ length: 1200 }, (_, i) => i) expect( - await fork.leafLines( - 'leaf', - base.map((i) => rec(i)), - ), + await fork.leafLines({ hash: 'leaf', type: 'T', entries: base.map((i) => rec(i)) }), ).toEqual(base.map((i) => line(i))) // Both go on: the parent merges its runs and packs away, sweeping at once. for (let c = 0; c < 10; c++) { @@ -458,10 +463,7 @@ describe('layout F', () => { await commit(fork, 600 + c * 30, 600 + c * 30 + 30, 3) } expect( - await fork.leafLines( - 'leaf', - base.map((i) => rec(i)), - ), + await fork.leafLines({ hash: 'leaf', type: 'T', entries: base.map((i) => rec(i)) }), ).toEqual(base.map((i) => line(i))) // The parent is deleted: what the fork reads stays, the rest goes. const gone = new Set() @@ -471,10 +473,7 @@ describe('layout F', () => { expect(dropped.kept).toBeGreaterThan(0) const forkView = open('fork') expect( - await forkView.leafLines( - 'leaf', - base.map((i) => rec(i)), - ), + await forkView.leafLines({ hash: 'leaf', type: 'T', entries: base.map((i) => rec(i)) }), ).toEqual(base.map((i) => line(i))) // Then the fork: nothing of either is left. gone.add('fork') @@ -515,7 +514,7 @@ describe('layout F', () => { // A load's units all land before its first maintain job. for (let u = 0; u < 300; u++) { const w = layout.writer() - for (let i = u * 5; i < u * 5 + 5; i++) w.add(rec(i), line(i)) + for (let i = u * 5; i < u * 5 + 5; i++) w.add('T', rec(i), line(i)) await w.finish() } // Each job queues the next while work is left, as the server's handler does. @@ -530,10 +529,7 @@ describe('layout F', () => { expect((await root.list('runs/c1/manifest/f/pending/')).keys).toEqual([]) const all = Array.from({ length: 1500 }, (_, i) => i) expect( - await layout.leafLines( - 'leaf', - all.map((i) => rec(i)), - ), + await layout.leafLines({ hash: 'leaf', type: 'T', entries: all.map((i) => rec(i)) }), ).toEqual(all.map((i) => line(i))) } finally { Object.assign(layoutConfig, saved) @@ -558,7 +554,7 @@ describe("a commit's held runs", () => { const layout = fLayout(root) const pending = async () => (await root.list('runs/c1/manifest/f/pending/')).keys const w = layout.writer({ hold: true }) - for (let i = 0; i < 100; i++) w.add(rec(i), line(i)) + for (let i = 0; i < 100; i++) w.add('T', rec(i), line(i)) const [run] = await w.finish() if (!run) throw new Error('no run') await layout.appendRuns([run]) @@ -568,7 +564,7 @@ describe("a commit's held runs", () => { // Folded in, merged away, and swept. for (let c = 1; c <= 4; c++) { const w = layout.writer() - for (let i = c * 100; i < c * 100 + 100; i++) w.add(rec(i), line(i)) + for (let i = c * 100; i < c * 100 + 100; i++) w.add('T', rec(i), line(i)) await w.finish() await layout.maintain() } @@ -585,10 +581,7 @@ describe("a commit's held runs", () => { await layout.maintain() const all = Array.from({ length: 500 }, (_, i) => i) expect( - await layout.leafLines( - 'leaf', - all.map((i) => rec(i)), - ), + await layout.leafLines({ hash: 'leaf', type: 'T', entries: all.map((i) => rec(i)) }), ).toEqual(all.map((i) => line(i))) } finally { layoutConfig.graceMs = saved @@ -609,7 +602,7 @@ describe("a commit's writer", () => { const rec = (i: number) => ({ key: keys[i]!, hash: `h${i}`, size: 10 }) const held = layout.writer({ hold: true }) for (let i = 0; i < 350; i++) { - held.add(rec(i), `line ${i}`) + held.add('T', rec(i), `line ${i}`) await held.drain?.() } const runs = await held.finish() @@ -617,11 +610,13 @@ describe("a commit's writer", () => { expect(runs.map((r) => r.layers.length > 0)).toEqual([true, true, true, true]) await layout.appendRuns(runs) const all = Array.from({ length: 350 }, (_, i) => i) - expect(await layout.leafLines('leaf', all.map(rec))).toEqual(all.map((i) => `line ${i}`)) + expect(await layout.leafLines({ hash: 'leaf', type: 'T', entries: all.map(rec) })).toEqual( + all.map((i) => `line ${i}`), + ) // Not held: each run is named as it's written. const w = fLayout(memoryStore()).writer() for (let i = 0; i < 350; i++) { - w.add(rec(i), `line ${i}`) + w.add('T', rec(i), `line ${i}`) await w.drain?.() } expect(await w.finish()).toEqual([]) @@ -640,7 +635,7 @@ describe('the maintain lease', () => { for (let c = 0; c < 4; c++) { const w = layout.writer() for (let i = c * 10; i < c * 10 + 10; i++) - w.add({ key: ids[i]!, hash: `h${i}`, size: 10 }, `line ${i}`) + w.add('T', { key: ids[i]!, hash: `h${i}`, size: 10 }, `line ${i}`) await w.finish() } const put = root.put.bind(root) @@ -672,10 +667,10 @@ describe('an untrusted repository in layout F', () => { const trusted = openRepo(store, { layout: 'f', recordScope: 'c1', trusted: true }) const w = trusted.records!.public.writer() // The second record's stored line isn't its entry's record. - entries.forEach((e, i) => w.add(e, i === 1 ? line(9) : line(i))) + entries.forEach((e, i) => w.add('T', e, i === 1 ? line(9) : line(i))) await w.finish() const repo = openRepo(store, { layout: 'f', recordScope: 'c1' }) - const leaf = { hash: 'leaf', entries } + const leaf = { hash: 'leaf', type: 'T', entries } expect(await repo.linesAt(leaf, [0, 2], 'public')).toEqual([line(0), line(2)]) await expect(repo.linesAt(leaf, [1], 'public')).rejects.toThrow(IntegrityError) await expect(repo.entryLine(leaf, 1, 'public')).rejects.toThrow(IntegrityError) @@ -740,16 +735,16 @@ describe('a sparse merge', () => { }) }) -describe('a layer range read', () => { - it('finds its pages by binary search, as a lookup does', async () => { +describe("a layer's read of a few keys", () => { + it('finds their pages by binary search', async () => { const store = memoryStore() const w = new RunWriter(store, 0) for (const id of ids) w.add(`${id}\u0000h`, 'v'.repeat(200)) const [layer] = await w.finish() const lru = repoLru(64 << 20) const reader = new LayerReader(store, lru, 's') - const id = ids.at(-10)! - const want = await reader.range(layer!, `${id}\u0000`, `${id}\u0000￿`) + const key = `${ids.at(-10)!}\u0000h` + const want = await reader.pick(layer!, [key]) expect(want).toHaveLength(1) // Count reads of the directory's first keys (it's cached now). const dir = (lru.get(`s/layerdir:${layer!.id}`) as { v: { first: string[] } }).v @@ -762,7 +757,7 @@ describe('a layer range read', () => { return Reflect.get(t, k, r) as unknown }, }) - expect(await reader.range(layer!, `${id}\u0000`, `${id}\u0000￿`)).toEqual(want) + expect(await reader.pick(layer!, [key])).toEqual(want) expect(reads).toBeLessThan(40) }) }) @@ -862,7 +857,7 @@ describe('a split merge whose inputs went', () => { for (let c = 0; c < 6; c++) { const w = layout.writer() for (let i = c * 10; i < c * 10 + 10; i++) - w.add({ key: ids[i]!, hash: `h${i}`, size: 10 }, `line ${i}`) + w.add('T', { key: ids[i]!, hash: `h${i}`, size: 10 }, `line ${i}`) await w.finish() } const plan = await layout.mergePlan((await layout.manifests.latest()).layers, 20) @@ -875,7 +870,9 @@ describe('a split merge whose inputs went', () => { expect((await store.list('runs/c1/merges/')).keys).toEqual([]) const m = await layout.manifests.latest() const want = Array.from({ length: 60 }, (_, i) => ({ key: ids[i]!, hash: `h${i}`, size: 10 })) - expect(await layout.leafLines('leaf', want)).toEqual(want.map((_, i) => `line ${i}`)) + expect(await layout.leafLines({ hash: 'leaf', type: 'T', entries: want })).toEqual( + want.map((_, i) => `line ${i}`), + ) // No layer file is left but the manifest's and those it retired (the merge's inputs). const id = (k: string) => k.slice('layers/'.length).split('.')[0]! const files = new Set( @@ -892,8 +889,8 @@ describe('a split merge whose inputs went', () => { }) }) -describe('layered index lookups', () => { - it('find what a range read finds, newest layer winning', async () => { +describe('layered index reads', () => { + it('find the same values a few keys or many at a time, newest layer winning', async () => { const store = memoryStore() const manifests = new ManifestStore(store, 'f') const layer = async (from: number, to: number, tag: string, level: number, seq: number) => { @@ -910,18 +907,112 @@ describe('layered index lookups', () => { await layer(5_000, 6_000, 'c', 0, 3) const index = new LayeredIndex(store, repoLru(8 << 20), 's', manifests) const m = await manifests.current() + const key = (i: number) => `${ids[i]}\u0000h${i % 7}` + const all = Array.from({ length: 200 }, (_, j) => 4_900 + j) + const many = await index.values(m, all.map(key)) + expect(many.size).toBe(200) for (const i of [0, 1, 4_999, 5_000, 5_500, 7_999, 8_000, 11_999, 12_000, 19_999]) { - const id = ids[i]! - const looked = await index.lookup(m, [id]) - const ranged = await index.range(m, `${id}\u0000`, `${id}\u0000￿`) - expect([...looked].sort()).toEqual([...ranged].sort()) - expect(looked.size).toBe(1) + const one = await index.values(m, [key(i)]) + expect(one.size).toBe(1) + if (many.has(key(i))) expect(one.get(key(i))).toBe(many.get(key(i))) } - const many = ids.slice(4_990, 5_010) - const looked = await index.lookup(m, many) - const ranged = await index.range(m, `${many[0]}\u0000`, `${many.at(-1)}\u0000￿`) - expect([...looked].sort()).toEqual([...ranged].sort()) // The newest layer's value wins where layers overlap. - expect([...(await index.lookup(m, [ids[5_500]!])).values()][0]).toMatch(/^c\./) + expect((await index.values(m, [key(5_500)])).get(key(5_500))).toMatch(/^c\./) + expect(many.get(key(5_000))).toMatch(/^c\./) + expect(many.get(key(4_999))).toMatch(/^a\./) + // A key no layer holds. + expect((await index.values(m, [`${ids[5]}\u0000nope`])).size).toBe(0) + }) +}) + +describe("layout F's reads and merges hold a bounded amount", () => { + /** `store`, counting reads of keys matching `re` and the most at once. */ + const watched = (store: Store, re: RegExp) => { + const n = { reads: 0, inFlight: 0, most: 0 } + const s = new Proxy(store, { + get(t, k) { + const v = Reflect.get(t, k, t) as unknown + if (k !== 'get' || typeof v !== 'function') return typeof v === 'function' ? v.bind(t) : v + return async (key: string, ...rest: unknown[]) => { + if (!re.test(key)) return (v as (...a: unknown[]) => unknown).call(t, key, ...rest) + n.reads++ + n.most = Math.max(n.most, ++n.inFlight) + try { + return await (v as (...a: unknown[]) => Promise).call(t, key, ...rest) + } finally { + n.inFlight-- + } + } + }, + }) + return { store: s, n } + } + + it("reads a leaf of a small type without the large type's records between its ids", async () => { + // As Crossref's: every type's ids are DOIs, so a small type's leaf spans + // much of the id space a large type fills. + const { store, n } = watched(memoryStore(), /\.ly$/) + const layout = fLayout(store) + const w = layout.writer() + for (let i = 0; i < 20_000; i++) + w.add('journal-article', { key: ids[i]!, hash: `h${i}`, size: 10 }, 'x'.repeat(100)) + const small = ids.filter((_, i) => i % 200 === 0) + for (const [i, id] of small.entries()) + w.add('dataset', { key: id, hash: `d${i}`, size: 10 }, `dataset ${i}`) + await w.finish() + const entries = small.map((id, i) => ({ key: id, hash: `d${i}`, size: 10 })) + const leaf = { hash: 'leaf', type: 'dataset', entries } + expect(await layout.leafLines(leaf)).toEqual(small.map((_, i) => `dataset ${i}`)) + // The large type's 20,000 entries fill over a hundred pages; the leaf's are one or two. + expect(n.reads).toBeLessThanOrEqual(3) + }) + + it('reads an index of many runs a few layers at a time, newest first', async () => { + const { store, n } = watched(memoryStore(), /\.dir$/) + const layout = fLayout(store) + // Forty commits' runs, as a large push leaves before merges catch up. + for (let r = 0; r < 40; r++) { + const w = layout.writer() + for (let i = r * 100; i < r * 100 + 100; i++) + w.add('T', { key: ids[i]!, hash: `h${i}`, size: 10 }, `line ${i}`) + await w.finish() + } + expect(runs((await layout.manifests.latest()).layers).length).toBe(40) + // The oldest run's records: every layer is looked in. + const entries = Array.from({ length: 100 }, (_, i) => ({ + key: ids[i]!, + hash: `h${i}`, + size: 10, + })) + n.most = 0 + expect(await layout.leafLines({ hash: 'leaf', type: 'T', entries })).toEqual( + entries.map((_, i) => `line ${i}`), + ) + expect(n.most).toBeLessThanOrEqual(4) + }) + + it('plans a reclaim only once merges bring the runs down', async () => { + const layout = fLayout() + for (let r = 0; r <= RECLAIM_MAX_RUNS; r++) { + const w = layout.writer() + w.add('T', { key: ids[r]!, hash: `h${r}`, size: 10 }, `line ${r}`) + await w.finish() + } + await expect(layout.reclaimPlan()).rejects.toThrow(TooManyRunsError) + while (await layout.maintain()); + await layout.maintain() + expect(runs((await layout.manifests.latest()).layers).length).toBeLessThanOrEqual( + RECLAIM_MAX_RUNS, + ) + await expect(layout.reclaimPlan()).resolves.toBeTruthy() + }) + + it("shares a merge's read budget among its inputs", () => { + expect(mergeRead(1)).toBe(1024 * 1024) + expect(mergeRead(16)).toBe(1024 * 1024) + expect(mergeRead(128)).toBe(128 * 1024) + // Never below a few pages, however many inputs. + expect(mergeRead(1_000)).toBe(64 * 1024) + expect(mergeRead(128) * 128).toBeLessThanOrEqual(MERGE_READ_BUDGET) }) }) diff --git a/packages/server/scripts/memory.ts b/packages/server/scripts/memory.ts index 9c5d22a..464ac53 100644 --- a/packages/server/scripts/memory.ts +++ b/packages/server/scripts/memory.ts @@ -47,6 +47,10 @@ * --read-ms N then read the last version whole through records.ndjson and * records.ndjson.gz, as a client taking a chunk every N ms (find-doi * over a range, 2026-10-18), sampling at every chunk (default 0: no read) + * --small-every K about one record in K is a `dataset`, ids spread over the same DOI + * space as the others: a small type whose leaves each span much of + * it, as Crossref's small types' do (layout F reads such a leaf by type) + * --max-mb N exit 1 if any job's peak is over N MB: a memory gate for a round * * Unit peaks are sampled at every spill, leaf end and body write, on the timer, and * at storage operations; a sample forces a GC only when the reading without one @@ -126,6 +130,8 @@ const CACHE = str('cache', 'none') const LAYOUT = str('layout', 'today') as 'today' | 'f' const TIMER_MS = opt('timer-ms', 0) const READ_MS = opt('read-ms', 0) +const SMALL_EVERY = opt('small-every', 0) +const MAX_MB = opt('max-mb', 0) const PIECES = args.includes('--pieces') const NO_GC = args.includes('--no-gc') const UNITS = args.includes('--units') || RECORD_KB > 0 || LEAF_MB.length > 0 @@ -633,7 +639,7 @@ function work(doi: string, version = 0) { return { id: doi, // The type follows the DOI, so an update lands on the same record. - type: doi.charCodeAt(doi.length - 1) % 5 ? 'journal-article' : 'book-chapter', + type: typeOf(doi), data: { DOI: doi, title: [words(10)], @@ -646,6 +652,12 @@ function work(doi: string, version = 0) { } } +/** A record's type, from its DOI: a fifth book chapters, and with --small-every a few datasets. */ +function typeOf(doi: string) { + if (SMALL_EVERY > 0 && (hash32(`small\u0000${doi}`) >>> 0) % SMALL_EVERY === 0) return 'dataset' + return doi.charCodeAt(doi.length - 1) % 5 ? 'journal-article' : 'book-chapter' +} + // --- Records of a set size (--record-kb) -------------------------------------------------- /** mulberry32 from a given seed, for records generated from their id. */ @@ -706,7 +718,7 @@ function sizedWork(doi: string, version: number) { version, reference: [] as object[], } - const rec = { id: doi, type: 'journal-article', data } + const rec = { id: doi, type: SMALL_EVERY ? typeOf(doi) : 'journal-article', data } const target = RECORD_KB * 1024 let size = Buffer.byteLength(JSON.stringify(rec)) for (let i = 0; size < target; i++) { @@ -900,7 +912,11 @@ async function prepareCache() { console.error(` cache warmed: ${n} objects, ${(bytes / MB).toFixed(0)} MB`) } -const schemas = { 'journal-article': { type: 'object' }, 'book-chapter': { type: 'object' } } +const schemas = { + 'journal-article': { type: 'object' }, + 'book-chapter': { type: 'object' }, + ...(SMALL_EVERY > 0 ? { dataset: { type: 'object' } } : {}), +} const dois = LEAF_MB.length > 0 ? leafIds() @@ -1025,3 +1041,9 @@ report( ), ) await rm(dir, { recursive: true, force: true }) +if (MAX_MB > 0) { + const over = rows.filter(([, v]) => v.peak / MB > MAX_MB) + for (const [k, v] of over) + console.error(`over ${MAX_MB} MB: ${k} peaked at ${(v.peak / MB).toFixed(1)} MB`) + if (over.length) process.exitCode = 1 +} diff --git a/packages/server/src/api/versions.ts b/packages/server/src/api/versions.ts index 5543862..c5ba143 100644 --- a/packages/server/src/api/versions.ts +++ b/packages/server/src/api/versions.ts @@ -370,9 +370,9 @@ export function versionRoutes() { const want = c.req.query('type') const types = view.types.filter((t) => t.public?.root && (!want || t.slug === want)) if (want && types.length === 0) return jsonError(c, 404, `No public records of type ${want}`) - const source = new RepoSource(recordTree, view.repo, { set: 'public' }) const members = async function* () { for (const t of types) { + const source = new RepoSource(recordTree, view.repo, { set: 'public', type: t.slug }) for await (const leaf of leaves(source, t.public!.root)) { const node = await source.node(leaf.hash) const entries = (node.kind === 'leaf' ? node.entries : []) as RecordEntry[] diff --git a/packages/server/src/cleanup/layout-reclaim.ts b/packages/server/src/cleanup/layout-reclaim.ts index 630bc8d..4430b1c 100644 --- a/packages/server/src/cleanup/layout-reclaim.ts +++ b/packages/server/src/cleanup/layout-reclaim.ts @@ -13,7 +13,14 @@ * the collection while a reclaim runs, so it needs the write fence first * (cleanup/fence.ts). The scale bench ran it on F through `startLayoutReclaim`. */ -import { KeyedLayout, markRange, type RecordSet, type Repo, recordRoots } from '@underlay/protocol' +import { + KeyedLayout, + markRange, + type RecordSet, + type Repo, + recordRoots, + TooManyRunsError, +} from '@underlay/protocol' import { eq } from 'drizzle-orm' import * as schema from '../db/schema.js' @@ -23,6 +30,9 @@ import type { JobMessage, Ports } from '../ports.js' /** Mostly-dead packs a rewrite job copies. */ const REWRITE_PACKS = 4 +/** How long a plan waits for merges to bring the index's runs down (TooManyRunsError). */ +const PLAN_RETRY_SECONDS = 300 + interface Target { collectionId: string set: RecordSet @@ -59,7 +69,19 @@ registerJob('layout.reclaim', async (job, ports) => { const { repo, layout } = await open(ports, t) switch (job.step) { case 'plan': { - const plan = await layout.reclaimPlan() + let plan + try { + plan = await layout.reclaimPlan() + } catch (err) { + if (!(err instanceof TooManyRunsError)) throw err + // Each range job reads every run at once: merge them down first, then plan. + await ports.jobs.enqueue({ type: 'layout.maintain', ...t }) + await ports.jobs.enqueue( + { type: 'layout.reclaim', step: 'plan', ...t }, + { delaySeconds: PLAN_RETRY_SECONDS }, + ) + return + } await ports.jobs.enqueueBatch( plan.ranges.map((_, i) => ({ type: 'layout.reclaim', diff --git a/packages/server/src/push/parallel.ts b/packages/server/src/push/parallel.ts index 6a9be61..71317af 100644 --- a/packages/server/src/push/parallel.ts +++ b/packages/server/src/push/parallel.ts @@ -909,6 +909,7 @@ async function runPendingUnit(ports: Ports, unit: UnitRow, session: SessionRow): bodyOf: bodyOfRecord, ...unitSink, set, + type: unit.type, holdRecords: true, }) const range = { after: unit.after, through: unit.through } @@ -956,7 +957,7 @@ async function runPendingUnit(ports: Ports, unit: UnitRow, session: SessionRow): // The most members compressing at a leaf's end: what a leaf's write holds raw. let mostCompressing = 0 const merged = await mergeTree( - new RepoSource(recordTree, repo, { onBodyRead, pointers: true, set }), + new RepoSource(recordTree, repo, { onBodyRead, pointers: true, set, type: unit.type }), sink, into, changes, @@ -1138,11 +1139,10 @@ export async function assembleParallel(ports: Ports, sessionId: string): Promise const plan = await getJson(ports, `${planDir(sessionId, planId)}/plan.json`) const repo = await ports.stores.forCollection(session.collectionId) // Base leaves keep large records as pointers, as in units (runPendingUnit). A source - // and a sink a set: layout F reads and writes each set's records apart. - const sources = { - public: new RepoSource(recordTree, repo, { pointers: true, set: 'public' }), - private: new RepoSource(recordTree, repo, { pointers: true, set: 'private' }), - } + // a tree and a sink a set (told each tree's type): layout F reads and writes each + // set's records apart, and finds them by type. + const sourceOf = (set: SetName, type: string) => + new RepoSource(recordTree, repo, { pointers: true, set, type }) const sinks = { public: new RepoSink(repo, { bodyOf: bodyOfRecord, set: 'public' }), private: new RepoSink(repo, { bodyOf: bodyOfRecord, set: 'private' }), @@ -1201,7 +1201,13 @@ export async function assembleParallel(ports: Ports, sessionId: string): Promise }, })) const treeStarted = Date.now() - const result = await assembleTree(sources[set], sinks[set], built[type]![set].root, segments) + sinks[set].type = type + const result = await assembleTree( + sourceOf(set, type), + sinks[set], + built[type]![set].root, + segments, + ) console.log( `[assemble] ${isolateTag()} ${sessionId} ${type}/${set}: ${segments.length} segments ` + `in ${Math.round((Date.now() - treeStarted) / 1000)}s, ${JSON.stringify(result.stats)}`, @@ -1452,7 +1458,7 @@ export async function runValidation( const inputs = await loadInputs(ports, sessionId) const validate = compileSchema(inputs.schemas[t.type]!) const repo = await ports.stores.forCollection(session.collectionId) - const source = new RepoSource(recordTree, repo, { set: t.set }) + const source = new RepoSource(recordTree, repo, { set: t.set, type: t.type }) const result: ValidationResult = { total: 0, errors: [] } for await (const e of entriesInRange(source, t.root, t.after, t.through)) { const errs = validate((JSON.parse(e.body!) as { data: unknown }).data) diff --git a/packages/server/src/versions/view.ts b/packages/server/src/versions/view.ts index ade8d6e..30bd21e 100644 --- a/packages/server/src/versions/view.ts +++ b/packages/server/src/versions/view.ts @@ -156,7 +156,7 @@ export async function* typeRecords( } // Each set's records from its own source: layout F keeps them apart. const it = (root: string | null, set: 'public' | 'private') => - iterate(new RepoSource(recordTree, view.repo, { set }), root, { + iterate(new RepoSource(recordTree, view.repo, { set, type: type.slug }), root, { payloads: opts.bodies ?? false, stream: true, ...(after !== undefined ? { after } : {}), @@ -226,7 +226,7 @@ export async function getRecord( ['private', view.owner ? type.private : null], ] as const) { if (!tree?.root) continue - const source = new RepoSource(recordTree, view.repo, { set }) + const source = new RepoSource(recordTree, view.repo, { set, type: type.slug }) const hit = await getEntry(source, tree.root, id) if (!hit) continue if (view.withheld.has(hit.hash)) return null @@ -246,7 +246,11 @@ async function bodyOf(source: RepoSource, root: string, key: string const i = node.entries.findIndex((e) => e.key === key) if (i < 0) throw new Error(`Record ${key} is not in leaf ${hash}`) return ( - await source.repo.linesAt({ hash, entries: node.entries }, [i], source.opts.set) + await source.repo.linesAt( + { hash, entries: node.entries, type: source.opts.type }, + [i], + source.opts.set, + ) )[0]! } // Streamed, stopping at the record: a leaf of large records is never held whole. @@ -300,7 +304,7 @@ export async function recordBodies( const at = entries.flatMap((e, i) => keys.has(`${type}\u0000${e.key}`) && !view.withheld.has(e.hash) ? [i] : [], ) - const lines = await source.repo.linesAt({ hash, entries }, at, leafSet.get(hash)) + const lines = await source.repo.linesAt({ hash, entries, type }, at, leafSet.get(hash)) at.forEach((i, j) => out.set(`${type}\u0000${entries[i]!.key}`, lines[j]!)) continue } diff --git a/packages/server/test/layouts.test.ts b/packages/server/test/layouts.test.ts index ca2ea26..3630132 100644 --- a/packages/server/test/layouts.test.ts +++ b/packages/server/test/layouts.test.ts @@ -95,7 +95,7 @@ async function records(h: Harness, collectionId: string) { if (root.private) sets.push(['private', (await repo.privateSet(root.private)).types.Author?.root ?? null]) for (const [set, tree] of sets) - for await (const e of iterate(new RepoSource(recordTree, repo, { set }), tree, { + for await (const e of iterate(new RepoSource(recordTree, repo, { set, type: 'Author' }), tree, { payloads: true, })) out.push(`${set} ${e.key} ${e.body}`) diff --git a/packages/server/test/mirrors.test.ts b/packages/server/test/mirrors.test.ts index 3ba40cb..db1ef52 100644 --- a/packages/server/test/mirrors.test.ts +++ b/packages/server/test/mirrors.test.ts @@ -198,7 +198,7 @@ describe('bucket mirrors of layout F (records in packs of the collection)', () = ['public', root.public.types.Book!], ['private', priv.types.Book!], ] as const) { - const src = new RepoSource(recordTree, repo, { set }) + const src = new RepoSource(recordTree, repo, { set, type: 'Book' }) expect((await verifyTree(src, t.root)).errors).toEqual([]) let n = 0 for await (const e of iterate(src, t.root, { payloads: true })) if (e.body) n++ @@ -249,7 +249,7 @@ describe('bucket mirrors of layout F (records in packs of the collection)', () = nodes: 'delta', }) const t = (await repo.root(v2.hash)).public.types.Book! - const src = new RepoSource(recordTree, repo, { set: 'public' }) + const src = new RepoSource(recordTree, repo, { set: 'public', type: 'Book' }) expect((await verifyTree(src, t.root)).errors).toEqual([]) let n = 0 for await (const e of iterate(src, t.root, { payloads: true })) if (e.body) n++ @@ -305,7 +305,7 @@ describe('bucket mirrors of layout F (records in packs of the collection)', () = expect(await readHead(repo, col!.id)).toMatchObject({ versionHash: head!.hash }) const t = (await repo.root(head!.hash)).public.types.Book! let n = 0 - const src = new RepoSource(recordTree, repo, { set: 'public' }) + const src = new RepoSource(recordTree, repo, { set: 'public', type: 'Book' }) for await (const e of iterate(src, t.root, { payloads: true })) if (e.body) n++ expect(n).toBe(t.count) } @@ -353,7 +353,7 @@ describe('bucket mirrors', () => { ['public', root.public.types.Book!], ['private', priv.types.Book!], ] as const) { - const src = new RepoSource(recordTree, repo, { set }) + const src = new RepoSource(recordTree, repo, { set, type: 'Book' }) expect((await verifyTree(src, t.root)).errors).toEqual([]) let n = 0 for await (const e of iterate(src, t.root, { payloads: true })) if (e.body) n++ diff --git a/packages/server/test/parallel.test.ts b/packages/server/test/parallel.test.ts index 7d86220..f4dd7da 100644 --- a/packages/server/test/parallel.test.ts +++ b/packages/server/test/parallel.test.ts @@ -206,7 +206,7 @@ describe('parallel commit', () => { expect(t.public.count).toBe(6012) let n = 0 for await (const e of iterate( - new RepoSource(recordTree, t.repo, { set: 'public' }), + new RepoSource(recordTree, t.repo, { set: 'public', type: 'Author' }), t.public.root, { payloads: true, diff --git a/packages/server/test/schema-parallel.test.ts b/packages/server/test/schema-parallel.test.ts index 17a6caa..8c8a8d8 100644 --- a/packages/server/test/schema-parallel.test.ts +++ b/packages/server/test/schema-parallel.test.ts @@ -120,10 +120,16 @@ async function planKeys(h: Harness, sid: string) { } /** Records of a tree read with their bodies from one set alone (layout F keeps them apart). */ -async function readsFrom(h: Harness, collectionId: string, set: RecordSet, root: string | null) { +async function readsFrom( + h: Harness, + collectionId: string, + set: RecordSet, + type: string, + root: string | null, +) { const repo = await h.ports.stores.forCollection(collectionId) let n = 0 - for await (const e of iterate(new RepoSource(recordTree, repo, { set }), root, { + for await (const e of iterate(new RepoSource(recordTree, repo, { set, type }), root, { payloads: true, })) if (e.body) n++ @@ -247,7 +253,7 @@ describe('schema and privacy changes in a parallel commit', () => { expect(ha.public.types.Doc).toBeUndefined() expect(ha.private?.types.Doc?.count).toBe(300) // Every record of the moved type reads from its new set alone. - expect(await readsFrom(h, a.id, 'private', ha.private!.types.Doc!.root)).toBe(300) + expect(await readsFrom(h, a.id, 'private', 'Doc', ha.private!.types.Doc!.root)).toBe(300) }, 30_000) it('makes a private type public by moving its tree, as the serial commit does', async () => { @@ -267,7 +273,7 @@ describe('schema and privacy changes in a parallel commit', () => { expect(pb.status.status).toBe('committed') expect(pa.status.result.changes).toEqual(pb.status.result.changes) const ha = await head(h, a.id) - expect(await readsFrom(h, a.id, 'public', ha.public.types.Doc!.root)).toBe( + expect(await readsFrom(h, a.id, 'public', 'Doc', ha.public.types.Doc!.root)).toBe( ha.public.types.Doc!.count, ) expect(ha).toEqual(await head(h, b.id)) diff --git a/packages/server/test/sync.test.ts b/packages/server/test/sync.test.ts index fcc0791..d769bf8 100644 --- a/packages/server/test/sync.test.ts +++ b/packages/server/test/sync.test.ts @@ -45,7 +45,9 @@ async function records(repo: Repo, version: string) { const root = await repo.root(version) const out: string[] = [] for (const [slug, t] of Object.entries(root.public.types)) { - for await (const e of iterate(new RepoSource(recordTree, repo), t.root, { payloads: true })) { + for await (const e of iterate(new RepoSource(recordTree, repo, { type: slug }), t.root, { + payloads: true, + })) { out.push(`${slug}/${e.key}:${e.body}`) } }