diff --git a/packages/protocol/src/repo/layouts/keyed.ts b/packages/protocol/src/repo/layouts/keyed.ts index 9e5305e..ff47313 100644 --- a/packages/protocol/src/repo/layouts/keyed.ts +++ b/packages/protocol/src/repo/layouts/keyed.ts @@ -194,7 +194,18 @@ export class KeyedLayout implements RecordStore { /** Every pack: the manifest's and the registry's (reclaim, forks). */ async allPacks(): Promise { const m = await this.manifests.latest() - return [...m.packs, ...(await this.registry.read()).packs] + return [...m.packs, ...(await this.#registered(m)).packs] + } + + /** + * The registry's objects, and its packs `m` doesn't list: a reclaim lists the packs + * it will rewrite in the manifest, then takes them out of the registry, so for a + * moment a pack can be in both. + */ + async #registered(m: Manifest): Promise<{ keys: string[]; packs: PackInfo[] }> { + const r = await this.registry.read() + const listed = new Set(m.packs.map((p) => p.id)) + return { keys: r.keys, packs: r.packs.filter((p) => !listed.has(p.id)) } } async #find(m: Manifest, entries: readonly RecordEntry[]): Promise<(string | undefined)[]> { @@ -321,14 +332,28 @@ export class KeyedLayout implements RecordStore { held ? (JSON.parse(await held.text()) as { token: string; at: number }) : null, ) .catch(() => null) - if (v?.token === token) return () => this.store.delete(key) + if (v?.token === token) return () => this.#release(key, token) // A lease older than a job can run is a job that died holding it. if (!v || Date.now() - v.at < MAINTAIN_LEASE_MS) return null - await this.store.delete(key) + await this.#release(key, v.token) } return null } + /** + * Delete the lease if `token` still holds it: a job that ran past its lease, which + * another then took, mustn't release the new holder's. The store has no conditional + * delete, so a takeover between this read and the delete still loses it; the window + * is one request, not the rest of a job. + */ + async #release(key: string, token: string): Promise { + const held = await this.store + .get(key) + .then(async (o) => (o ? (JSON.parse(await o.text()) as { token: string }).token : null)) + .catch(() => null) + if (held === token) await this.store.delete(key) + } + /** * Merge the newest runs, while each older one is no larger than all the newer * ones together, when there are LAYER_FANOUT of them or more. A merged run @@ -478,6 +503,8 @@ export class KeyedLayout implements RecordStore { unlisted?: string[] newLayers?: LayerInfo[] newPacks?: PackInfo[] + /** Registry packs to list in the manifest too (reclaim's, until their rewrite swaps them out). */ + relist?: PackInfo[] /** New layers keep their sequence (a merged run); else they're the newest (re-pointed records). */ keepSeq?: boolean }): Promise { @@ -504,7 +531,8 @@ export class KeyedLayout implements RecordStore { // A pack merge's layer is the newest: its places win over the old ones. const seq = topSeq(m) + 1 for (const l of newLayers) m.layers.push(o.keepSeq ? l : { ...l, seq }) - m.packs.push(...listed) + const have = new Set(m.packs.map((p) => p.id)) + m.packs.push(...listed, ...(o.relist ?? []).filter((p) => !have.has(p.id))) for (const id of oldLayers) m.retired.push( { key: layerKey(id), at: now }, @@ -726,7 +754,7 @@ export class KeyedLayout implements RecordStore { async reclaimPlan(rangeEntries = RECLAIM_RANGE_ENTRIES): Promise { const m = await this.manifests.latest() const ranges = await this.#cutRanges(m.layers, rangeEntries) - const registry = await this.registry.read() + const registry = await this.#registered(m) const plan: ReclaimPlan = { id: newId(), layers: m.layers, @@ -840,26 +868,32 @@ export class KeyedLayout implements RecordStore { // One run across the ranges, at the level its whole size puts it. const level = Math.max(plan.level, levelFor(stats.entriesKept)) for (const f of files) f.level = level + const registered: PackInfo[] = [] + for (const key of plan.registry) { + const o = await this.store.get(key) + if (o) registered.push(...(JSON.parse(await o.text()) as PackInfo[])) + } const m = await this.manifests.latest() const listed = new Set(m.packs.map((p) => p.id)) + const rewriting = new Set(rewrite) const swapped = await this.#swap({ oldLayers: plan.layers.map((l) => l.id), oldPacks: drop.filter((id) => listed.has(id)), unlisted: drop.filter((id) => !listed.has(id)), newLayers: files, + // Registry packs being rewritten move to the manifest: listed somewhere until + // their rewrite swaps them out (else a reclaim planned meanwhile drops their + // live records, and a fork or a mirror misses them). + relist: registered.filter((p) => rewriting.has(p.id)), keepSeq: true, }) if (swapped) { - // The registry's objects as one, without the dropped packs or the ones being rewritten - // (their rewrite lists what it writes). + // The registry's objects as one, without the dropped packs or the ones now listed. const gone = new Set([...drop, ...rewrite]) - const kept: PackInfo[] = [] - for (const key of plan.registry) { - const o = await this.store.get(key) - if (o) - kept.push(...(JSON.parse(await o.text()) as PackInfo[]).filter((p) => !gone.has(p.id))) - } - await this.registry.replace(plan.registry, kept) + await this.registry.replace( + plan.registry, + registered.filter((p) => !gone.has(p.id)), + ) } await this.#dropPlan(plan) await this.manifests.settle() @@ -883,7 +917,7 @@ export class KeyedLayout implements RecordStore { */ async reclaimRewrite(packIds: readonly string[]): Promise> { const m = await this.manifests.latest() - // Packs the manifest lists must still be there; the others reclaim took out of the registry. + // Packs the manifest lists must still be there; the others (an older reclaim's) are in neither. const listed = new Set(m.packs.map((p) => p.id)) const retired = new Set(m.retired.map((r) => r.key)) const ids = packIds.filter((id) => !retired.has(packKey(id))) @@ -944,7 +978,7 @@ export class KeyedLayout implements RecordStore { if (!(from instanceof HomeStore) || !(to instanceof HomeStore)) throw new Error('forkInto: F needs both collections in their own prefixes') const m = await this.manifests.latest() - const registered = (await this.registry.read()).packs + const registered = (await this.#registered(m)).packs // Each file's home: where the parent read it from (its own parent's), else the parent. const mine = await from.inherited() const inherited: Inherited = { packs: {}, layers: {} } @@ -988,7 +1022,7 @@ export class KeyedLayout implements RecordStore { const home = this.store if (!(home instanceof HomeStore)) throw new Error('mirrorFiles: layout F only') const m = await this.manifests.latest() - const registry = await this.registry.read() + const registry = await this.#registered(m) const inh = await home.inherited() const at = (h: string | undefined, key: string) => `runs/${h ?? home.home}/${key}` const files: { key: string; bytes?: number; optional?: boolean }[] = [] @@ -1018,6 +1052,26 @@ export class KeyedLayout implements RecordStore { } } + /** + * F: of the files a mirror copied that `mirrorFiles` no longer lists, which it may + * delete (`drop`) and which to forget without deleting (`forget`). Another + * collection's (a fork's parent's, at its own keys) are forgotten: that collection + * may be mirrored at the same location. This set's own stay while a live fork reads + * them, as here: that fork's mirror there may list them. The rest are dropped. + */ + async mirrorDroppable(keys: readonly string[]): Promise<{ drop: string[]; forget: string[] }> { + const home = this.store + if (!(home instanceof HomeStore)) throw new Error('mirrorDroppable: layout F only') + const own = `runs/${home.home}/` + const held = keys.some((k) => k.startsWith(own)) ? await home.forkHeld() : new Set() + const drop: string[] = [] + const forget: string[] = [] + for (const k of keys) + if (!k.startsWith(own)) forget.push(k) + else if (!held.has(fileId(k.slice(own.length)) ?? '')) drop.push(k) + return { drop, forget } + } + /** * F: delete what a deleted collection wrote, but for files a live fork still * reads (they go when that fork does); then the same for each collection it diff --git a/packages/protocol/src/repo/layouts/kit.ts b/packages/protocol/src/repo/layouts/kit.ts index e5119c7..bb9ae83 100644 --- a/packages/protocol/src/repo/layouts/kit.ts +++ b/packages/protocol/src/repo/layouts/kit.ts @@ -408,6 +408,18 @@ interface LayerDir { off: number[] } +/** The last page whose first key is ≤ `key` (0 if none is), by binary search. */ +function pageAt(dir: LayerDir, key: string): number { + let lo = 0 + let hi = dir.first.length - 1 + while (lo < hi) { + const mid = (lo + hi + 1) >> 1 + if (compareUtf8(dir.first[mid]!, key) <= 0) lo = mid + else hi = mid - 1 + } + return lo +} + export class LayerWriter { readonly id = newId() #pages: Promise[] = [] @@ -622,13 +634,7 @@ export class LayerReader { const pages = new Set() for (const id of maybe) { // Pages whose range can hold id's keys: from the last page starting at or before id. - let lo = 0 - let hi = dir.first.length - 1 - while (lo < hi) { - const mid = (lo + hi + 1) >> 1 - if (compareUtf8(dir.first[mid]!, id) <= 0) lo = mid - else hi = mid - 1 - } + const lo = pageAt(dir, id) pages.add(lo) for ( let p = lo + 1; @@ -649,10 +655,8 @@ export class LayerReader { if (compareUtf8(layer.max, from) < 0 || compareUtf8(layer.min, to) > 0) return [] const dir = await this.#dir(layer.id) // The last page whose first key is ≤ from, through the last whose first key is ≤ to. - let lo = 0 - while (lo + 1 < dir.first.length && compareUtf8(dir.first[lo + 1]!, from) <= 0) lo++ - let hi = lo - while (hi + 1 < dir.first.length && compareUtf8(dir.first[hi + 1]!, to) <= 0) hi++ + const lo = pageAt(dir, from) + const hi = Math.max(lo, pageAt(dir, to)) const out: [string, string][] = [] const pages = await Promise.all( Array.from({ length: hi - lo + 1 }, (_, j) => this.#page(layer.id, dir, lo + j)), @@ -678,10 +682,7 @@ export class LayerReader { if (to !== undefined && compareUtf8(layer.min, to) >= 0) return const dir = await this.#dir(layer.id) const off = dir.off - let start = 0 - if (from !== undefined) - while (start + 1 < dir.first.length && compareUtf8(dir.first[start + 1]!, from) <= 0) start++ - for (let p = start; p < dir.first.length;) { + for (let p = from === undefined ? 0 : pageAt(dir, from); p < dir.first.length;) { if (to !== undefined && compareUtf8(dir.first[p]!, to) >= 0) return let e = p while (e + 1 < dir.first.length && off[e + 2]! - off[p]! <= MERGE_READ) e++ @@ -1079,6 +1080,17 @@ export class ManifestStore { */ async appendRuns(runs: PendingRun[]): Promise { if (!runs.length) return + // A retry (assembly appended them, then stopped before saving its tree) names them + // once only: a second object, folded in after the first's layers merged away, would + // bring them back as the newest run, of files swept since. A tree's runs go in one + // object, all or none, so they were named if the index lists any of their layers + // (pending ones included) or retired one, or its files are gone (swept: a unit's + // layer files are deleted no other way). + const m = await this.latest() + const known = new Set([...m.layers.map((l) => layerKey(l.id)), ...m.retired.map((r) => r.key)]) + if (runs.some((r) => r.layers.some((l) => known.has(layerKey(l.id))))) return + const first = runs.find((r) => r.layers.length)?.layers[0] + if (first && !(await this.store.head(layerDirKey(first.id)))) return await this.#putPending({ runs }) } diff --git a/packages/protocol/src/repo/repo.ts b/packages/protocol/src/repo/repo.ts index 6175f36..c702865 100644 --- a/packages/protocol/src/repo/repo.ts +++ b/packages/protocol/src/repo/repo.ts @@ -159,6 +159,8 @@ export interface RepoOptions { /** A stored node that is a delta: `{"b":,"p":[tuples put],"d":[keys removed]}`. */ const DELTA_PREFIX = '{"b":"' +/** Whether a stored node's JSON is a delta (node deltas), not the node itself. */ +export const isNodeDelta = (stored: string) => stored.startsWith(DELTA_PREFIX) /** A delta larger than this share of the full node is stored full instead (a new keyframe). */ const DELTA_MAX_SHARE = 0.25 @@ -401,20 +403,32 @@ export class Repo { const lines = await this.#resolve( await this.#inSet(set, (r) => r.leafLines(leaf.hash, leaf.entries)), ) - // A packed layout's records come from its packs, not a body the leaf hashes: on an - // untrusted location (a mirror read by a third party) each is checked as a body's is. - if (!this.trusted) - lines.forEach((l, i) => { - const e = leaf.entries[i]! - if (sha256Hex(l) !== e.hash) - throw new IntegrityError(`Leaf ${leaf.hash}: record ${i} fails its hash`) - if (utf8ByteLength(l) !== e.size || !l.startsWith(`{"id":${JSON.stringify(e.key)},"type":`)) - throw new IntegrityError(`Leaf ${leaf.hash}: record ${i} isn't its entry's record`) - }) + this.#checkPacked(leaf, lines) this.lru.set(this.#ck(key), { lines, approxBytes: lines.reduce((n, l) => n + l.length * 2, 0) }) return lines } + /** + * A packed layout's records come from its packs, not a body the leaf hashes: on an + * untrusted location (a mirror read by a third party) each line read, resolved, is + * checked as a body's is, against its entry (`at[j]`, else the j-th). + */ + #checkPacked( + leaf: { hash: string; entries: readonly RecordEntry[] }, + lines: readonly string[], + at?: readonly number[], + ): void { + if (this.trusted) return + lines.forEach((l, j) => { + const i = at ? at[j]! : j + const e = leaf.entries[i]! + if (sha256Hex(l) !== e.hash) + throw new IntegrityError(`Leaf ${leaf.hash}: record ${i} fails its hash`) + if (utf8ByteLength(l) !== e.size || !l.startsWith(`{"id":${JSON.stringify(e.key)},"type":`)) + throw new IntegrityError(`Leaf ${leaf.hash}: record ${i} isn't its entry's record`) + }) + } + /** * Some of a leaf's records (by entry index), resolved. A packed layout reads only * those; leaf bodies are read whole (or from memory) and picked from. @@ -428,7 +442,11 @@ export class Repo { const lines = await this.bodyLines(leaf) return at.map((i) => lines[i]!) } - return this.#resolve(await this.#inSet(set, (r) => r.linesAt(leaf.hash, leaf.entries, at))) + const lines = await this.#resolve( + await this.#inSet(set, (r) => r.linesAt(leaf.hash, leaf.entries, at)), + ) + this.#checkPacked(leaf, lines, at) + return lines } /** One entry's record line (resolved): a changed record's old body, in a packed layout. */ @@ -438,8 +456,10 @@ export class Repo { set?: RecordSet, ): Promise { if (!this.records) return (await this.bodyLines(leaf))[i]! - const line = await this.#inSet(set, (r) => r.entryLine(leaf.hash, leaf.entries, i)) - return line.startsWith(REF_PREFIX) ? this.#outOfLine(line) : line + const stored = await this.#inSet(set, (r) => r.entryLine(leaf.hash, leaf.entries, i)) + const line = stored.startsWith(REF_PREFIX) ? await this.#outOfLine(stored) : stored + this.#checkPacked(leaf, [line], [i]) + return line } async #bodyLines( @@ -863,7 +883,10 @@ export class RepoSink implements TreeSink { } leaf(desc: NodeDesc, json: string, entries: readonly E[], spilled: number): void { + // The base is the leaf being rewritten, for this leaf alone: a later one (after + // reused nodes, or appended) would only be diffed against an unrelated node. const base = this.#base + this.#base = undefined this.#run(keys.node(desc.hash), () => this.repo.putNode(desc.hash, json, base)) if (!this.opts.bodyOf) return if (this.repo.records) { diff --git a/packages/protocol/src/repo/sync.ts b/packages/protocol/src/repo/sync.ts index 181c97c..4532ca2 100644 --- a/packages/protocol/src/repo/sync.ts +++ b/packages/protocol/src/repo/sync.ts @@ -40,7 +40,14 @@ import { versionHash, } from '../format.js' import { gunzipText, gzip, splitLines } from './gzip.js' -import { IntegrityError, keys, OUT_OF_LINE_BYTES, outOfLineHash, type Repo } from './repo.js' +import { + IntegrityError, + isNodeDelta, + keys, + OUT_OF_LINE_BYTES, + outOfLineHash, + type Repo, +} from './repo.js' /** Which sets a sync moves: `public`, or `all` (public and private). */ export type SyncSets = 'public' | 'all' @@ -193,15 +200,14 @@ export async function* treeObjects( for await (const n of newNodes(source as NodeSource, tree.base, tree.target, { after, })) { - // A receiver checks each node against its hash: one stored as a delta (node - // deltas) goes in full. A mirror copies it as stored, with the node it's against. - const node = { - key: keys.node(n.hash), - bytes: - opts.bodies && repo.nodeDeltas - ? await gzip(await repo.nodeJson(n.hash)) - : await raw(repo, keys.node(n.hash)), - } + // A node stored as a delta (node deltas) goes in full: a receiver checks each node + // against its hash, and a mirror may not hold the node a delta is against (another + // version's, a fork's parent's, or a private leaf a public mirror never gets). The + // cost: a mirror stores rewritten leaves in full, a delta being under a quarter of one. + let bytes = await raw(repo, keys.node(n.hash)) + if (n.level === 0 && isNodeDelta(await gunzipText(bytes))) + bytes = await gzip(await repo.nodeJson(n.hash)) + const node = { key: keys.node(n.hash), bytes } if (n.level !== 0) { yield node continue diff --git a/packages/protocol/src/tree/merge.ts b/packages/protocol/src/tree/merge.ts index a5ef161..e0f4c73 100644 --- a/packages/protocol/src/tree/merge.ts +++ b/packages/protocol/src/tree/merge.ts @@ -102,8 +102,9 @@ export function emptyStats(): MergeStats { export class Peekable { readonly #it: Iterator | AsyncIterator #head: IteratorResult | undefined - /** Items taken and put back, served before the iterator. */ + /** Items taken and put back, served before the iterator, from `#at` on (no shift: O(1) a take). */ #back: T[] = [] + #at = 0 constructor(src: Iterable | AsyncIterable) { this.#it = @@ -113,28 +114,43 @@ export class Peekable { } async peek(): Promise { - if (this.#back.length) return this.#back[0] + if (this.#at < this.#back.length) return this.#back[this.#at] this.#head ??= await this.#it.next() return this.#head.done ? undefined : this.#head.value } async next(): Promise { - if (this.#back.length) return this.#back.shift() + if (this.#at < this.#back.length) { + const v = this.#back[this.#at++] + if (this.#at === this.#back.length) { + this.#back = [] + this.#at = 0 + } + return v + } const v = await this.peek() this.#head = undefined return v } - /** The items while `ok` holds, left in place: a look further ahead than peek. */ - async lookAhead(ok: (v: T) => boolean): Promise { + /** The items while `ok` holds, `max` at most, left in place: a look further ahead than peek. */ + async lookAhead(ok: (v: T) => boolean, max = Infinity): Promise { const out: T[] = [] - for (let v = await this.peek(); v !== undefined && ok(v); v = await this.peek()) + for ( + let v = await this.peek(); + v !== undefined && ok(v) && out.length < max; + v = await this.peek() + ) out.push((await this.next())!) - this.#back = [...out, ...this.#back] + this.#back = [...out, ...this.#back.slice(this.#at)] + this.#at = 0 return out } } +/** Changes a sparse leaf's merge looks ahead for the old records they replace (one read). */ +const OLD_BATCH = 1024 + export async function mergeTree( source: NodeSource, sink: TreeSink, @@ -193,15 +209,15 @@ export async function mergeTree( needOld ? source.withPayload!(desc.hash, old) : old /** - * A sparse leaf's old records that the changes landing in it replace or remove, - * read in one batch (file references need them), by key. + * A sparse leaf's old records that `first` and the changes after it landing in the + * leaf replace or remove, read in one batch (file references need them), by key. It + * looks OLD_BATCH changes ahead at most: a leaf's range can take any number of + * inserts, and looking to its last key held them all. */ - const oldBodies = async (desc: NodeDesc, olds: E[]): Promise> => { - const keys = new Set( - (await pending.lookAhead((c) => compareUtf8(c.key, desc.lastKey) <= 0)).map((c) => c.key), - ) + const oldBodies = async (desc: NodeDesc, olds: E[], first: string): Promise> => { + const ahead = await pending.lookAhead((c) => compareUtf8(c.key, desc.lastKey) <= 0, OLD_BATCH) + const keys = new Set([first, ...ahead.map((c) => c.key)]) const at = olds.flatMap((o, i) => (keys.has(spec.key(o)) ? [i] : [])) - if (!at.length) return new Map() const got = await source.payloadsAt!(desc.hash, olds, at) return new Map(got.map((e) => [spec.key(e), e])) } @@ -212,14 +228,21 @@ export async function mergeTree( let entries: AsyncIterable | E[] = sparse ? sparse(desc.hash) : leafEntryStream(source, desc.hash) - let olds: Map | null = null + let list: E[] | null = null + const olds = new Map() if (needOld && source.payloadsAt) { - const list: E[] = [] + list = [] for await (const e of entries) list.push(e) - olds = await oldBodies(desc, list) entries = list } - const withOldOf = async (old: E) => olds?.get(spec.key(old)) ?? withOld(desc, old) + const withOldOf = async (old: E) => { + if (!list) return withOld(desc, old) + const k = spec.key(old) + if (!olds.has(k)) for (const [key, e] of await oldBodies(desc, list, k)) olds.set(key, e) + const e = olds.get(k)! + olds.delete(k) + return e + } for await (const old of entries) { const oldKey = spec.key(old) // A leaf straddling a unit's range edge: keep only the range's entries. diff --git a/packages/protocol/test/repo/layouts.test.ts b/packages/protocol/test/repo/layouts.test.ts index 0ec1bec..c7cae33 100644 --- a/packages/protocol/test/repo/layouts.test.ts +++ b/packages/protocol/test/repo/layouts.test.ts @@ -6,17 +6,37 @@ import { afterEach, beforeEach, describe, expect, it } from 'vitest' import { + buildTree, + decodeNode, + describeNode, + encodeLeaf, + gunzipText, + hashNode, headConfig, HomeStore, + IntegrityError, KeyedLayout, layoutConfig, LiveHashes, + MapSource, + MemorySink, memoryStore, type MergePlan, + mergeTree, + type NodeSource, + openRepo, + type RecordEntry, + recordCanonical, + RepoSink, + recordTree, + sha256Hex, + treeObjects, + utf8ByteLength, } from '../../src/index.js' import { buildFilter, filterHas, + LayerReader, levelFor, LayeredIndex, ManifestStore, @@ -109,6 +129,11 @@ describe('reclaim a key range at a time', () => { expect(swapped!.rewrite.length).toBeGreaterThan(0) // A second caller doesn't swap again. expect(await layout.reclaimSwap(plan)).toBeNull() + // Until its rewrite swaps it out, a pack being rewritten is still listed (once): a + // reclaim planned meanwhile, a fork or a mirror would otherwise lose its records. + const between = (await layout.allPacks()).map((p) => p.id) + expect(swapped!.rewrite.filter((id) => !between.includes(id))).toEqual([]) + expect(new Set(between).size).toBe(between.length) for (const id of swapped!.rewrite) await layout.reclaimRewrite([id]) const packs = await layout.allPacks() @@ -518,6 +543,240 @@ describe('layout F', () => { }) }) +describe("a commit's held runs", () => { + const rec = (i: number, v = 1) => ({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 }) + const line = (i: number, v = 1) => `line ${v}.${i}` + + it('are named once, however often a retry appends them', async () => { + // Assembly appends a tree's runs, then saves the tree: a retry between the two + // appends them again, perhaps after the first copy was folded in and merged away + // (its files swept). Folded in again, they'd be the newest run, of files gone. + const saved = layoutConfig.graceMs + layoutConfig.graceMs = 0 + try { + const root = memoryStore() + const layout = fLayout(root) + const pending = async () => (await root.list('runs/c1/manifest/f/pending/')).keys + const w = layout.writer() + for (let i = 0; i < 100; i++) w.add(rec(i), line(i)) + const run = (await w.finish({ hold: true }))! + await layout.appendRuns([run]) + // Before the first copy is folded in. + await layout.appendRuns([run]) + expect(await pending()).toHaveLength(1) + // Folded in, merged away, and swept. + for (let c = 1; c <= 4; c++) { + const w = layout.writer() + for (let i = c * 100; i < c * 100 + 100; i++) w.add(rec(i), line(i)) + await w.finish() + await layout.maintain() + } + await new Promise((r) => setTimeout(r, 5)) + await layout.maintain() + const id = run.layers[0]!.id + expect((await layout.manifests.latest()).layers.some((l) => l.id === id)).toBe(false) + expect((await root.list(`runs/c1/layers/${id}`)).keys).toEqual([]) + // The retry writes nothing (a commit's own pending object may still be listed, + // folded in but not yet deleted, depending on what the last maintain wrote). + const before = await pending() + await layout.appendRuns([run]) + expect(await pending()).toEqual(before) + await layout.maintain() + const all = Array.from({ length: 500 }, (_, i) => i) + expect( + await layout.leafLines( + 'leaf', + all.map((i) => rec(i)), + ), + ).toEqual(all.map((i) => line(i))) + } finally { + layoutConfig.graceMs = saved + } + }) +}) + +describe('the maintain lease', () => { + it('is left alone when another job took it over', async () => { + // A job that ran past its lease, which another job then took: releasing it at + // the end must not release the new holder's. + const root = memoryStore() + const layout = fLayout(root) + for (let c = 0; c < 4; c++) { + const w = layout.writer() + for (let i = c * 10; i < c * 10 + 10; i++) + w.add({ key: ids[i]!, hash: `h${i}`, size: 10 }, `line ${i}`) + await w.finish() + } + const put = root.put.bind(root) + let taken = false + root.put = async (key, body, opts) => { + // Taken over while this job merges. + if (!taken && key.startsWith('runs/c1/manifest/f/0')) { + taken = true + await put('runs/c1/maintain/lease', JSON.stringify({ token: 'other', at: Date.now() })) + } + return put(key, body, opts) + } + await layout.maintain() + expect(taken).toBe(true) + const lease = await root.get('runs/c1/maintain/lease') + expect((JSON.parse(await lease!.text()) as { token: string }).token).toBe('other') + }) +}) + +describe('an untrusted repository in layout F', () => { + it('checks every record it reads against its entry, however it reads it', async () => { + const store = memoryStore() + const line = (i: number) => recordCanonical(`r${i}`, 'T', { n: i }) + const entries = [0, 1, 2].map((i) => ({ + key: `r${i}`, + hash: sha256Hex(line(i)), + size: utf8ByteLength(line(i)), + })) + const trusted = openRepo(store, { layout: 'f', recordScope: 'c1', trusted: true }) + const w = trusted.records!.public.writer() + // The second record's stored line isn't its entry's record. + entries.forEach((e, i) => w.add(e, i === 1 ? line(9) : line(i))) + await w.finish() + const repo = openRepo(store, { layout: 'f', recordScope: 'c1' }) + const leaf = { hash: 'leaf', entries } + expect(await repo.linesAt(leaf, [0, 2], 'public')).toEqual([line(0), line(2)]) + await expect(repo.linesAt(leaf, [1], 'public')).rejects.toThrow(IntegrityError) + await expect(repo.entryLine(leaf, 1, 'public')).rejects.toThrow(IntegrityError) + await expect(repo.bodyLines(leaf, 'public')).rejects.toThrow(IntegrityError) + const stream = async () => { + for await (const _ of repo.bodyLineStream(leaf, undefined, { set: 'public' })); + } + await expect(stream()).rejects.toThrow(IntegrityError) + }) +}) + +describe('a sparse merge', () => { + it("reads ahead of a leaf's changes a batch at a time, not to its last key", async () => { + // A leaf's range can take any number of inserts: looking ahead to its last key for + // the old records the changes replace held every one of them at once. + const e = (k: string, v = 0): RecordEntry => ({ + key: k, + hash: sha256Hex(`${k}#${v}`), + size: 10, + }) + const sink = new MemorySink() + const base = new MapSource(recordTree, sink.nodes, sink.leaves) + const root = buildTree(recordTree, sink, [e('a'), e('z')])! + expect(root.level).toBe(0) + const reads: number[] = [] + const source: NodeSource = { + spec: recordTree, + node: (h) => base.node(h), + leafEntries: (h) => base.leafEntries(h), + sparseStream: async function* (h) { + yield* await base.leafEntries(h) + }, + withPayload: async (_h, x) => ({ ...x, body: 'old' }), + payloadsAt: async (_h, entries, at) => { + reads.push(...at) + return at.map((i) => ({ ...entries[i]!, body: 'old' })) + }, + } + let pulled = 0 + let applied = 0 + let most = 0 + const changes = (async function* () { + yield { key: 'a', entry: e('a', 1) } + for (let i = 0; i < 20_000; i++) { + most = Math.max(most, ++pulled - applied) + const k = `m${String(i).padStart(6, '0')}` + yield { key: k, entry: e(k) } + } + yield { key: 'z', entry: e('z', 1) } + })() + const olds: (string | undefined)[] = [] + await mergeTree(source, new MemorySink(), root.hash, changes, { + onChange: (before) => { + applied++ + if (before) olds.push(before.body) + }, + }) + expect(applied).toBe(20_002) + expect(olds).toEqual(['old', 'old']) + expect(reads.sort()).toEqual([0, 1]) + expect(most).toBeLessThan(2_000) + }) +}) + +describe('a layer range read', () => { + it('finds its pages by binary search, as a lookup does', async () => { + const store = memoryStore() + const w = new RunWriter(store, 0) + for (const id of ids) w.add(`${id}\u0000h`, 'v'.repeat(200)) + const [layer] = await w.finish() + const lru = repoLru(64 << 20) + const reader = new LayerReader(store, lru, 's') + const id = ids.at(-10)! + const want = await reader.range(layer!, `${id}\u0000`, `${id}\u0000￿`) + expect(want).toHaveLength(1) + // Count reads of the directory's first keys (it's cached now). + const dir = (lru.get(`s/layerdir:${layer!.id}`) as { v: { first: string[] } }).v + const first = dir.first + expect(first.length).toBeGreaterThan(200) + let reads = 0 + dir.first = new Proxy(first, { + get(t, k, r) { + if (typeof k === 'string' && /^\d+$/.test(k)) reads++ + return Reflect.get(t, k, r) as unknown + }, + }) + expect(await reader.range(layer!, `${id}\u0000`, `${id}\u0000￿`)).toEqual(want) + expect(reads).toBeLessThan(40) + }) +}) + +describe('node deltas', () => { + const e = (i: number, v = 0) => ({ + key: `k${String(i).padStart(3, '0')}`, + hash: sha256Hex(`${i}#${v}`), + size: 10, + }) + const base = Array.from({ length: 40 }, (_, i) => e(i)) + const changed = (at: number) => base.map((x, i) => (i === at ? e(i, 1) : x)) + /** Write one leaf of `entries` through `sink`; its hash. */ + const write = async (sink: RepoSink, entries: RecordEntry[]) => { + const json = encodeLeaf(recordTree, entries) + const hash = hashNode(json) + sink.leaf(describeNode(recordTree, decodeNode(recordTree, json, hash)), json, entries, 0) + await sink.flush() + return hash + } + const repoOf = () => + openRepo(memoryStore(), { layout: 'f', recordScope: 'c1', nodes: 'delta', trusted: true }) + + it('diff a rewritten leaf against the leaf it replaces, and no later leaf', async () => { + const repo = repoOf() + const a = await write(new RepoSink(repo), base) + const sink = new RepoSink(repo) + sink.rewriting(a) + const b = await write(sink, changed(3)) + // The sink's next leaf rewrites nothing (a merge reused the nodes between, or appends). + const c = await write(sink, changed(7)) + expect(await repo.deltaBase(b)).toBe(a) + expect(await repo.deltaBase(c)).toBeNull() + }) + + it('go to a mirror in full: it may not hold the node a delta is against', async () => { + const repo = repoOf() + const a = await write(new RepoSink(repo), base) + const sink = new RepoSink(repo) + sink.rewriting(a) + const b = await write(sink, changed(3)) + expect(await repo.deltaBase(b)).toBe(a) + const tree = { set: 'public', kind: 'records', slug: 'T', base: null, target: b } as const + const sent = [] + for await (const o of treeObjects(repo, tree)) sent.push(o) + expect(sent.map((o) => o.key)).toEqual([`nodes/${b}`]) + expect(sha256Hex(await gunzipText(sent[0]!.bytes))).toBe(b) + }) +}) + describe("F's manifest head", () => { // Split merges' swaps, reclaim and forks settle the head from their own jobs, beside // the maintain job: when it was last written must come from the head, not from an diff --git a/packages/server/src/locations/mirror.ts b/packages/server/src/locations/mirror.ts index 2709f35..f8fbc35 100644 --- a/packages/server/src/locations/mirror.ts +++ b/packages/server/src/locations/mirror.ts @@ -102,13 +102,17 @@ async function copyRecordFiles( contentType: 'application/json', }) await dest.put(manifestKeys.head, JSON.stringify(manifest), { contentType: 'application/json' }) - // What no manifest lists any more (merged away) goes from the mirror too. + // What no manifest lists any more (merged away) goes from the mirror too, but for + // files another placement at this location may hold (KeyedLayout.mirrorDroppable). const listed = new Set(files.map((f) => f.key)) - for (const k of [...copied]) - if (!listed.has(k)) { - await dest.delete(k).catch(() => undefined) - copied.delete(k) - } + const { drop, forget } = await repo.records.mirrorDroppable( + [...copied].filter((k) => !listed.has(k)), + ) + for (const k of drop) { + await dest.delete(k).catch(() => undefined) + copied.delete(k) + } + for (const k of forget) copied.delete(k) await remember() return true } diff --git a/packages/server/src/stores.ts b/packages/server/src/stores.ts index c391182..daf1340 100644 --- a/packages/server/src/stores.ts +++ b/packages/server/src/stores.ts @@ -48,8 +48,17 @@ export interface PlatformStorage { const PRIMARY_TTL_MS = 60_000 const primaryCache = new Map() +/** + * Layout F's repositories (one a collection) an isolate keeps, the most recently + * used: each holds its manifests' state, and an isolate serving thousands of + * collections would otherwise keep every one it ever opened. + */ +export const collectionRepos = { max: 256 } + export function createStores(db: Db, cache: Cache, platform: PlatformStorage): Stores { const repos = new Map() + /** F's, by location and collection; a Map's order is its recency (a hit moves to the end). */ + const runRepos = new Map() const files = platform.files ?? presigning(platform.bucket) const forLocation = async (locationId: string): Promise => { @@ -82,8 +91,12 @@ export function createStores(db: Db, cache: Cache, platform: PlatformStorage): S */ const forCollectionRuns = async (locationId: string, collectionId: string): Promise => { const k = `${locationId}/${collectionId}` - const known = repos.get(k) - if (known) return known + const known = runRepos.get(k) + if (known) { + runRepos.delete(k) + runRepos.set(k, known) + return known + } const loc = await forLocation(locationId) const repo = new Repo(loc.blobs, { cache, @@ -94,7 +107,8 @@ export function createStores(db: Db, cache: Cache, platform: PlatformStorage): S recordScope: collectionId, nodes: 'delta', }) - repos.set(k, repo) + runRepos.set(k, repo) + if (runRepos.size > collectionRepos.max) runRepos.delete(runRepos.keys().next().value!) return repo } const located = (locationId: string, collectionId: string, layout: LayoutName) => diff --git a/packages/server/src/versions/fork.ts b/packages/server/src/versions/fork.ts index e359f2a..2c62eff 100644 --- a/packages/server/src/versions/fork.ts +++ b/packages/server/src/versions/fork.ts @@ -10,7 +10,7 @@ * repository never references another location. */ import { type LayoutName, makeRoot, newSalt } from '@underlay/protocol' -import { eq } from 'drizzle-orm' +import { and, eq } from 'drizzle-orm' import { FenceError, fenced } from '../cleanup/fence.js' import * as schema from '../db/schema.js' @@ -70,6 +70,21 @@ export async function forkCollection( collection: typeof schema.collections.$inferSelect version: typeof schema.versions.$inferSelect }> { + // The fork is made on the platform location (createCollectionRows), and a repository + // never references another location's objects: in layout F the fork would list files + // its location doesn't hold. Checked by location, as F opens a repository a collection. + const [primary] = await ports.db + .select({ locationId: schema.placements.locationId }) + .from(schema.placements) + .where( + and( + eq(schema.placements.collectionId, source.collection.id), + eq(schema.placements.role, 'primary'), + ), + ) + .limit(1) + if (primary?.locationId !== schema.PLATFORM_LOCATION_ID) + throw new Error('Forks across storage locations are not supported yet') const repo = await ports.stores.forCollection(source.collection.id) const root = await repo.root(source.version.hash) const priv = includePrivate && root.private ? await repo.privateSet(root.private) : null @@ -86,8 +101,7 @@ export async function forkCollection( // at a time; a fork of the public set only never references the private set's. for (const set of priv ? (['public', 'private'] as const) : (['public'] as const)) await repo.records[set].forkInto!(targetRepo.records[set]) - } else if (targetRepo !== repo) - throw new Error('Forks across storage locations are not supported yet') + } const newRoot = makeRoot(root.metadata, root.public, priv) const v = source.version diff --git a/packages/server/test/layouts.test.ts b/packages/server/test/layouts.test.ts index 8a51dfb..ca2ea26 100644 --- a/packages/server/test/layouts.test.ts +++ b/packages/server/test/layouts.test.ts @@ -12,6 +12,8 @@ import { afterAll, afterEach, describe, expect, it } from 'vitest' import * as schema from '../src/db/schema.js' import { parallelConfig } from '../src/push/parallel.js' +import { collectionRepos } from '../src/stores.js' +import { forkCollection } from '../src/versions/fork.js' import { cleanup, type Harness, harness } from './harness.js' afterAll(cleanup) @@ -213,3 +215,67 @@ describe("today's layout and layout F side by side", () => { expect(sha(a.out.join('\n'))).toBe(sha(b.out.join('\n'))) }, 60_000) }) + +describe("layout F's repositories", () => { + it('are kept for the most recently used collections only', async () => { + // One a collection: an isolate serving thousands of collections kept every one. + collectionRepos.max = 2 + try { + const h = await harness() + const [a, b, c] = [ + await h.collection('a', { recordLayout: 'f' }), + await h.collection('b', { recordLayout: 'f' }), + await h.collection('c', { recordLayout: 'f' }), + ] + const open = (x: { id: string }) => h.ports.stores.forCollection(x.id) + const ra = await open(a) + const rb = await open(b) + // A hit counts as a use: `b` is now the oldest, and `c` takes its place. + expect(await open(a)).toBe(ra) + await open(c) + expect(await open(a)).toBe(ra) + expect(await open(b)).not.toBe(rb) + } finally { + collectionRepos.max = 256 + } + }) +}) + +describe('forks', () => { + it('are refused for a collection on another storage location', async () => { + // A fork is made on the platform location, and a repository never references + // another location's objects (F would list files its location doesn't hold). + const h = await harness() + const user = await h.member() + await h.ports.db.insert(schema.storageLocations).values({ + id: 'other', + kind: 'platform', + name: 'Other', + prefix: 'other', + status: 'active', + }) + const c = await h.collection('lib') + await h.ports.db + .update(schema.placements) + .set({ locationId: 'other' }) + .where(eq(schema.placements.collectionId, c.id)) + const path = '/api/collections/org/lib' + await push(h, user, path, { schemas: { Author } }, [rec('a1', 'A')], [], false) + const [col] = await h.ports.db + .select() + .from(schema.collections) + .where(eq(schema.collections.id, c.id)) + const [version] = await h.ports.db + .select() + .from(schema.versions) + .where(eq(schema.versions.collectionId, c.id)) + await expect( + forkCollection( + h.ports, + { collection: col!, version: version! }, + { organizationId: 'org1', slug: 'lib-fork', name: 'lib-fork', public: false }, + true, + ), + ).rejects.toThrow(/across storage locations/) + }) +}) diff --git a/packages/server/test/mirrors.test.ts b/packages/server/test/mirrors.test.ts index 5d1920d..3ba40cb 100644 --- a/packages/server/test/mirrors.test.ts +++ b/packages/server/test/mirrors.test.ts @@ -255,6 +255,62 @@ describe('bucket mirrors of layout F (records in packs of the collection)', () = for await (const e of iterate(src, t.root, { payloads: true })) if (e.body) n++ expect(n).toBe(t.count) }) + + it.each(['fork', 'parent'])( + 'keeps the files a fork and its parent share, mirrored to one location, when the %s merges them away', + async (merger) => { + // A fork reads its parent's files at the parent's keys, and a mirror copies them + // there: one placement deleting what its collection no longer lists took files + // the other's still does. + const { h, user, path } = await setup({ newCollections: 'f' }) + const f = await fake() + const loc = await addLocation(h, user, f, 'mirror') + const mirror = async (p: string) => + expect( + ( + await h.request(`${p}/placements`, { + method: 'POST', + user, + json: { locationId: loc.body.location.id, sets: 'public+private' }, + }) + ).status, + ).toBe(201) + await mirror(path) + await h.drain() + const forkPath = '/api/collections/org/books-fork' + const forked = await h.request(`${path}/fork`, { + method: 'POST', + user, + json: { targetOrg: 'org', slug: 'books-fork' }, + }) + expect(forked.status).toBe(201) + await mirror(forkPath) + await h.drain() + // Small commits, until one side's pack merge takes packs both list. + for (let i = 0; i < 8; i++) { + await push(h, user, merger === 'fork' ? forkPath : path, {}, [book(`n${i}`, 'n')]) + await h.drain() + } + // Each mirror reads from the bucket alone. + for (const slug of ['books', 'books-fork']) { + const [col] = await h.ports.db + .select() + .from(schema.collections) + .where(eq(schema.collections.slug, slug)) + const [head] = await h.ports.db + .select() + .from(schema.versions) + .where(eq(schema.versions.id, col!.headVersionId!)) + const repo = mirrorRepo(f, 'mirror', col!) + expect(await readHead(repo, col!.id)).toMatchObject({ versionHash: head!.hash }) + const t = (await repo.root(head!.hash)).public.types.Book! + let n = 0 + const src = new RepoSource(recordTree, repo, { set: 'public' }) + for await (const e of iterate(src, t.root, { payloads: true })) if (e.body) n++ + expect(n).toBe(t.count) + } + }, + ) }) describe('bucket mirrors', () => {