From fefc32024071a2a45d66766b53edbb6ff52ae443 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 21 Oct 2026 21:24:11 +0200 Subject: [PATCH 1/3] Layout F: the index split, each level pack carrying its own index and the history leveled --- packages/protocol/src/repo/layouts/keyed.ts | 568 ++- packages/protocol/src/repo/layouts/kit.ts | 44 +- packages/protocol/src/repo/layouts/levels.ts | 64 +- packages/protocol/test/repo/leveled.test.ts | 119 +- .../server/drizzle/0029_push_pack_indexed.sql | 1 + .../server/drizzle/meta/0029_snapshot.json | 3941 +++++++++++++++++ packages/server/drizzle/meta/_journal.json | 9 +- packages/server/src/cleanup/layout-reclaim.ts | 15 +- packages/server/src/db/schema.ts | 2 + packages/server/src/push/packs.ts | 2 + packages/server/test/cleanup.test.ts | 5 +- 11 files changed, 4628 insertions(+), 142 deletions(-) create mode 100644 packages/server/drizzle/0029_push_pack_indexed.sql create mode 100644 packages/server/drizzle/meta/0029_snapshot.json diff --git a/packages/protocol/src/repo/layouts/keyed.ts b/packages/protocol/src/repo/layouts/keyed.ts index d540290..bb18b51 100644 --- a/packages/protocol/src/repo/layouts/keyed.ts +++ b/packages/protocol/src/repo/layouts/keyed.ts @@ -41,7 +41,10 @@ import { type PackHooks, PackWriter, packDirKey, + packIndex, + packIndexId, packKey, + packKeys, parseLoc, topSeq, layerDirKey, @@ -52,12 +55,16 @@ import { import { LEVEL_RATIO, levelConfig, + levelKey, type LevelShard, + type Levels, LevelShards, levelRaw, levelTargets, MAX_LEVEL, + overlaps, shardKey, + type Tree, } from './levels.js' import { type KeyRange, LiveHashes } from './reclaim.js' import { @@ -93,11 +100,20 @@ export interface ReclaimPlan { /** Where the new run goes: the plan's lowest level and newest sequence (newer commits still win). */ level: number seq: number + /** + * Leveled: the levels and the history's level 0 as planned. The index holds level + * 0's entries only, so a range job counts these packs' live records through each + * pack's own index. + */ + levels?: Levels + history?: PackInfo[] } interface RangeResult { files: LayerInfo[] perPack: Record + /** Leveled: each level or history pack's live records in the range, and its records. */ + leveled?: Record kept: number dropped: number } @@ -119,6 +135,8 @@ export interface MergePlan { level: number seq: number at: number + /** Leveled: a merge of every run, settling this many of `stats.indexStale` when it swaps in. */ + stale?: number } const mergeKey = (plan: string, name: string) => `merges/${plan}/${name}` @@ -240,7 +258,7 @@ export class KeyedLayout implements RecordStore { this.packs = new PackReader(store, lru, scope) this.index = new LayeredIndex(store, lru, scope, this.manifests) this.layers = this.index.reader - this.levels = new LevelShards(store) + this.levels = new LevelShards(store, lru) } /** Whether no pack merge will take `p` again: it then lives in the registry, not the manifest. */ @@ -288,28 +306,32 @@ export class KeyedLayout implements RecordStore { /** * Whether `p` is listed in the registry rather than the manifest or a level: in a - * leveled collection only history packs are (every other is merged again). + * leveled collection none is (every pack is merged again, history ones too). */ #final(p: PackInfo, leveled: boolean): boolean { - return leveled ? !!p.history : KeyedLayout.final(p) + return !leveled && KeyedLayout.final(p) } - /** Every pack of levels 1… , by level. */ - async #levelPacks(m: Manifest): Promise> { - const out = new Map() + /** Packs a leveled lookup found swept (#leafLines): their index entries are skipped. */ + readonly #gone = new Set() + + /** Every pack of the levels (both trees'), by level key. */ + async #levelPacks(m: Manifest): Promise> { + const out = new Map() for (const [l, shards] of Object.entries(m.levels ?? {})) - if (shards.length) out.set(Number(l), await this.levels.packs(shards)) + if (shards.length) out.set(l, await this.levels.packs(shards)) return out } + /** Every pack the levels and the history's level 0 list (a leveled collection's, past level 0). */ + async #leveledPacks(m: Manifest): Promise { + return [...[...(await this.#levelPacks(m)).values()].flat(), ...(m.history ?? [])] + } + /** Every pack: the manifest's, the levels' and the registry's (reclaim, forks). */ async allPacks(): Promise { const m = await this.manifests.latest() - return [ - ...m.packs, - ...[...(await this.#levelPacks(m)).values()].flat(), - ...(await this.#registered(m)).packs, - ] + return [...m.packs, ...(await this.#leveledPacks(m)), ...(await this.#registered(m)).packs] } /** @@ -330,10 +352,59 @@ export class KeyedLayout implements RecordStore { entries: readonly RecordEntry[], ): Promise<(string | undefined)[]> { const keys = entries.map((e) => recordKey(type, e.key, e.hash)) - const found = await this.index.values(m, keys) + const found = + m.levels || m.history ? await this.#findLeveled(m, keys) : await this.index.values(m, keys) return keys.map((k) => found.get(k)) } + /** + * A leveled collection's lookup: level 0 through the index (commits' runs); then + * every current level at once, each pack through its own index; then the history's + * level 0 and levels for what's left (an old version's replaced records). A level + * holds at most a pack or two of a leaf's range, so a leaf reads a few directories + * and pages a level, however many days of changes went in. + * + * The index also has entries of packs that left level 0 (merged down, or a sorted + * load adopted as the bottom level) until its runs merge: a place stays good while + * its pack is there (any copy of a key is the same record), and one whose pack was + * swept is skipped (#leafLines). + */ + async #findLeveled(m: Manifest, keys: readonly string[]): Promise> { + const found = new Map() + for (const [k, v] of await this.index.values(m, keys)) + if (!this.#gone.has(parseLoc(v).pack)) found.set(k, v) + let left = [...new Set(keys)].filter((k) => !found.has(k)).sort(compareUtf8) + const probe = async (packs: PackInfo[]) => { + const got = await Promise.all( + packs.map((p) => + p.indexed + ? this.layers.pick( + packIndex(p), + left.filter((k) => compareUtf8(k, p.lo!) >= 0 && compareUtf8(k, p.hi!) <= 0), + ) + : [], + ), + ) + for (const r of got) for (const [k, v] of r) if (!found.has(k)) found.set(k, v) + left = left.filter((k) => !found.has(k)) + } + const levelsOf = async (tree: Tree) => { + const out: PackInfo[] = [] + for (let l = 1; l <= MAX_LEVEL; l++) { + const shards = m.levels?.[levelKey(tree, l)] + if (shards?.length) + out.push(...(await this.levels.overlapping(shards, left[0]!, left.at(-1)!))) + } + return out + } + if (left.length) await probe(await levelsOf('')) + if (left.length) { + const zeroH = (m.history ?? []).filter((p) => overlaps(p.lo!, p.hi!, left[0]!, left.at(-1)!)) + await probe([...zeroH, ...(await levelsOf('h'))]) + } + return found + } + async leafLines(leaf: TypedLeaf): Promise { if (!leaf.entries.length) return [] try { @@ -341,12 +412,31 @@ export class KeyedLayout implements RecordStore { } catch (err) { // A head behind the newest manifest (F moves it at most once a second) can name // files a merge has since swept: read again from the newest. - if (!(err instanceof Error) || !/^Missing (pack|layer)/.test(err.message)) throw err + if (!(err instanceof Error) || !/^Missing (pack|layer|level shard)/.test(err.message)) + throw err return this.#leafLines(leaf, await this.manifests.latest()) } } async #leafLines(leaf: TypedLeaf, m: Manifest): Promise { + for (let tries = 0; ; tries++) { + const places = await this.#places(leaf, m) + try { + return await this.packs.lines(places) + } catch (err) { + // Leveled: an index entry naming a pack that merged down and was swept since + // (the index drops those when its runs merge). Its records are in the levels: the + // pack is skipped from now on (ids are never reused) and the leaf found again. + const gone = err instanceof Error && /^Missing pack (?:directory )?(\S+)$/.exec(err.message) + if (!gone || !(m.levels || m.history) || this.#gone.has(gone[1]!) || tries >= 8) throw err + if (this.#gone.size > 100_000) this.#gone.clear() + this.#gone.add(gone[1]!) + } + } + } + + /** Where each of a leaf's records is, or MissingRecordError. */ + async #places(leaf: TypedLeaf, m: Manifest): Promise { let found = await this.#find(m, leaf.type, leaf.entries) // Another isolate may have committed since this one read the manifest. if (found.some((x) => x === undefined)) @@ -358,7 +448,7 @@ export class KeyedLayout implements RecordStore { `Layout ${this.name}: leaf ${leaf.hash} has no stored ${leaf.type} record ${e.key} (${e.hash})`, ) } - return this.packs.lines(found as string[]) + return found as string[] } linesAt(leaf: TypedLeaf, at: readonly number[]): Promise { @@ -381,7 +471,9 @@ export class KeyedLayout implements RecordStore { * last starts a new pack. */ uploadWriter(hooks: PackHooks = {}): UploadWriter { - const pw = new PackWriter(this.store, { hooks }) + // A leveled collection's packs each carry their own index: a sorted load's are its + // bottom level, which no index layer covers. + const pw = new PackWriter(this.store, { hooks, index: this.leveled() }) return { append: (r) => pw.append(recordKey(r.type, r.id, r.hash), r.line), ready: () => pw.ready(), @@ -438,8 +530,10 @@ export class KeyedLayout implements RecordStore { */ async #adoptLeveled(name: string, packs: readonly PackInfo[]): Promise { const sorted = [...packs].sort((a, b) => compareUtf8(a.lo ?? '', b.lo ?? '')) + // Each carrying its own index (nothing else will once level 0's index entries stop + // counting for them), and none overlapping. const disjoint = sorted.every( - (p, i) => p.lo && p.hi && (i === 0 || compareUtf8(p.lo, sorted[i - 1]!.hi!) > 0), + (p, i) => p.indexed && p.lo && p.hi && (i === 0 || compareUtf8(p.lo, sorted[i - 1]!.hi!) > 0), ) const empty = (m: Manifest) => !m.packs.length && !Object.values(m.levels ?? {}).some((s) => s.length) @@ -448,7 +542,9 @@ export class KeyedLayout implements RecordStore { let placed = false await this.manifests.update((m) => { if (m.adopted?.includes(name) || !empty(m)) return null - m.levels = { ...m.levels, [MAX_LEVEL]: shards } + m.levels = { ...m.levels, [levelKey('', MAX_LEVEL)]: shards } + // The load's commit indexed every record: entries no lookup takes now (#mergeLayers). + m.stats.indexStale = (m.stats.indexStale ?? 0) + packs.reduce((n, p) => n + p.records, 0) m.adopted = [...(m.adopted ?? []), name].slice(-ADOPTED_KEPT) placed = true return m @@ -477,14 +573,12 @@ export class KeyedLayout implements RecordStore { const registered = await this.store.get(this.registry.keyFor(`upload-${name}`)) if (registered) for (const p of JSON.parse(await registered.text()) as PackInfo[]) listed.add(p.id) - for (const ps of (await this.#levelPacks(m)).values()) for (const p of ps) listed.add(p.id) + for (const p of await this.#leveledPacks(m)) listed.add(p.id) const gone = ids.filter((id) => !listed.has(id) && !listed.has(packKey(id))) - for (let i = 0; i < gone.length; i += 16) - await Promise.all( - gone - .slice(i, i + 16) - .flatMap((id) => [this.store.delete(packKey(id)), this.store.delete(packDirKey(id))]), - ) + // With its own index, if it had one (deleting what isn't there is nothing). + const keys = gone.flatMap((id) => packKeys({ id, indexed: true })) + for (let i = 0; i < keys.length; i += 16) + await Promise.all(keys.slice(i, i + 16).map((k) => this.store.delete(k))) return gone } @@ -519,7 +613,9 @@ export class KeyedLayout implements RecordStore { const leveled = await this.leveled() for (;;) { const layers = await this.#mergeLayers(opts.split) - const packs = leveled ? await this.#mergeLevels() : await this.#mergePacks() + const packs = leveled + ? [await this.#mergeLevels(''), await this.#mergeLevels('h')].some(Boolean) + : await this.#mergePacks() if (!layers && !packs) break if (Date.now() >= until || spent() >= layoutConfig.maintainCalls) { more = true @@ -590,6 +686,7 @@ export class KeyedLayout implements RecordStore { const blocked = new Set( (split ? await this.#mergesInFlight() : []).flatMap((p) => runs(p.layers).map((r) => r.id)), ) + const started = Date.now() const m = await this.manifests.latest() const all = runs(m.layers).sort((a, b) => b.seq - a.seq) const size = (r: (typeof all)[number]) => r.files.reduce((n, f) => n + f.entries, 0) @@ -599,15 +696,33 @@ export class KeyedLayout implements RecordStore { if (!blocked.has(r.id)) free.at(-1)!.push(r) else if (free.at(-1)!.length) free.push([]) let inputs: typeof all | null = null - for (const rs of free) { - let take = 1 - let newer = rs.length ? size(rs[0]!) : 0 - while (take < rs.length && size(rs[take]!) <= newer) newer += size(rs[take++]!) - if (take >= LAYER_FANOUT) { - inputs = rs.slice(0, take) - break - } + // Leveled: the index serves level 0 only, and entries of packs that went down are + // dead weight the size rule below may never reach (newer runs shrink as the keeper + // drops theirs, and a sorted load's run points only at the bottom level). Once + // they're half the index, every run merges at once: only a merge of them all is sure + // to meet every stale entry, so it waits for no split merge to hold any. + const stale = m.levels || m.history ? (m.stats.indexStale ?? 0) : 0 + let settles = 0 + if ( + stale && + all.length && + free[0]!.length === all.length && + all.length <= layoutConfig.mergeInputs && + stale >= Math.max(levelConfig.staleEntries, all.reduce((n, r) => n + size(r), 0) / 2) + ) { + inputs = all + settles = stale } + if (!inputs) + for (const rs of free) { + let take = 1 + let newer = rs.length ? size(rs[0]!) : 0 + while (take < rs.length && size(rs[take]!) <= newer) newer += size(rs[take++]!) + if (take >= LAYER_FANOUT) { + inputs = rs.slice(0, take) + break + } + } // Too many runs (small ones stranded behind a larger one, which the rule above // never reaches; a pack merge's run, the newest when written, strands all before // it): merge the longest stretch of adjacent runs no larger than the smallest @@ -640,7 +755,13 @@ export class KeyedLayout implements RecordStore { // The newest of them: still adjacent; the next pass takes the rest. inputs = inputs.slice(0, layoutConfig.mergeInputs) if (split && inputs.reduce((n, r) => n + size(r), 0) > layoutConfig.mergeJobEntries) { - await split(await this.mergePlan(inputs.flatMap((r) => r.files))) + await split( + await this.mergePlan( + inputs.flatMap((r) => r.files), + undefined, + settles, + ), + ) // Its inputs are a block now; other runs may still merge. return true } @@ -648,9 +769,15 @@ export class KeyedLayout implements RecordStore { let n = 0 // Entries of dropped packs never reach a merge: reclaim rewrites the index as it drops them. const read = mergeRead(inputs.length) + const keep = await this.#indexKeeper(m) + let dropped = 0 for await (const [k, v] of mergeSorted( inputs.map((r) => runEntries(this.layers, r.files, undefined, undefined, read)), )) { + if (keep && !(await keep(v))) { + dropped++ + continue + } out.add(k, v) if (++n % 1024 === 0) await out.ready() } @@ -658,10 +785,36 @@ export class KeyedLayout implements RecordStore { return this.#swap({ oldLayers: inputs.flatMap((r) => r.files.map((f) => f.id)), newLayers: files, + settles: settles || dropped, + ...(settles ? { settledAt: started } : {}), keepSeq: true, }) } + /** + * Leveled: which index entries a merge of index layers keeps (null: every one, a + * tiered collection). The index serves level 0 only, so an entry goes once its pack + * has: it's in a level or the history (its own index finds it there), retired, or + * gone. Not a pack that's merely unlisted yet: an upload's packs are listed only + * after its commit's entries naming them are in the index. + */ + async #indexKeeper(m: Manifest): Promise<((place: string) => Promise) | null> { + if (!m.levels && !m.history) return null + const zero = new Set(m.packs.map((p) => p.id)) + const left = new Set((await this.#leveledPacks(m)).map((p) => p.id)) + for (const r of m.retired) + if (r.key.startsWith('packs/') && r.key.endsWith('.pk')) left.add(r.key.slice(6, -3)) + const known = new Map>() + return (place: string) => { + const pack = parseLoc(place).pack + if (zero.has(pack)) return Promise.resolve(true) + if (left.has(pack)) return Promise.resolve(false) + let k = known.get(pack) + if (!k) known.set(pack, (k = this.store.head(packDirKey(pack)).then((h) => h !== null))) + return k + } + } + /** * Merge eight small packs of a tier and type into one, re-pointing their records in * a new layer. A merge takes one type's packs, so its output holds one type and a @@ -721,27 +874,32 @@ export class KeyedLayout implements RecordStore { } /** - * Leveled: one merge down, from the level most over its target (levels.ts). Level 0's - * oldest packs merge into the base level; a pack of a level below it merges into - * the next, picking the one whose overlap there is smallest for its size (RocksDB's - * kMinOverlappingRatio: each merge rewrites as little as it can). Both sides go - * through as streams into the next level's packs, cut at 16 MiB and at types, and a - * new layer points the index at where each record now is. + * Leveled: one merge down in `tree`, from the level most over its target (levels.ts). + * Level 0's oldest packs merge into the base level; a pack of a level below it merges + * into the next, picking the one whose overlap there is smallest for its size + * (RocksDB's kMinOverlappingRatio: each merge rewrites as little as it can). Both + * sides go through as streams into the next level's packs, cut at 16 MiB and at + * types, each carrying its own index: nothing else points at them, so a merge + * writes no index layer. * - * A record id found with several hashes keeps the newest (level 0's newest pack, - * then the upper level) in the levels; the older hashes go to this month's history - * packs. That's a guess at which is current: a revert brings back an old hash, then - * read from history. Only locality depends on it, since the index says where every - * copy is. + * In the current versions' tree a record id found with several hashes keeps the + * newest (level 0's newest pack, then the upper level) in the levels; the older + * hashes go to the history's level 0. That's a guess at which is current: a revert + * brings back an old hash, then read from history. Only locality depends on it, + * since a lookup goes on to the history for whatever the levels don't hold. The + * history's own merges keep every copy (one of each key). */ - async #mergeLevels(): Promise { + async #mergeLevels(tree: Tree): Promise { const m = await this.manifests.latest() - const { base, targets } = levelTargets(m.levels) - const zero = m.packs.filter((p) => !p.history) + const { base, targets } = levelTargets(m.levels, tree) + const zero = tree ? (m.history ?? []) : m.packs.filter((p) => !p.history) const zeroRaw = zero.reduce((n, p) => n + p.raw, 0) // Level 0 merges once it's a tenth of the level it merges into: while a collection // is small its base is the bottom, and a few small commits shouldn't rewrite it all. - const zeroTrigger = Math.max(levelConfig.level0Raw, levelRaw(m.levels, base) / LEVEL_RATIO) + const zeroTrigger = Math.max( + levelConfig.level0Raw, + levelRaw(m.levels, base, tree) / LEVEL_RATIO, + ) let from = -1 let score = 1 if (zero.length >= levelConfig.level0Packs || zeroRaw >= zeroTrigger) { @@ -749,7 +907,7 @@ export class KeyedLayout implements RecordStore { score = Math.max(zero.length / levelConfig.level0Packs, zeroRaw / zeroTrigger) } for (let l = 1; l < MAX_LEVEL; l++) { - const raw = levelRaw(m.levels, l) + const raw = levelRaw(m.levels, l, tree) if (!raw) continue // A level above the base has no room: what it holds goes down first. const s = targets[l]! > 0 ? raw / targets[l]! : Infinity @@ -759,6 +917,7 @@ export class KeyedLayout implements RecordStore { } } if (from < 0) return false + const shardsOf = (l: number) => m.levels?.[levelKey(tree, l)] ?? [] let upper: PackInfo[] let target: number if (from === 0) { @@ -773,8 +932,8 @@ export class KeyedLayout implements RecordStore { target = base } else { target = from + 1 - const here = await this.levels.packs(m.levels?.[from] ?? []) - const below = await this.levels.packs(m.levels?.[target] ?? []) + const here = await this.levels.packs(shardsOf(from)) + const below = await this.levels.packs(shardsOf(target)) // Each pack's overlap below, by a walk of both in key order. let best: PackInfo | null = null let bestRatio = Infinity @@ -794,7 +953,7 @@ export class KeyedLayout implements RecordStore { } const lo = upper.reduce((a, p) => (compareUtf8(p.lo!, a) < 0 ? p.lo! : a), upper[0]!.lo!) const hi = upper.reduce((a, p) => (compareUtf8(p.hi!, a) > 0 ? p.hi! : a), upper[0]!.hi!) - const lower = await this.levels.overlapping(m.levels?.[target] ?? [], lo, hi) + const lower = await this.levels.overlapping(shardsOf(target), lo, hi) // Newest first: level 0's newest pack, …, its oldest, then the level below. const ranked = [...(from === 0 ? [...upper].reverse() : upper), ...lower] const reader = this.packs @@ -805,9 +964,17 @@ export class KeyedLayout implements RecordStore { })(), ) const month = new Date().toISOString().slice(0, 7) - const pw = new PackWriter(this.store, { level: target, packRaw: levelConfig.packRaw }) - const hw = new PackWriter(this.store, { history: month, packRaw: levelConfig.packRaw }) - const lw = new RunWriter(this.store, 0) + const pw = new PackWriter(this.store, { + level: target, + packRaw: levelConfig.packRaw, + index: true, + ...(tree ? { history: month } : {}), + }) + const hw = new PackWriter(this.store, { + history: month, + packRaw: levelConfig.packRaw, + index: true, + }) let group: { key: string; line: string; rank: number }[] = [] let n = 0 const flush = async () => { @@ -818,14 +985,14 @@ export class KeyedLayout implements RecordStore { for (const x of group) { // A copy of a key already written (the same record twice) is one record. if (x.key === last) continue - if (x.key === cur.key) { - if (x !== cur) continue - lw.add(x.key, pw.append(x.key, x.line)) - } else lw.add(x.key, hw.append(x.key, x.line)) + if (tree || x.key === cur.key) { + if (!tree && x !== cur) continue + pw.append(x.key, x.line) + } else hw.append(x.key, x.line) last = x.key } group = [] - if (++n % 64 === 0) await Promise.all([pw.ready(), hw.ready(), lw.ready()]) + if (++n % 64 === 0) await Promise.all([pw.ready(), hw.ready()]) } for await (const x of mergeAll(streams)) { const id = x.key.slice(0, x.key.lastIndexOf('\u0000')) @@ -838,49 +1005,64 @@ export class KeyedLayout implements RecordStore { await flush() const packs = await pw.finish() const history = await hw.finish() - const files = await lw.finish() return this.#swapLevels({ - zero: from === 0 ? upper.map((p) => p.id) : [], - from: from > 0 ? { level: from, ids: upper.map((p) => p.id) } : null, + tree, + zero: from === 0 ? upper : [], + from: from > 0 ? { level: from, packs: upper } : null, target, - lower: lower.map((p) => p.id), + lower, packs, history, - layers: files, + stat: 'levelMerges', }) } /** * A leveled merge's result in one manifest change: its inputs out of level 0 or - * their level, its output in the target level, its history packs in the registry, - * and its layer the newest. Only if the levels it changed are as it read them and - * level 0's inputs are still listed; otherwise everything it wrote goes. + * their level, its output in the target level, its history packs in the history's + * level 0. Only if the levels it changed are as it read them and level 0's inputs + * are still listed; otherwise everything it wrote goes. */ async #swapLevels(o: { - zero: string[] - from: { level: number; ids: string[] } | null - target: number - lower: string[] + tree: Tree + zero: PackInfo[] + from: { level: number; packs: PackInfo[] } | null + /** The level its output goes to; null for none (the history's level 0 takes it). */ + target: number | null + lower: PackInfo[] packs: PackInfo[] history: PackInfo[] - layers: LayerInfo[] + stat: 'levelMerges' | 'levelRewrites' }): Promise { const m0 = await this.manifests.latest() - const changed = new Map>>() - if (o.from) + const changed = new Map>>() + if (o.from) { + const key = levelKey(o.tree, o.from.level) changed.set( - o.from.level, - await this.levels.replace(m0.levels?.[o.from.level] ?? [], new Set(o.from.ids), []), + key, + await this.levels.replace( + m0.levels?.[key] ?? [], + new Set(o.from.packs.map((p) => p.id)), + [], + ), ) - changed.set( - o.target, - await this.levels.replace(m0.levels?.[o.target] ?? [], new Set(o.lower), o.packs), - ) - const registered = await this.registry.add(o.history) - const zero = new Set(o.zero) + } + if (o.target !== null) { + const key = levelKey(o.tree, o.target) + changed.set( + key, + await this.levels.replace( + m0.levels?.[key] ?? [], + new Set(o.lower.map((p) => p.id)), + o.packs, + ), + ) + } + const zero = new Set(o.zero.map((p) => p.id)) let lost = false await this.manifests.update((m) => { - const have = new Set(m.packs.map((p) => p.id)) + const listed = o.tree ? (m.history ?? []) : m.packs + const have = new Set(listed.map((p) => p.id)) if ( ![...zero].every((id) => have.has(id)) || [...changed.keys()].some((l) => !sameShards(m.levels?.[l], m0.levels?.[l])) @@ -889,24 +1071,27 @@ export class KeyedLayout implements RecordStore { return null } const now = Date.now() - m.packs = m.packs.filter((p) => !zero.has(p.id)) + if (o.tree) m.history = listed.filter((p) => !zero.has(p.id)) + else { + m.packs = listed.filter((p) => !zero.has(p.id)) + // Their records' index entries now lead nowhere a lookup takes (#mergeLayers). + const gone = o.zero.reduce((n, p) => n + p.records, 0) + if (gone) m.stats.indexStale = (m.stats.indexStale ?? 0) + gone + } + if (o.history.length) m.history = [...(m.history ?? []), ...o.history] m.levels = { ...m.levels } for (const [l, c] of changed) m.levels[l] = c.shards - const seq = topSeq(m) + 1 - for (const l of o.layers) m.layers.push({ ...l, seq }) - for (const id of [...o.zero, ...(o.from?.ids ?? []), ...o.lower]) - m.retired.push({ key: packKey(id), at: now }, { key: packDirKey(id), at: now }) + for (const p of [...o.zero, ...(o.from?.packs ?? []), ...o.lower]) + for (const k of packKeys(p)) m.retired.push({ key: k, at: now }) for (const c of changed.values()) for (const id of c.replaced) m.retired.push({ key: shardKey(id), at: now }) - m.stats.levelMerges = (m.stats.levelMerges ?? 0) + 1 + m.stats[o.stat] = (m.stats[o.stat] ?? 0) + 1 return m }) if (!lost) return true const gone = [ - ...o.layers.flatMap((l) => [layerKey(l.id), layerDirKey(l.id), layerFilterKey(l.id)]), - ...[...o.packs, ...o.history].flatMap((p) => [packKey(p.id), packDirKey(p.id)]), + ...[...o.packs, ...o.history].flatMap(packKeys), ...[...changed.values()].flatMap((c) => c.written.map(shardKey)), - ...(registered ? [registered] : []), ] for (let i = 0; i < gone.length; i += 16) await Promise.all(gone.slice(i, i + 16).map((k) => this.store.delete(k))) @@ -932,16 +1117,21 @@ export class KeyedLayout implements RecordStore { relist?: PackInfo[] /** New layers keep their sequence (a merged run); else they're the newest (re-pointed records). */ keepSeq?: boolean + /** Of `stats.indexStale`, how many entries this merge dropped (or met, merging every run). */ + settles?: number + /** A merge of every run: when it started (no run names a pack retired before then). */ + settledAt?: number }): Promise { const oldLayers = new Set(o.oldLayers ?? []) const oldPacks = new Set(o.oldPacks ?? []) + const unlisted = new Set(o.unlisted ?? []) const newLayers = o.newLayers ?? [] const newPacks = o.newPacks ?? [] const leveled = await this.leveled() const listed = newPacks.filter((p) => !this.#final(p, leveled)) const registered = await this.registry.add(newPacks.filter((p) => this.#final(p, leveled))) const m0 = leveled ? await this.manifests.latest() : null - const changed = new Map>>() + const changed = new Map>>() const inLevels = new Set() if (m0) { const gone = new Set([...oldPacks, ...(o.unlisted ?? [])]) @@ -955,7 +1145,7 @@ export class KeyedLayout implements RecordStore { let lost = false await this.manifests.update((m) => { const haveL = new Set(m.layers.map((l) => l.id)) - const haveP = new Set(m.packs.map((p) => p.id)) + const haveP = new Set([...m.packs, ...(m.history ?? [])].map((p) => p.id)) if ( ![...oldLayers].every((id) => haveL.has(id)) || ![...oldPacks].every((id) => haveP.has(id) || inLevels.has(id)) || @@ -965,8 +1155,11 @@ export class KeyedLayout implements RecordStore { return null } const now = Date.now() + if (o.settles) m.stats.indexStale = Math.max(0, (m.stats.indexStale ?? 0) - o.settles) + if (o.settledAt) m.stats.indexSettledAt = Math.max(m.stats.indexSettledAt ?? 0, o.settledAt) m.layers = m.layers.filter((l) => !oldLayers.has(l.id)) m.packs = m.packs.filter((p) => !oldPacks.has(p.id)) + if (m.history) m.history = m.history.filter((p) => !oldPacks.has(p.id) && !unlisted.has(p.id)) if (changed.size) { m.levels = { ...m.levels } for (const [l, c] of changed) { @@ -986,7 +1179,8 @@ export class KeyedLayout implements RecordStore { { key: layerFilterKey(id), at: now }, ) for (const id of [...oldPacks, ...(o.unlisted ?? [])]) - m.retired.push({ key: packKey(id), at: now }, { key: packDirKey(id), at: now }) + for (const key of packKeys(leveled ? { id, indexed: true } : { id })) + m.retired.push({ key, at: now }) return m }) if (lost) { @@ -996,8 +1190,7 @@ export class KeyedLayout implements RecordStore { this.store.delete(layerDirKey(l.id)), this.store.delete(layerFilterKey(l.id)), ]) - for (const p of newPacks) - await Promise.all([this.store.delete(packKey(p.id)), this.store.delete(packDirKey(p.id))]) + for (const p of newPacks) await Promise.all(packKeys(p).map((k) => this.store.delete(k))) for (const c of changed.values()) await Promise.all(c.written.map((id) => this.store.delete(shardKey(id)))) if (registered) await this.store.delete(registered) @@ -1011,6 +1204,13 @@ export class KeyedLayout implements RecordStore { const m = await this.manifests.current(0) const cutoff = Date.now() - layoutConfig.graceMs let due = m.retired.filter((r) => r.at < cutoff) + // Leveled: the index may still name a pack retired since its runs last all merged + // (#mergeLayers), so that pack stays until they have; or until nothing it counts as + // stale is left (the count only ever overestimates). + if ((m.levels || m.history) && m.stats.indexStale) { + const settled = m.stats.indexSettledAt ?? 0 + due = due.filter((r) => !r.key.startsWith('packs/') || r.at < settled) + } if (!due.length) return true // F: a live fork may still read what this collection merged away; those stay // retired (swept once the fork goes) rather than deleted. @@ -1043,7 +1243,7 @@ export class KeyedLayout implements RecordStore { const swapped = await this.reclaimSwap(plan) if (!swapped) return {} for (let i = 0; i < swapped.rewrite.length; i += 4) - await this.reclaimRewrite(swapped.rewrite.slice(i, i + 4)) + await this.reclaimRewrite(swapped.rewrite.slice(i, i + 4), async () => h) return swapped.stats } @@ -1054,9 +1254,13 @@ export class KeyedLayout implements RecordStore { * each of about `rangeEntries` index entries. Saved for the range jobs. */ /** Key ranges over `layers` of about `rangeEntries` entries each, cut at record ids. */ - async #cutRanges(layers: readonly LayerInfo[], rangeEntries: number): Promise { + async #cutRanges( + layers: readonly LayerInfo[], + rangeEntries: number, + extra: [string, number][] = [], + ): Promise { // Cut points from the layers' page keys, sampled to keep a plan small. - const cuts: [string, number][] = [] + const cuts: [string, number][] = [...extra] const total = layers.reduce((n, l) => n + l.entries, 0) for (const l of layers) { const { keys, perPage } = await this.layers.pageKeys(l) @@ -1085,8 +1289,10 @@ export class KeyedLayout implements RecordStore { async mergePlan( layers: LayerInfo[], rangeEntries = layoutConfig.mergeJobEntries, + stale = 0, ): Promise { const plan: MergePlan = { + ...(stale ? { stale } : {}), id: newId(), layers, ranges: await this.#cutRanges(layers, rangeEntries), @@ -1116,12 +1322,14 @@ export class KeyedLayout implements RecordStore { const out = new RunWriter(this.store, plan.level) const inputs = runs(plan.layers).sort((a, b) => b.seq - a.seq) const read = mergeRead(inputs.length) + const keep = await this.#indexKeeper(await this.manifests.latest()) let n = 0 for await (const [k, v] of mergeSorted( inputs.map((x) => runEntries(this.layers, x.files, r.lo ?? undefined, r.hi ?? undefined, read), ), )) { + if (keep && !(await keep(v))) continue out.add(k, v) if (++n % 1024 === 0) await out.ready() } @@ -1155,6 +1363,7 @@ export class KeyedLayout implements RecordStore { oldLayers: plan.layers.map((l) => l.id), newLayers: files.map((f) => ({ ...f, level, seq: plan.seq, run: plan.id })), keepSeq: true, + ...(plan.stale ? { settles: plan.stale, settledAt: plan.at } : {}), }) await this.#dropMerge(plan, false) await this.manifests.settle() @@ -1207,13 +1416,20 @@ export class KeyedLayout implements RecordStore { const n = runs(m.layers).length if (n > RECLAIM_MAX_RUNS) throw new TooManyRunsError(`Reclaim: ${n} index runs, more than ${RECLAIM_MAX_RUNS}`) - const ranges = await this.#cutRanges(m.layers, rangeEntries) + // Leveled: the levels' packs cut ranges too (by their first keys, each its records), + // since the index no longer has their entries. + const leveled = m.levels || m.history ? await this.#leveledPacks(m) : null + const ranges = await this.#cutRanges( + m.layers, + rangeEntries, + (leveled ?? []).map((p) => [filterKey(p.lo!), p.records]), + ) const registry = await this.#registered(m) - const levelPacks = [...(await this.#levelPacks(m)).values()].flat() const plan: ReclaimPlan = { id: newId(), layers: m.layers, - packs: [...m.packs, ...levelPacks, ...registry.packs].map((p) => ({ + ...(leveled ? { levels: m.levels ?? {}, history: m.history ?? [] } : {}), + packs: [...m.packs, ...registry.packs].map((p) => ({ id: p.id, records: p.records, })), @@ -1264,6 +1480,7 @@ export class KeyedLayout implements RecordStore { } const files = await out.finish(plan.seq) const result: RangeResult = { files, perPack, kept, dropped } + if (plan.levels || plan.history) result.leveled = await this.#countLeveled(plan, r, live) await this.store.put( reclaimKey(plan.id, `range-${String(i).padStart(6, '0')}`), JSON.stringify(result), @@ -1278,6 +1495,40 @@ export class KeyedLayout implements RecordStore { return done >= plan.ranges.length } + /** + * Leveled: the live records in range `r` of each level and history pack the plan + * lists, read from its own index (a pack across ranges is counted in each). + */ + async #countLeveled( + plan: ReclaimPlan, + r: KeyRange, + live: LiveHashes, + ): Promise> { + // Index keys in [lo, hi) are the ones whose type and id are (reclaim.ts, KeyRange). + const within = (lo: string, hi: string) => + (r.hi === null || compareUtf8(lo, r.hi) < 0) && (r.lo === null || compareUtf8(hi, r.lo) >= 0) + const packs = (plan.history ?? []).filter((p) => within(p.lo!, p.hi!)) + for (const shards of Object.values(plan.levels ?? {})) + for (const p of await this.levels.packs(shards.filter((s) => within(s.lo, s.hi)))) + if (within(p.lo!, p.hi!)) packs.push(p) + const out: Record = {} + for (const p of packs) { + let n = 0 + for await (const [k] of runEntries( + this.layers, + [packIndex(p)], + r.lo ?? undefined, + r.hi ?? undefined, + mergeRead(1), + )) { + const z = k.lastIndexOf('\u0000') + if (live.has(k.slice(0, z), k.slice(z + 1))) n++ + } + out[p.id] = [n, p.records] + } + return out + } + /** * Step 3, once every range is done (and only once: the first caller claims it): * swap the ranges' run in for the plan's layers, drop packs with nothing live, @@ -1292,6 +1543,7 @@ export class KeyedLayout implements RecordStore { if ((await (await this.store.get(reclaimKey(plan.id, 'swap')))?.text()) !== token) return null const files: LayerInfo[] = [] const live = new Map() + const leveled = new Map() const stats = { ranges: plan.ranges.length, entriesKept: 0, @@ -1311,6 +1563,8 @@ export class KeyedLayout implements RecordStore { const r = JSON.parse(await o.text()) as RangeResult files.push(...r.files.map((f) => ({ ...f, seq: plan.seq, run: plan.id }))) for (const [p, n] of Object.entries(r.perPack)) live.set(p, (live.get(p) ?? 0) + n) + for (const [p, [n, records]] of Object.entries(r.leveled ?? {})) + leveled.set(p, [(leveled.get(p)?.[0] ?? 0) + n, records]) stats.entriesKept += r.kept stats.entriesDropped += r.dropped } @@ -1322,6 +1576,14 @@ export class KeyedLayout implements RecordStore { else if (n < p.records / 2) rewrite.push(p.id) stats.recordsDropped += p.records - n } + // Leveled packs, in place: one with nothing live goes, one under half is copied + // within its level (reclaimRewrite). + for (const [id, [n, records]] of leveled) { + if (n === 0) drop.push(id) + else if (n < records / 2) rewrite.push(id) + stats.recordsDropped += records - n + } + stats.packs += leveled.size stats.packsDropped = drop.length stats.packsRewritten = rewrite.length // One run across the ranges, at the level its whole size puts it. @@ -1370,12 +1632,84 @@ export class KeyedLayout implements RecordStore { } /** - * Step 4, a few packs a job: copy their live records into new packs and point - * the index at them with a new, newest layer. A record is live when the index - * still points at this copy of it. + * Step 4, a few packs a job: copy their live records into new packs. Level 0's (and + * a tiered collection's) are pointed at by a new, newest index layer, and a record + * is live when the index still points at this copy of it. A leveled collection's + * level and history packs are copied in place, within their level: no index entry + * says what's live there, so `mark` marks the pack's key range again (the server + * job's markRange over the retained versions). */ - async reclaimRewrite(packIds: readonly string[]): Promise> { + async reclaimRewrite( + packIds: readonly string[], + mark?: (range: KeyRange) => Promise, + ): Promise> { const m = await this.manifests.latest() + const stats = { packsRead: 0, recordsKept: 0, recordsDropped: 0 } + const leveled = new Map( + (m.levels || m.history ? await this.#leveledPacks(m) : []).map((p) => [p.id, p]), + ) + const zeroH = new Set((m.history ?? []).map((p) => p.id)) + for (const id of packIds) { + const p = leveled.get(id) + if (!p) continue + if (!mark) throw new Error('reclaimRewrite: a level pack needs its range marked') + const [kept, dropped] = await this.#rewriteLeveled(p, zeroH.has(id), mark) + stats.packsRead++ + stats.recordsKept += kept + stats.recordsDropped += dropped + } + const rest = packIds.filter((id) => !leveled.has(id)) + if (rest.length) + for (const [k, v] of Object.entries(await this.#rewriteRepointed(m, rest))) + stats[k as keyof typeof stats] += v + return stats + } + + /** A leveled pack's live records copied to new packs of its own level (or the history's level 0). */ + async #rewriteLeveled( + p: PackInfo, + inZero: boolean, + mark: (range: KeyRange) => Promise, + ): Promise<[number, number]> { + // Its ids from the first through the last (no id holds a NUL, so this is just past it). + const live = await mark({ lo: filterKey(p.lo!), hi: `${filterKey(p.hi!)}\u0000` }) + const pw = new PackWriter(this.store, { + packRaw: levelConfig.packRaw, + index: true, + ...(p.level ? { level: p.level } : {}), + ...(p.history ? { history: p.history } : {}), + }) + let kept = 0 + let dropped = 0 + for await (const { key, line } of this.packs.all(p.id, mergeRead(1))) { + const z = key.lastIndexOf('\u0000') + if (!live.has(key.slice(0, z), key.slice(z + 1))) { + dropped++ + continue + } + pw.append(key, line) + if (++kept % 1024 === 0) await pw.ready() + } + const packs = await pw.finish() + // Within its old range, so its level still doesn't overlap. + await this.#swapLevels({ + tree: p.history ? 'h' : '', + zero: inZero ? [p] : [], + from: null, + target: inZero ? null : p.level!, + lower: inZero ? [] : [p], + packs: inZero ? [] : packs, + history: inZero ? packs : [], + stat: 'levelRewrites', + }) + return [kept, dropped] + } + + /** Reclaim's rewrite of packs the index points into. */ + async #rewriteRepointed( + m: Manifest, + packIds: readonly string[], + ): Promise> { // Packs the manifest lists must still be there; the others (an older reclaim's) are in neither. const listed = new Set(m.packs.map((p) => p.id)) const retired = new Set(m.retired.map((r) => r.key)) @@ -1443,8 +1777,12 @@ export class KeyedLayout implements RecordStore { // Each file's home: where the parent read it from (its own parent's), else the parent. const mine = await from.inherited() const inherited: Inherited = { packs: {}, layers: {} } - for (const p of [...m.packs, ...registered, ...[...levelPacks.values()].flat()]) + for (const p of [...m.packs, ...registered, ...(await this.#leveledPacks(m))]) { inherited.packs[p.id] = mine.packs[p.id] ?? from.home + // A pack's own index is a layer file, read from where the pack is. + const index = packIndexId(p.id) + if (p.indexed) inherited.layers[index] = mine.layers[index] ?? from.home + } for (const l of m.layers) inherited.layers[l.id] = mine.layers[l.id] ?? from.home const homes = new Set([...Object.values(inherited.packs), ...Object.values(inherited.layers)]) // Every home first learns of the fork: from then on it keeps what the fork lists. @@ -1467,6 +1805,7 @@ export class KeyedLayout implements RecordStore { const top = Math.max(0, ...m.layers.map((l) => l.seq)) await target.manifests.update((n) => { if (Object.keys(levels).length) n.levels = levels + if (m.history) n.history = [...m.history] n.packs.push(...m.packs) n.layers.push(...m.layers.map((l) => ({ ...l, seq: l.seq - top - 1 }))) return n @@ -1503,10 +1842,13 @@ export class KeyedLayout implements RecordStore { { key: at(h, layerFilterKey(l.id)), optional: true }, ) } - const levelPacks = [...(await this.#levelPacks(m)).values()].flat() - for (const p of [...m.packs, ...levelPacks, ...registry.packs]) { + for (const p of [...m.packs, ...(await this.#leveledPacks(m)), ...registry.packs]) { const h = inh.packs[p.id] files.push({ key: at(h, packKey(p.id)), bytes: p.bytes }, { key: at(h, packDirKey(p.id)) }) + if (!p.indexed) continue + const index = packIndexId(p.id) + const hi = inh.layers[index] + files.push({ key: at(hi, layerKey(index)) }, { key: at(hi, layerDirKey(index)) }) } for (const k of registry.keys) files.push({ key: at(undefined, k) }) // A leveled collection's mark and its levels' shards: the copy's own, written once. diff --git a/packages/protocol/src/repo/layouts/kit.ts b/packages/protocol/src/repo/layouts/kit.ts index bea9a55..a545332 100644 --- a/packages/protocol/src/repo/layouts/kit.ts +++ b/packages/protocol/src/repo/layouts/kit.ts @@ -87,8 +87,32 @@ export interface PackInfo { level?: number /** A pack of replaced versions a leveled merge moved out, by month written (`2026-10`). */ history?: string + /** + * The pack carries its own index (`packIndex`): its records' places, in a layer of + * its own, so a leveled collection finds what its levels hold without the index + * every copy used to need an entry in. + */ + indexed?: true } +/** The layer indexing a pack's records (`PackInfo.indexed`): named after the pack. */ +export const packIndexId = (pack: string) => `${pack}i` +export const packIndex = (p: PackInfo): LayerInfo => ({ + id: packIndexId(p.id), + level: 0, + seq: 0, + entries: p.records, + bytes: 0, + min: p.lo!, + max: p.hi!, +}) +/** The objects of a pack, with its index's when it has one. */ +export const packKeys = (p: { id: string; indexed?: true }) => [ + packKey(p.id), + packDirKey(p.id), + ...(p.indexed ? [layerKey(packIndexId(p.id)), layerDirKey(packIndexId(p.id))] : []), +] + /** The record type a key leads with (recordKey); '' for a key without one. */ const typeOf = (key: string) => { const i = key.indexOf('\u0000') @@ -115,6 +139,7 @@ export class PackWriter { #type = '' #lo = '' #hi = '' + #index: LayerWriter | null = null readonly done: PackInfo[] = [] readonly #writing = new Set>() #error: unknown = null @@ -128,6 +153,12 @@ export class PackWriter { history?: string /** Raw bytes a pack closes at (PACK_RAW). */ packRaw?: number + /** + * Write each pack's own index (`PackInfo.indexed`) beside it; a promise is read + * when the pack is written (the index is built as records come, then dropped + * if it turns out unwanted). + */ + index?: boolean | Promise } = {}, ) {} @@ -144,10 +175,12 @@ export class PackWriter { if (!this.#id) { this.#id = newId() this.#lo = key + if (this.opts.index) this.#index = new LayerWriter(this.store, 0, packIndexId(this.#id)) } this.#hi = key this.#lines.push(`${key}\t${line}`) const at = loc(this.#id, this.#blocks.length, this.#lines.length - 1) + this.#index?.add(key, at) this.#raw += key.length + line.length + 2 this.#records++ if (this.#raw >= BLOCK_RAW) this.#seal() @@ -192,7 +225,9 @@ export class PackWriter { type: this.#type, lo: this.#lo, hi: this.#hi, + index: this.#index, } + this.#index = null this.#id = '' this.#blocks = [] this.#records = 0 @@ -213,6 +248,7 @@ export class PackWriter { type: string lo: string hi: string + index: LayerWriter | null }) { const parts = await Promise.all(job.blocks) const offsets = [0] @@ -220,6 +256,9 @@ export class PackWriter { await this.opts.hooks?.before?.(job.id) await putJoined(this.store, packKey(job.id), parts, { ifAbsent: true }) await this.store.put(packDirKey(job.id), JSON.stringify(offsets), { ifAbsent: true }) + // Its index after the pack itself: a listed pack's index is always there. + const indexed = job.index !== null && (await this.opts.index) === true + if (indexed) await job.index!.finish() const pack: PackInfo = { id: job.id, tier: this.opts.tier ?? 0, @@ -231,6 +270,7 @@ export class PackWriter { hi: job.hi, ...(this.opts.level ? { level: this.opts.level } : {}), ...(this.opts.history ? { history: this.opts.history } : {}), + ...(indexed ? { indexed: true as const } : {}), } this.done.push(pack) await this.opts.hooks?.after?.(pack) @@ -531,7 +571,6 @@ function pageAt(dir: LayerDir, key: string): number { } export class LayerWriter { - readonly id = newId() #pages: Promise[] = [] #first: string[] = [] #lines: string[] = [] @@ -546,6 +585,7 @@ export class LayerWriter { constructor( readonly store: Store, readonly level: number, + readonly id = newId(), ) {} /** Add in key order (a repeated key keeps the first). */ @@ -880,6 +920,8 @@ export interface Manifest { * one's packs. `packs` above is level 0. */ levels?: Levels + /** A leveled collection's history level 0: replaced versions merges moved out, not yet merged down. */ + history?: PackInfo[] } const empty = (): Manifest => ({ seq: 0, layers: [], packs: [], retired: [], stats: {} }) diff --git a/packages/protocol/src/repo/layouts/levels.ts b/packages/protocol/src/repo/layouts/levels.ts index 7474a06..628f543 100644 --- a/packages/protocol/src/repo/layouts/levels.ts +++ b/packages/protocol/src/repo/layouts/levels.ts @@ -15,6 +15,7 @@ * key range touches. Readers never read them: records are found through the index. */ import { compareUtf8 } from '../../utf8.js' +import type { Lru } from '../lru.js' import type { Store } from '../types.js' import { newId, PACK_RAW, type PackInfo } from './kit.js' @@ -44,6 +45,11 @@ export const levelConfig = { packRaw: PACK_RAW, /** Packs a shard lists at most. */ shardPacks: 512, + /** + * Index entries that can no longer serve a lookup (Manifest.stats.indexStale) past + * which, once they're half the index, every run merges at once to drop them. + */ + staleEntries: 1_000_000, } /** A shard of a level's listing, as the manifest names it. */ @@ -57,8 +63,17 @@ export interface LevelShard { bytes: number } +/** + * By level key: `1`…`6` the current versions' levels, `h1`…`h6` the history's (replaced + * versions, every one kept, leveled the same way so an old version's lookup reads a + * pack a level, not one a month). + */ export type Levels = Record +/** The two trees a leveled collection keeps: current versions, and the history. */ +export type Tree = '' | 'h' +export const levelKey = (tree: Tree, level: number) => `${tree}${level}` + export const shardKey = (id: string) => `manifest/f/levels/${id}.json` /** Whether [aLo, aHi] and [bLo, bHi] share a key. */ @@ -68,8 +83,8 @@ export const overlaps = (aLo: string, aHi: string, bLo: string, bHi: string) => const byLo = (a: PackInfo, b: PackInfo) => compareUtf8(a.lo!, b.lo!) /** Raw bytes a level holds. */ -export const levelRaw = (levels: Levels | undefined, level: number) => - (levels?.[level] ?? []).reduce((n, s) => n + s.raw, 0) +export const levelRaw = (levels: Levels | undefined, level: number, tree: Tree = '') => + (levels?.[levelKey(tree, level)] ?? []).reduce((n, s) => n + s.raw, 0) /** * Each level's target size from the bottom up, and the base level, where level 0 @@ -78,9 +93,12 @@ export const levelRaw = (levels: Levels | undefined, level: number) => * 0's merges and goes down a pack at a time). Levels above the base have no room * (target 0). */ -export function levelTargets(levels: Levels | undefined): { base: number; targets: number[] } { +export function levelTargets( + levels: Levels | undefined, + tree: Tree = '', +): { base: number; targets: number[] } { const targets = Array.from({ length: MAX_LEVEL + 1 }).fill(0) - let t = levelRaw(levels, MAX_LEVEL) + let t = levelRaw(levels, MAX_LEVEL, tree) targets[MAX_LEVEL] = Infinity let base = MAX_LEVEL - 1 targets[base] = t / LEVEL_RATIO @@ -96,19 +114,34 @@ export function levelTargets(levels: Levels | undefined): { base: number; target /** The shards a leveled collection's levels are listed in (each read once: they never change). */ export class LevelShards { - readonly #read = new Map>() + readonly #loading = new Map>() + + /** Shards are written once: kept in the layout's memory cache, by id. */ + constructor( + readonly store: Store, + readonly lru: Lru, + ) {} - constructor(readonly store: Store) {} + #keep(id: string, packs: PackInfo[], bytes: number) { + this.lru.set(`shard:${id}`, { packs, approxBytes: bytes * 2 }) + } read(id: string): Promise { - let p = this.#read.get(id) + const hit = this.lru.get(`shard:${id}`) as { packs: PackInfo[] } | undefined + if (hit) return Promise.resolve(hit.packs) + let p = this.#loading.get(id) if (!p) { - p = this.store.get(shardKey(id)).then(async (o) => { - if (!o) throw new Error(`Missing level shard ${id}`) - return JSON.parse(await o.text()) as PackInfo[] - }) - p.catch(() => this.#read.delete(id)) - this.#read.set(id, p) + p = this.store + .get(shardKey(id)) + .then(async (o) => { + if (!o) throw new Error(`Missing level shard ${id}`) + const text = await o.text() + const packs = JSON.parse(text) as PackInfo[] + this.#keep(id, packs, text.length) + return packs + }) + .finally(() => this.#loading.delete(id)) + this.#loading.set(id, p) } return p } @@ -134,8 +167,9 @@ export class LevelShards { for (let i = 0; i < packs.length; i += levelConfig.shardPacks) { const ps = packs.slice(i, i + levelConfig.shardPacks) const id = newId() - this.#read.set(id, Promise.resolve(ps)) - await this.store.put(shardKey(id), JSON.stringify(ps), { ifAbsent: true }) + const body = JSON.stringify(ps) + await this.store.put(shardKey(id), body, { ifAbsent: true }) + this.#keep(id, ps, body.length) out.push({ id, lo: ps[0]!.lo!, diff --git a/packages/protocol/test/repo/leveled.test.ts b/packages/protocol/test/repo/leveled.test.ts index b598500..74f8ac0 100644 --- a/packages/protocol/test/repo/leveled.test.ts +++ b/packages/protocol/test/repo/leveled.test.ts @@ -30,6 +30,7 @@ beforeEach(() => { level0MergeRaw: 8_000, packRaw: 3_000, shardPacks: 3, + staleEntries: 50, }) }) afterEach(() => { @@ -53,21 +54,24 @@ const ids = Array.from( ).sort() const line = (id: string, v: number) => `${id} v${v} ${'x'.repeat(80)}` -/** Every level's packs, checked apart and in key order; and the level each says it's in. */ +/** + * Every level's packs by level key (`1`…, the history's `h1`…), checked apart and in + * key order; and the level each says it's in. + */ async function levelPacks(l: KeyedLayout) { const m = await l.manifests.latest() - const out = new Map() + const out = new Map() for (const [lv, shards] of Object.entries(m.levels ?? {})) { const ps = await l.levels.packs(shards) for (let i = 1; i < ps.length; i++) expect(compareUtf8(ps[i]!.lo!, ps[i - 1]!.hi!), `level ${lv} overlaps`).toBeGreaterThan(0) - for (const p of ps) expect(p.level).toBe(Number(lv)) + for (const p of ps) expect(p.level).toBe(Number(lv.replace('h', ''))) for (const s of shards) { const mine = await l.levels.read(s.id) expect(s.lo).toBe(mine[0]!.lo) expect(s.hi).toBe(mine.at(-1)!.hi) } - if (ps.length) out.set(Number(lv), ps) + if (ps.length) out.set(lv, ps) } return { m, levels: out } } @@ -128,7 +132,10 @@ describe('leveled packs', () => { expect(history.length).toBeGreaterThan(0) expect(history.every((p) => p.history === new Date().toISOString().slice(0, 7))).toBe(true) // The levels hold about one version an id: the newest, or one a merge hasn't met yet. - const inLevels = [...levels.values()].flat().reduce((n, p) => n + p.records, 0) + const inLevels = [...levels] + .filter(([lv]) => !lv.startsWith('h')) + .flatMap(([, ps]) => ps) + .reduce((n, p) => n + p.records, 0) expect(inLevels).toBeLessThan(current.size * 1.5) }) @@ -147,13 +154,13 @@ describe('leveled packs', () => { expect(m.packs).toEqual([]) expect( levels - .get(MAX_LEVEL)! + .get(String(MAX_LEVEL))! .map((p) => p.id) .sort(), ).toEqual(packs.map((p) => p.id).sort()) // A retry lists nothing again. await l.adoptPacks('load', packs) - expect((await levelPacks(l)).levels.get(MAX_LEVEL)!.length).toBe(packs.length) + expect((await levelPacks(l)).levels.get(String(MAX_LEVEL))!.length).toBe(packs.length) // An upload out of order into a collection holding data goes to level 0. const u = l.uploadWriter() for (const id of [ids[5]!, ids[1]!]) u.append({ type: 'T', id, hash: 'h2', line: line(id, 2) }) @@ -197,6 +204,104 @@ describe('leveled packs', () => { expect((await l.allPacks()).length).toBeLessThan(before) }) + it("keep the index to level 0, finding what the levels hold through each pack's own index", async () => { + const l = await leveledLayout() + const { all } = await commits(l, 40) + const { m, levels } = await levelPacks(l) + expect([...levels.values()].flat().every((p) => p.indexed)).toBe(true) + // Tiered, the index has an entry a copy; here only what level 0 held since the last index merge. + expect(m.layers.reduce((n, x) => n + x.entries, 0)).toBeLessThan(all.length / 4) + // The history is leveled too. + expect([...levels.keys()].some((k) => k.startsWith('h'))).toBe(true) + await readAll(l, all) + }) + + it('reclaim copies a mostly-dead level pack within its level', async () => { + const l = await leveledLayout() + const { current, all } = await commits(l, 40) + // The current versions and every third older one: history packs end up under half live. + const kept = all.filter(([id, v], i) => current.get(id) === v || i % 3 === 0) + const stats = await l.reclaim({ + records: kept.map(([id, v]) => `T\u0000${id}\u0000h${v}`), + } as never) + expect(stats.recordsDropped).toBe(all.length - kept.length) + expect(stats.packsRewritten).toBeGreaterThan(0) + expect((await l.manifests.latest()).stats.levelRewrites).toBe(stats.packsRewritten) + while (await l.maintain()); + await levelPacks(l) + await readAll(l, kept) + const records = (await l.allPacks()).reduce((n, p) => n + p.records, 0) + expect(records).toBeLessThan(all.length) + expect(records).toBeGreaterThanOrEqual(kept.length) + }) + + it('keep the index entries of an upload not yet adopted through index merges', async () => { + const l = await leveledLayout() + await commits(l, 10) + // An upload's packs are written, and its commit's entries indexed, before the packs are listed. + const fresh = ids.slice(0, 40).map((id) => `${id}u`) + const w = l.uploadWriter() + const places = fresh.map((id) => w.append({ type: 'T', id, hash: 'h1', line: line(id, 1) })) + const packs = await w.finish() + const before = new Set((await l.manifests.latest()).layers.map((x) => x.id)) + const cw = l.writer() + fresh.forEach((id, i) => cw.addPlaced('T', { key: id, hash: 'h1', size: 10 }, places[i]!)) + await cw.finish() + const run = (await l.manifests.latest()).layers.filter((x) => !before.has(x.id)) + expect(run.length).toBeGreaterThan(0) + // Index merges meanwhile take that run in (once the runs after it add up to its size). + await commits(l, 30, 3) + const now = new Set((await l.manifests.latest()).layers.map((x) => x.id)) + expect(run.some((x) => now.has(x.id))).toBe(false) + await l.adoptPacks('late', packs) + // In level 0 now, found through the index (before merges carry them down, after which + // their packs' own indexes would find them whatever the index held). + const versions = fresh.map((id) => [id, 1] as [string, number]) + await readAll(l, versions) + while (await l.maintain()); + await readAll(l, versions) + }) + + it('sweep a pack that went down only once no index run names it', async () => { + const store = memoryStore() + const l = await leveledLayout(store) + // No merge of every run yet: the index may still name packs level 0 merged down. + levelConfig.staleEntries = 1e9 + await commits(l, 12) + const m = await l.manifests.latest() + expect(m.stats.levelMerges).toBeGreaterThan(0) + const held = m.retired.filter((r) => r.key.endsWith('.pk')) + expect(held.length).toBeGreaterThan(0) + for (const r of held) expect(await store.head(`runs/c1/${r.key}`)).not.toBeNull() + // Once one runs, they go. + levelConfig.staleEntries = 50 + while (await l.maintain()); + await l.maintain() + const after = await l.manifests.latest() + expect(after.stats.indexSettledAt).toBeGreaterThan(0) + for (const r of held) expect(await store.head(`runs/c1/${r.key}`)).toBeNull() + }) + + it('merge away the index entries of a sorted load once it is the bottom level', async () => { + const l = await leveledLayout() + const w = l.uploadWriter() + const places = ids.map((id) => w.append({ type: 'T', id, hash: 'h1', line: line(id, 1) })) + const packs = await w.finish() + const cw = l.writer() + ids.forEach((id, i) => cw.addPlaced('T', { key: id, hash: 'h1', size: 10 }, places[i]!)) + await cw.finish() + await l.adoptPacks('load', packs) + expect((await l.manifests.latest()).stats.indexStale).toBe(ids.length) + await l.maintain() + const m = await l.manifests.latest() + expect(m.layers.reduce((n, x) => n + x.entries, 0)).toBe(0) + expect(m.stats.indexStale).toBe(0) + await readAll( + l, + ids.map((id) => [id, 1] as [string, number]), + ) + }) + it('fork as leveled, the fork reading its parent and its merges leaving the parent alone', async () => { const root = memoryStore() const parent = await leveledLayout(root, 'p') diff --git a/packages/server/drizzle/0029_push_pack_indexed.sql b/packages/server/drizzle/0029_push_pack_indexed.sql new file mode 100644 index 0000000..39a8fc6 --- /dev/null +++ b/packages/server/drizzle/0029_push_pack_indexed.sql @@ -0,0 +1 @@ +ALTER TABLE `push_packs` ADD `indexed` integer; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0029_snapshot.json b/packages/server/drizzle/meta/0029_snapshot.json new file mode 100644 index 0000000..215f53f --- /dev/null +++ b/packages/server/drizzle/meta/0029_snapshot.json @@ -0,0 +1,3941 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "b00f6d34-a259-47af-bd33-7495e141fa13", + "prevId": "715b9c5b-f4b6-4fa6-8d5a-b7a936a3e6d9", + "tables": { + "abuse_reports": { + "name": "abuse_reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "contact": { + "name": "contact", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reporter_id": { + "name": "reporter_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "resolved_by": { + "name": "resolved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "abuse_reports_status_idx": { + "name": "abuse_reports_status_idx", + "columns": [ + "status", + "created_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 86400000 + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 10 + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_request": { + "name": "last_request", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": [ + "key" + ], + "isUnique": false + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": [ + "reference_id" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_collections": { + "name": "ark_collections", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "ark_id": { + "name": "ark_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "custom_url": { + "name": "custom_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_collections_ark_id_unique": { + "name": "ark_collections_ark_id_unique", + "columns": [ + "ark_id" + ], + "isUnique": true + } + }, + "foreignKeys": { + "ark_collections_collection_id_collections_id_fk": { + "name": "ark_collections_collection_id_collections_id_fk", + "tableFrom": "ark_collections", + "tableTo": "collections", + "columnsFrom": [ + "collection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_record_types": { + "name": "ark_record_types", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirect_url_field": { + "name": "redirect_url_field", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "ark_record_types_collection_id_collections_id_fk": { + "name": "ark_record_types_collection_id_collections_id_fk", + "tableFrom": "ark_record_types", + "tableTo": "collections", + "columnsFrom": [ + "collection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "ark_record_types_collection_id_record_type_pk": { + "columns": [ + "collection_id", + "record_type" + ], + "name": "ark_record_types_collection_id_record_type_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_shoulders": { + "name": "ark_shoulders", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "shoulder": { + "name": "shoulder", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_shoulders_shoulder_unique": { + "name": "ark_shoulders_shoulder_unique", + "columns": [ + "shoulder" + ], + "isUnique": true + } + }, + "foreignKeys": { + "ark_shoulders_organization_id_organization_id_fk": { + "name": "ark_shoulders_organization_id_organization_id_fk", + "tableFrom": "ark_shoulders", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cleanup_runs": { + "name": "cleanup_runs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "step": { + "name": "step", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dry_run": { + "name": "dry_run", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "requested_by": { + "name": "requested_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mark_run_id": { + "name": "mark_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lease": { + "name": "lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stats": { + "name": "stats", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "cleanup_runs_created_idx": { + "name": "cleanup_runs_created_idx", + "columns": [ + "created_at" + ], + "isUnique": false + }, + "cleanup_runs_step_idx": { + "name": "cleanup_runs_step_idx", + "columns": [ + "step", + "status" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_tombstones": { + "name": "collection_tombstones", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "versions": { + "name": "versions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_by": { + "name": "deleted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_layout": { + "name": "record_layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'today'" + }, + "records_dropped_at": { + "name": "records_dropped_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_webhooks": { + "name": "collection_webhooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bump_filter": { + "name": "bump_filter", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_delivery_at": { + "name": "last_delivery_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "collection_webhooks_collection_idx": { + "name": "collection_webhooks_collection_idx", + "columns": [ + "collection_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "collection_webhooks_collection_id_collections_id_fk": { + "name": "collection_webhooks_collection_id_collections_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "collections", + "columnsFrom": [ + "collection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collections": { + "name": "collections", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "head_version_id": { + "name": "head_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_salt": { + "name": "private_salt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_files_root": { + "name": "public_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "version_count": { + "name": "version_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "history_bytes": { + "name": "history_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_push_at": { + "name": "last_push_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_started_at": { + "name": "reconcile_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_state": { + "name": "reconcile_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_seq": { + "name": "reconciled_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "reconciled_files_root": { + "name": "reconciled_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_layout": { + "name": "record_layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'today'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "collections_org_slug_uq": { + "name": "collections_org_slug_uq", + "columns": [ + "organization_id", + "slug" + ], + "isUnique": true + }, + "collections_public_updated_idx": { + "name": "collections_public_updated_idx", + "columns": [ + "public", + "updated_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "collections_organization_id_organization_id_fk": { + "name": "collections_organization_id_organization_id_fk", + "tableFrom": "collections", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "commit_units": { + "name": "commit_units", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "plan_id": { + "name": "plan_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ord": { + "name": "ord", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "after": { + "name": "after", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "through": { + "name": "through", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "gap": { + "name": "gap", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "slices_key": { + "name": "slices_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_key": { + "name": "output_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "queued_at": { + "name": "queued_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "commit_units_plan_idx": { + "name": "commit_units_plan_idx", + "columns": [ + "plan_id", + "set", + "type", + "ord" + ], + "isUnique": false + }, + "commit_units_status_idx": { + "name": "commit_units_status_idx", + "columns": [ + "session_id", + "status", + "queued_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "commit_units_session_id_push_sessions_id_fk": { + "name": "commit_units_session_id_push_sessions_id_fk", + "tableFrom": "commit_units", + "tableTo": "push_sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "denylist": { + "name": "denylist", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "file_uploads": { + "name": "file_uploads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "multipart_upload_id": { + "name": "multipart_upload_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "file_uploads_hash_idx": { + "name": "file_uploads_hash_idx", + "columns": [ + "hash" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "files": { + "name": "files", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "forks": { + "name": "forks", + "columns": { + "child_collection_id": { + "name": "child_collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "parent_collection_id": { + "name": "parent_collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_seq": { + "name": "parent_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'public'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "forks_child_collection_id_collections_id_fk": { + "name": "forks_child_collection_id_collections_id_fk", + "tableFrom": "forks", + "tableTo": "collections", + "columnsFrom": [ + "child_collection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instance_settings": { + "name": "instance_settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invitation": { + "name": "invitation", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": [ + "organization_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": [ + "inviter_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "jobs": { + "name": "jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "run_at": { + "name": "run_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locked_until": { + "name": "locked_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "jobs_ready_idx": { + "name": "jobs_ready_idx", + "columns": [ + "status", + "run_at" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "member": { + "name": "member", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": [ + "organization_id" + ], + "isUnique": false + }, + "member_user_id_idx": { + "name": "member_user_id_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organization": { + "name": "organization", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ark_naan": { + "name": "ark_naan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kf_org_id": { + "name": "kf_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "columns": [ + "slug" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "placements": { + "name": "placements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location_id": { + "name": "location_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "synced_seq": { + "name": "synced_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until": { + "name": "lease_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "placements_one_primary_uq": { + "name": "placements_one_primary_uq", + "columns": [ + "collection_id" + ], + "isUnique": true, + "where": "\"placements\".\"role\" = 'primary' AND \"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_collection_location_uq": { + "name": "placements_collection_location_uq", + "columns": [ + "collection_id", + "location_id" + ], + "isUnique": true, + "where": "\"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_org_location_uq": { + "name": "placements_org_location_uq", + "columns": [ + "organization_id", + "location_id" + ], + "isUnique": true, + "where": "\"placements\".\"organization_id\" IS NOT NULL" + }, + "placements_location_idx": { + "name": "placements_location_idx", + "columns": [ + "location_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "placements_collection_id_collections_id_fk": { + "name": "placements_collection_id_collections_id_fk", + "tableFrom": "placements", + "tableTo": "collections", + "columnsFrom": [ + "collection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_organization_id_organization_id_fk": { + "name": "placements_organization_id_organization_id_fk", + "tableFrom": "placements", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_location_id_storage_locations_id_fk": { + "name": "placements_location_id_storage_locations_id_fk", + "tableFrom": "placements", + "tableTo": "storage_locations", + "columnsFrom": [ + "location_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_packs": { + "name": "push_packs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pack_id": { + "name": "pack_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_set": { + "name": "record_set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "records": { + "name": "records", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw": { + "name": "raw", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lo_key": { + "name": "lo_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hi_key": { + "name": "hi_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "indexed": { + "name": "indexed", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "push_packs_session_id_push_sessions_id_fk": { + "name": "push_packs_session_id_push_sessions_id_fk", + "tableFrom": "push_packs", + "tableTo": "push_sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_packs_session_id_pack_id_pk": { + "columns": [ + "session_id", + "pack_id" + ], + "name": "push_packs_session_id_pack_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_runs": { + "name": "push_runs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "first_key": { + "name": "first_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_key": { + "name": "last_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "merging_into": { + "name": "merging_into", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "push_runs_tier_idx": { + "name": "push_runs_tier_idx", + "columns": [ + "session_id", + "tier", + "merging_into" + ], + "isUnique": false + } + }, + "foreignKeys": { + "push_runs_session_id_push_sessions_id_fk": { + "name": "push_runs_session_id_push_sessions_id_fk", + "tableFrom": "push_runs", + "tableTo": "push_sessions", + "columnsFrom": [ + "session_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_runs_session_id_seq_pk": { + "columns": [ + "session_id", + "seq" + ], + "name": "push_runs_session_id_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_sessions": { + "name": "push_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_version_id": { + "name": "base_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "strip_unknown_fields": { + "name": "strip_unknown_fields", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "records_received": { + "name": "records_received", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "runs": { + "name": "runs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "result": { + "name": "result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finalize_started_at": { + "name": "finalize_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_started": { + "name": "commit_started", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "in_key_order": { + "name": "in_key_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "overlapping_uploads": { + "name": "overlapping_uploads", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cleaned_at": { + "name": "cleaned_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_phase": { + "name": "commit_phase", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "progress_at": { + "name": "progress_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stalls": { + "name": "stalls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "commit_plan": { + "name": "commit_plan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "assembly_lease": { + "name": "assembly_lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "push_sessions_collection_idx": { + "name": "push_sessions_collection_idx", + "columns": [ + "collection_id" + ], + "isUnique": false + }, + "push_sessions_expires_idx": { + "name": "push_sessions_expires_idx", + "columns": [ + "status", + "expires_at" + ], + "isUnique": false + }, + "push_sessions_user_idx": { + "name": "push_sessions_user_idx", + "columns": [ + "user_id", + "status" + ], + "isUnique": false + } + }, + "foreignKeys": { + "push_sessions_collection_id_collections_id_fk": { + "name": "push_sessions_collection_id_collections_id_fk", + "tableFrom": "push_sessions", + "tableTo": "collections", + "columnsFrom": [ + "collection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compaction_parts": { + "name": "ref_compaction_parts", + "columns": { + "compaction_id": { + "name": "compaction_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "part": { + "name": "part", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_compaction_parts_compaction_id_part_pk": { + "columns": [ + "compaction_id", + "part" + ], + "name": "ref_compaction_parts_compaction_id_part_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compactions": { + "name": "ref_compactions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_runs": { + "name": "input_runs", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output_run": { + "name": "output_run", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parts": { + "name": "parts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix_length": { + "name": "prefix_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'running'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_index_units": { + "name": "ref_index_units", + "columns": { + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unit": { + "name": "unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "events": { + "name": "events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_index_units_version_id_unit_pk": { + "columns": [ + "version_id", + "unit" + ], + "name": "ref_index_units_version_id_unit_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_segments": { + "name": "ref_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_hash": { + "name": "first_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_hash": { + "name": "last_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'live'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ref_segments_range_idx": { + "name": "ref_segments_range_idx", + "columns": [ + "state", + "first_hash", + "last_hash" + ], + "isUnique": false + }, + "ref_segments_run_idx": { + "name": "ref_segments_run_idx", + "columns": [ + "run_id" + ], + "isUnique": false + }, + "ref_segments_tier_idx": { + "name": "ref_segments_tier_idx", + "columns": [ + "tier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_labels": { + "name": "schema_labels", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "schema_labels_label_idx": { + "name": "schema_labels_label_idx", + "columns": [ + "label" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "schema_labels_schema_hash_label_pk": { + "columns": [ + "schema_hash", + "label" + ], + "name": "schema_labels_schema_hash_label_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_usage": { + "name": "schema_usage", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type_slug": { + "name": "type_slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_seq": { + "name": "from_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "to_seq": { + "name": "to_seq", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "schema_usage_hash_idx": { + "name": "schema_usage_hash_idx", + "columns": [ + "schema_hash" + ], + "isUnique": false + } + }, + "foreignKeys": { + "schema_usage_collection_id_collections_id_fk": { + "name": "schema_usage_collection_id_collections_id_fk", + "tableFrom": "schema_usage", + "tableTo": "collections", + "columnsFrom": [ + "collection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "schema_usage_collection_id_type_slug_set_from_seq_pk": { + "columns": [ + "collection_id", + "type_slug", + "set", + "from_seq" + ], + "name": "schema_usage_collection_id_type_slug_set_from_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schemas": { + "name": "schemas", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": [ + "token" + ], + "isUnique": true + }, + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": [ + "user_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": [ + "user_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_fence": { + "name": "storage_fence", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "epoch": { + "name": "epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "window_until": { + "name": "window_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "window_run_id": { + "name": "window_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_locations": { + "name": "storage_locations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucket": { + "name": "bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "credentials": { + "name": "credentials", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unverified'" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checked_at": { + "name": "checked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "storage_locations_organization_id_organization_id_fk": { + "name": "storage_locations_organization_id_organization_id_fk", + "tableFrom": "storage_locations", + "tableTo": "organization", + "columnsFrom": [ + "organization_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "usage_rollups": { + "name": "usage_rollups", + "columns": { + "day": { + "name": "day", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metric": { + "name": "metric", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "amount": { + "name": "amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "usage_rollups_account_idx": { + "name": "usage_rollups_account_idx", + "columns": [ + "account_id", + "day" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "usage_rollups_day_account_id_collection_id_metric_pk": { + "columns": [ + "day", + "account_id", + "collection_id", + "metric" + ], + "name": "usage_rollups_day_account_id_collection_id_metric_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": [ + "email" + ], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": [ + "identifier" + ], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "versions": { + "name": "versions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "major": { + "name": "major", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "minor": { + "name": "minor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "patch": { + "name": "patch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pushed_by": { + "name": "pushed_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_record_count": { + "name": "public_record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_file_count": { + "name": "public_file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "public_total_bytes": { + "name": "public_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "type_counts": { + "name": "type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_type_counts": { + "name": "public_type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "has_private": { + "name": "has_private", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "public_refs_root": { + "name": "public_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_refs_root": { + "name": "private_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refs_indexed": { + "name": "refs_indexed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "changes": { + "name": "changes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "push_session_id": { + "name": "push_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "versions_collection_seq_uq": { + "name": "versions_collection_seq_uq", + "columns": [ + "collection_id", + "seq" + ], + "isUnique": true + }, + "versions_collection_semver_uq": { + "name": "versions_collection_semver_uq", + "columns": [ + "collection_id", + "semver" + ], + "isUnique": true + }, + "versions_hash_idx": { + "name": "versions_hash_idx", + "columns": [ + "hash" + ], + "isUnique": false + }, + "versions_published_idx": { + "name": "versions_published_idx", + "columns": [ + "published_at" + ], + "isUnique": false + }, + "versions_push_session_idx": { + "name": "versions_push_session_idx", + "columns": [ + "push_session_id" + ], + "isUnique": false + } + }, + "foreignKeys": { + "versions_collection_id_collections_id_fk": { + "name": "versions_collection_id_collections_id_fk", + "tableFrom": "versions", + "tableTo": "collections", + "columnsFrom": [ + "collection_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhook_deliveries": { + "name": "webhook_deliveries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "webhook_id": { + "name": "webhook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bump_type": { + "name": "bump_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'version.created'" + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "delivered_at": { + "name": "delivered_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhook_deliveries_webhook_idx": { + "name": "webhook_deliveries_webhook_idx", + "columns": [ + "webhook_id", + "created_at" + ], + "isUnique": false + }, + "webhook_deliveries_pending_idx": { + "name": "webhook_deliveries_pending_idx", + "columns": [ + "status", + "next_attempt_at" + ], + "isUnique": false + } + }, + "foreignKeys": { + "webhook_deliveries_webhook_id_collection_webhooks_id_fk": { + "name": "webhook_deliveries_webhook_id_collection_webhooks_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collection_webhooks", + "columnsFrom": [ + "webhook_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} \ No newline at end of file diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index 11a6e3d..23fec22 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -204,6 +204,13 @@ "when": 1792606671953, "tag": "0028_push_pack_ranges", "breakpoints": true + }, + { + "idx": 29, + "version": "6", + "when": 1792609221259, + "tag": "0029_push_pack_indexed", + "breakpoints": true } ] -} +} \ No newline at end of file diff --git a/packages/server/src/cleanup/layout-reclaim.ts b/packages/server/src/cleanup/layout-reclaim.ts index 4430b1c..a0beb33 100644 --- a/packages/server/src/cleanup/layout-reclaim.ts +++ b/packages/server/src/cleanup/layout-reclaim.ts @@ -6,7 +6,8 @@ * range one job a range: mark the range's live records over every retained * version, keep the index entries they need; the last range done * swaps the new index in, drops dead packs, queues the rewrites - * rewrite a few mostly-dead packs a job: copy their live records out + * rewrite a few mostly-dead packs a job: copy their live records out (a leveled + * collection's level packs in place, their range marked again) * * Nothing starts it yet: records only go dead when versions are thinned (a * deleted collection's files go whole, cleanup/sweep.ts), and nothing may commit to @@ -20,6 +21,7 @@ import { type Repo, recordRoots, TooManyRunsError, + type TypedRoot, } from '@underlay/protocol' import { eq } from 'drizzle-orm' @@ -111,9 +113,16 @@ registerJob('layout.reclaim', async (job, ports) => { await ports.jobs.enqueueBatch(jobs) return } - case 'rewrite': - await layout.reclaimRewrite(String(job.packs).split(',')) + case 'rewrite': { + // A level's pack has no index entries to say what's live: marked again over its + // key range (the retained versions read once a job). + let roots: Promise | undefined + await layout.reclaimRewrite(String(job.packs).split(','), async (r) => { + roots ??= retained(ports, t).then((v) => recordRoots(repo, v)) + return (await markRange(repo, await roots, r)).live + }) return + } default: throw new Error(`layout.reclaim: unknown step ${String(job.step)}`) } diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index 6ec7161..2f4d58c 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -708,6 +708,8 @@ export const pushPacks = sqliteTable( */ loKey: text('lo_key'), hiKey: text('hi_key'), + /** Whether it carries its own index (PackInfo.indexed): a leveled collection's do. */ + indexed: integer('indexed', { mode: 'boolean' }), createdAt: createdAt(), }, (t) => [primaryKey({ columns: [t.sessionId, t.packId] })], diff --git a/packages/server/src/push/packs.ts b/packages/server/src/push/packs.ts index c3473d5..e3befff 100644 --- a/packages/server/src/push/packs.ts +++ b/packages/server/src/push/packs.ts @@ -54,6 +54,7 @@ const hooksFor = (ports: Ports, sessionId: string, set: RecordSet): PackHooks => recordType: pack.type, loKey: keyText(pack.lo), hiKey: keyText(pack.hi), + indexed: pack.indexed ?? false, }) .where(and(eq(schema.pushPacks.sessionId, sessionId), eq(schema.pushPacks.packId, pack.id))) }, @@ -146,6 +147,7 @@ export async function adoptSessionPacks(ports: Ports, repo: Repo, sessionId: str ...(r.loKey && r.hiKey ? { lo: JSON.parse(r.loKey) as string, hi: JSON.parse(r.hiKey) as string } : {}), + ...(r.indexed ? { indexed: true as const } : {}), })) if (packs.length) await repo.records[set].adoptPacks!(sessionId, packs) } diff --git a/packages/server/test/cleanup.test.ts b/packages/server/test/cleanup.test.ts index d70ea84..07a7a36 100644 --- a/packages/server/test/cleanup.test.ts +++ b/packages/server/test/cleanup.test.ts @@ -308,8 +308,9 @@ describe('storage cleanup, steps 2 and 3: mark and sweep', () => { const fresh = await h.collection('new') const puts = h.bucket.puts await push(h, user, 'new', records, 'the pdf') - // Mostly reused (leveled, the new collection's mark is one more). - expect(h.bucket.puts - puts).toBeLessThan(layoutConfig.levelNew ? 17 : 15) + // Mostly reused (leveled, the new collection's mark is one more, and the upload's + // pack carries its own index: two). + expect(h.bucket.puts - puts).toBeLessThan(layoutConfig.levelNew ? 19 : 15) ageAll(h) const sweep = await run(h, 'sweep') From 22c89cc0da517a1593d52d3a353cae723b2f67fc Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 21 Oct 2026 21:27:48 +0200 Subject: [PATCH 2/3] Layout F index split: lookups skip retired packs' entries, a lower stale floor --- packages/protocol/src/repo/layouts/keyed.ts | 37 +++++++++++++++----- packages/protocol/src/repo/layouts/levels.ts | 6 ++-- 2 files changed, 32 insertions(+), 11 deletions(-) diff --git a/packages/protocol/src/repo/layouts/keyed.ts b/packages/protocol/src/repo/layouts/keyed.ts index bb18b51..592a6c7 100644 --- a/packages/protocol/src/repo/layouts/keyed.ts +++ b/packages/protocol/src/repo/layouts/keyed.ts @@ -314,6 +314,19 @@ export class KeyedLayout implements RecordStore { /** Packs a leveled lookup found swept (#leafLines): their index entries are skipped. */ readonly #gone = new Set() + readonly #retired = new WeakMap>() + + /** The packs `m` lists as retired, by id. */ + #retiredPacks(m: Manifest): Set { + let out = this.#retired.get(m) + if (!out) { + out = new Set() + for (const r of m.retired) + if (r.key.startsWith('packs/') && r.key.endsWith('.pk')) out.add(r.key.slice(6, -3)) + this.#retired.set(m, out) + } + return out + } /** Every pack of the levels (both trees'), by level key. */ async #levelPacks(m: Manifest): Promise> { @@ -364,15 +377,20 @@ export class KeyedLayout implements RecordStore { * holds at most a pack or two of a leaf's range, so a leaf reads a few directories * and pages a level, however many days of changes went in. * - * The index also has entries of packs that left level 0 (merged down, or a sorted - * load adopted as the bottom level) until its runs merge: a place stays good while - * its pack is there (any copy of a key is the same record), and one whose pack was - * swept is skipped (#leafLines). + * The index also has entries of packs that left level 0 until its runs merge. One + * whose pack is retired (merged down: kept, and listed, until no run names it) is + * skipped, its record read where it is now; a sorted load's, adopted as the bottom + * level, is good until a merge rewrites that pack (any copy of a key is the same + * record). One whose pack is gone anyway (a head older than the sweep) is skipped + * once its read fails (#leafLines). */ async #findLeveled(m: Manifest, keys: readonly string[]): Promise> { + const retired = this.#retiredPacks(m) const found = new Map() - for (const [k, v] of await this.index.values(m, keys)) - if (!this.#gone.has(parseLoc(v).pack)) found.set(k, v) + for (const [k, v] of await this.index.values(m, keys)) { + const pack = parseLoc(v).pack + if (!retired.has(pack) && !this.#gone.has(pack)) found.set(k, v) + } let left = [...new Set(keys)].filter((k) => !found.has(k)).sort(compareUtf8) const probe = async (packs: PackInfo[]) => { const got = await Promise.all( @@ -801,9 +819,10 @@ export class KeyedLayout implements RecordStore { async #indexKeeper(m: Manifest): Promise<((place: string) => Promise) | null> { if (!m.levels && !m.history) return null const zero = new Set(m.packs.map((p) => p.id)) - const left = new Set((await this.#leveledPacks(m)).map((p) => p.id)) - for (const r of m.retired) - if (r.key.startsWith('packs/') && r.key.endsWith('.pk')) left.add(r.key.slice(6, -3)) + const left = new Set([ + ...(await this.#leveledPacks(m)).map((p) => p.id), + ...this.#retiredPacks(m), + ]) const known = new Map>() return (place: string) => { const pack = parseLoc(place).pack diff --git a/packages/protocol/src/repo/layouts/levels.ts b/packages/protocol/src/repo/layouts/levels.ts index 628f543..48f1464 100644 --- a/packages/protocol/src/repo/layouts/levels.ts +++ b/packages/protocol/src/repo/layouts/levels.ts @@ -47,9 +47,11 @@ export const levelConfig = { shardPacks: 512, /** * Index entries that can no longer serve a lookup (Manifest.stats.indexStale) past - * which, once they're half the index, every run merges at once to drop them. + * which, once they're half the index, every run merges at once to drop them. The + * index is about level 0 and these, so that merge is small (50k entries: ~3 MB); + * until it runs, a lookup reads their runs for nothing, and merged-down packs stay. */ - staleEntries: 1_000_000, + staleEntries: 50_000, } /** A shard of a level's listing, as the manifest names it. */ From 178ccd07b34e323fcad69e96b6257572dc402b62 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 21 Oct 2026 21:29:26 +0200 Subject: [PATCH 3/3] Format the 0029 migration's drizzle files --- .../server/drizzle/meta/0029_snapshot.json | 429 ++++-------------- packages/server/drizzle/meta/_journal.json | 2 +- 2 files changed, 100 insertions(+), 331 deletions(-) diff --git a/packages/server/drizzle/meta/0029_snapshot.json b/packages/server/drizzle/meta/0029_snapshot.json index 215f53f..022394f 100644 --- a/packages/server/drizzle/meta/0029_snapshot.json +++ b/packages/server/drizzle/meta/0029_snapshot.json @@ -83,10 +83,7 @@ "indexes": { "abuse_reports_status_idx": { "name": "abuse_reports_status_idx", - "columns": [ - "status", - "created_at" - ], + "columns": ["status", "created_at"], "isUnique": false } }, @@ -194,9 +191,7 @@ "indexes": { "account_user_id_idx": { "name": "account_user_id_idx", - "columns": [ - "user_id" - ], + "columns": ["user_id"], "isUnique": false } }, @@ -205,12 +200,8 @@ "name": "account_user_id_user_id_fk", "tableFrom": "account", "tableTo": "user", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["user_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -387,16 +378,12 @@ "indexes": { "apikey_key_idx": { "name": "apikey_key_idx", - "columns": [ - "key" - ], + "columns": ["key"], "isUnique": false }, "apikey_reference_id_idx": { "name": "apikey_reference_id_idx", - "columns": [ - "reference_id" - ], + "columns": ["reference_id"], "isUnique": false } }, @@ -449,9 +436,7 @@ "indexes": { "ark_collections_ark_id_unique": { "name": "ark_collections_ark_id_unique", - "columns": [ - "ark_id" - ], + "columns": ["ark_id"], "isUnique": true } }, @@ -460,12 +445,8 @@ "name": "ark_collections_collection_id_collections_id_fk", "tableFrom": "ark_collections", "tableTo": "collections", - "columnsFrom": [ - "collection_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -505,22 +486,15 @@ "name": "ark_record_types_collection_id_collections_id_fk", "tableFrom": "ark_record_types", "tableTo": "collections", - "columnsFrom": [ - "collection_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } }, "compositePrimaryKeys": { "ark_record_types_collection_id_record_type_pk": { - "columns": [ - "collection_id", - "record_type" - ], + "columns": ["collection_id", "record_type"], "name": "ark_record_types_collection_id_record_type_pk" } }, @@ -563,9 +537,7 @@ "indexes": { "ark_shoulders_shoulder_unique": { "name": "ark_shoulders_shoulder_unique", - "columns": [ - "shoulder" - ], + "columns": ["shoulder"], "isUnique": true } }, @@ -574,12 +546,8 @@ "name": "ark_shoulders_organization_id_organization_id_fk", "tableFrom": "ark_shoulders", "tableTo": "organization", - "columnsFrom": [ - "organization_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -711,17 +679,12 @@ "indexes": { "cleanup_runs_created_idx": { "name": "cleanup_runs_created_idx", - "columns": [ - "created_at" - ], + "columns": ["created_at"], "isUnique": false }, "cleanup_runs_step_idx": { "name": "cleanup_runs_step_idx", - "columns": [ - "step", - "status" - ], + "columns": ["step", "status"], "isUnique": false } }, @@ -897,9 +860,7 @@ "indexes": { "collection_webhooks_collection_idx": { "name": "collection_webhooks_collection_idx", - "columns": [ - "collection_id" - ], + "columns": ["collection_id"], "isUnique": false } }, @@ -908,12 +869,8 @@ "name": "collection_webhooks_collection_id_collections_id_fk", "tableFrom": "collection_webhooks", "tableTo": "collections", - "columnsFrom": [ - "collection_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1105,18 +1062,12 @@ "indexes": { "collections_org_slug_uq": { "name": "collections_org_slug_uq", - "columns": [ - "organization_id", - "slug" - ], + "columns": ["organization_id", "slug"], "isUnique": true }, "collections_public_updated_idx": { "name": "collections_public_updated_idx", - "columns": [ - "public", - "updated_at" - ], + "columns": ["public", "updated_at"], "isUnique": false } }, @@ -1125,12 +1076,8 @@ "name": "collections_organization_id_organization_id_fk", "tableFrom": "collections", "tableTo": "organization", - "columnsFrom": [ - "organization_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1255,21 +1202,12 @@ "indexes": { "commit_units_plan_idx": { "name": "commit_units_plan_idx", - "columns": [ - "plan_id", - "set", - "type", - "ord" - ], + "columns": ["plan_id", "set", "type", "ord"], "isUnique": false }, "commit_units_status_idx": { "name": "commit_units_status_idx", - "columns": [ - "session_id", - "status", - "queued_at" - ], + "columns": ["session_id", "status", "queued_at"], "isUnique": false } }, @@ -1278,12 +1216,8 @@ "name": "commit_units_session_id_push_sessions_id_fk", "tableFrom": "commit_units", "tableTo": "push_sessions", - "columnsFrom": [ - "session_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["session_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1417,9 +1351,7 @@ "indexes": { "file_uploads_hash_idx": { "name": "file_uploads_hash_idx", - "columns": [ - "hash" - ], + "columns": ["hash"], "isUnique": false } }, @@ -1528,12 +1460,8 @@ "name": "forks_child_collection_id_collections_id_fk", "tableFrom": "forks", "tableTo": "collections", - "columnsFrom": [ - "child_collection_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["child_collection_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1638,9 +1566,7 @@ "indexes": { "invitation_organization_id_idx": { "name": "invitation_organization_id_idx", - "columns": [ - "organization_id" - ], + "columns": ["organization_id"], "isUnique": false } }, @@ -1649,12 +1575,8 @@ "name": "invitation_organization_id_organization_id_fk", "tableFrom": "invitation", "tableTo": "organization", - "columnsFrom": [ - "organization_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -1662,12 +1584,8 @@ "name": "invitation_inviter_id_user_id_fk", "tableFrom": "invitation", "tableTo": "user", - "columnsFrom": [ - "inviter_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1749,10 +1667,7 @@ "indexes": { "jobs_ready_idx": { "name": "jobs_ready_idx", - "columns": [ - "status", - "run_at" - ], + "columns": ["status", "run_at"], "isUnique": false } }, @@ -1805,16 +1720,12 @@ "indexes": { "member_organization_id_idx": { "name": "member_organization_id_idx", - "columns": [ - "organization_id" - ], + "columns": ["organization_id"], "isUnique": false }, "member_user_id_idx": { "name": "member_user_id_idx", - "columns": [ - "user_id" - ], + "columns": ["user_id"], "isUnique": false } }, @@ -1823,12 +1734,8 @@ "name": "member_organization_id_organization_id_fk", "tableFrom": "member", "tableTo": "organization", - "columnsFrom": [ - "organization_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -1836,12 +1743,8 @@ "name": "member_user_id_user_id_fk", "tableFrom": "member", "tableTo": "user", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["user_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -1943,9 +1846,7 @@ "indexes": { "organization_slug_unique": { "name": "organization_slug_unique", - "columns": [ - "slug" - ], + "columns": ["slug"], "isUnique": true } }, @@ -2047,35 +1948,25 @@ "indexes": { "placements_one_primary_uq": { "name": "placements_one_primary_uq", - "columns": [ - "collection_id" - ], + "columns": ["collection_id"], "isUnique": true, "where": "\"placements\".\"role\" = 'primary' AND \"placements\".\"collection_id\" IS NOT NULL" }, "placements_collection_location_uq": { "name": "placements_collection_location_uq", - "columns": [ - "collection_id", - "location_id" - ], + "columns": ["collection_id", "location_id"], "isUnique": true, "where": "\"placements\".\"collection_id\" IS NOT NULL" }, "placements_org_location_uq": { "name": "placements_org_location_uq", - "columns": [ - "organization_id", - "location_id" - ], + "columns": ["organization_id", "location_id"], "isUnique": true, "where": "\"placements\".\"organization_id\" IS NOT NULL" }, "placements_location_idx": { "name": "placements_location_idx", - "columns": [ - "location_id" - ], + "columns": ["location_id"], "isUnique": false } }, @@ -2084,12 +1975,8 @@ "name": "placements_collection_id_collections_id_fk", "tableFrom": "placements", "tableTo": "collections", - "columnsFrom": [ - "collection_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -2097,12 +1984,8 @@ "name": "placements_organization_id_organization_id_fk", "tableFrom": "placements", "tableTo": "organization", - "columnsFrom": [ - "organization_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" }, @@ -2110,12 +1993,8 @@ "name": "placements_location_id_storage_locations_id_fk", "tableFrom": "placements", "tableTo": "storage_locations", - "columnsFrom": [ - "location_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["location_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -2212,22 +2091,15 @@ "name": "push_packs_session_id_push_sessions_id_fk", "tableFrom": "push_packs", "tableTo": "push_sessions", - "columnsFrom": [ - "session_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["session_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } }, "compositePrimaryKeys": { "push_packs_session_id_pack_id_pk": { - "columns": [ - "session_id", - "pack_id" - ], + "columns": ["session_id", "pack_id"], "name": "push_packs_session_id_pack_id_pk" } }, @@ -2328,11 +2200,7 @@ "indexes": { "push_runs_tier_idx": { "name": "push_runs_tier_idx", - "columns": [ - "session_id", - "tier", - "merging_into" - ], + "columns": ["session_id", "tier", "merging_into"], "isUnique": false } }, @@ -2341,22 +2209,15 @@ "name": "push_runs_session_id_push_sessions_id_fk", "tableFrom": "push_runs", "tableTo": "push_sessions", - "columnsFrom": [ - "session_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["session_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } }, "compositePrimaryKeys": { "push_runs_session_id_seq_pk": { - "columns": [ - "session_id", - "seq" - ], + "columns": ["session_id", "seq"], "name": "push_runs_session_id_seq_pk" } }, @@ -2561,25 +2422,17 @@ "indexes": { "push_sessions_collection_idx": { "name": "push_sessions_collection_idx", - "columns": [ - "collection_id" - ], + "columns": ["collection_id"], "isUnique": false }, "push_sessions_expires_idx": { "name": "push_sessions_expires_idx", - "columns": [ - "status", - "expires_at" - ], + "columns": ["status", "expires_at"], "isUnique": false }, "push_sessions_user_idx": { "name": "push_sessions_user_idx", - "columns": [ - "user_id", - "status" - ], + "columns": ["user_id", "status"], "isUnique": false } }, @@ -2588,12 +2441,8 @@ "name": "push_sessions_collection_id_collections_id_fk", "tableFrom": "push_sessions", "tableTo": "collections", - "columnsFrom": [ - "collection_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -2624,10 +2473,7 @@ "foreignKeys": {}, "compositePrimaryKeys": { "ref_compaction_parts_compaction_id_part_pk": { - "columns": [ - "compaction_id", - "part" - ], + "columns": ["compaction_id", "part"], "name": "ref_compaction_parts_compaction_id_part_pk" } }, @@ -2738,10 +2584,7 @@ "foreignKeys": {}, "compositePrimaryKeys": { "ref_index_units_version_id_unit_pk": { - "columns": [ - "version_id", - "unit" - ], + "columns": ["version_id", "unit"], "name": "ref_index_units_version_id_unit_pk" } }, @@ -2820,25 +2663,17 @@ "indexes": { "ref_segments_range_idx": { "name": "ref_segments_range_idx", - "columns": [ - "state", - "first_hash", - "last_hash" - ], + "columns": ["state", "first_hash", "last_hash"], "isUnique": false }, "ref_segments_run_idx": { "name": "ref_segments_run_idx", - "columns": [ - "run_id" - ], + "columns": ["run_id"], "isUnique": false }, "ref_segments_tier_idx": { "name": "ref_segments_tier_idx", - "columns": [ - "tier" - ], + "columns": ["tier"], "isUnique": false } }, @@ -2876,19 +2711,14 @@ "indexes": { "schema_labels_label_idx": { "name": "schema_labels_label_idx", - "columns": [ - "label" - ], + "columns": ["label"], "isUnique": false } }, "foreignKeys": {}, "compositePrimaryKeys": { "schema_labels_schema_hash_label_pk": { - "columns": [ - "schema_hash", - "label" - ], + "columns": ["schema_hash", "label"], "name": "schema_labels_schema_hash_label_pk" } }, @@ -2944,9 +2774,7 @@ "indexes": { "schema_usage_hash_idx": { "name": "schema_usage_hash_idx", - "columns": [ - "schema_hash" - ], + "columns": ["schema_hash"], "isUnique": false } }, @@ -2955,24 +2783,15 @@ "name": "schema_usage_collection_id_collections_id_fk", "tableFrom": "schema_usage", "tableTo": "collections", - "columnsFrom": [ - "collection_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } }, "compositePrimaryKeys": { "schema_usage_collection_id_type_slug_set_from_seq_pk": { - "columns": [ - "collection_id", - "type_slug", - "set", - "from_seq" - ], + "columns": ["collection_id", "type_slug", "set", "from_seq"], "name": "schema_usage_collection_id_type_slug_set_from_seq_pk" } }, @@ -3075,16 +2894,12 @@ "indexes": { "session_token_unique": { "name": "session_token_unique", - "columns": [ - "token" - ], + "columns": ["token"], "isUnique": true }, "session_user_id_idx": { "name": "session_user_id_idx", - "columns": [ - "user_id" - ], + "columns": ["user_id"], "isUnique": false } }, @@ -3093,12 +2908,8 @@ "name": "session_user_id_user_id_fk", "tableFrom": "session", "tableTo": "user", - "columnsFrom": [ - "user_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["user_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -3257,12 +3068,8 @@ "name": "storage_locations_organization_id_organization_id_fk", "tableFrom": "storage_locations", "tableTo": "organization", - "columnsFrom": [ - "organization_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -3313,22 +3120,14 @@ "indexes": { "usage_rollups_account_idx": { "name": "usage_rollups_account_idx", - "columns": [ - "account_id", - "day" - ], + "columns": ["account_id", "day"], "isUnique": false } }, "foreignKeys": {}, "compositePrimaryKeys": { "usage_rollups_day_account_id_collection_id_metric_pk": { - "columns": [ - "day", - "account_id", - "collection_id", - "metric" - ], + "columns": ["day", "account_id", "collection_id", "metric"], "name": "usage_rollups_day_account_id_collection_id_metric_pk" } }, @@ -3393,9 +3192,7 @@ "indexes": { "user_email_unique": { "name": "user_email_unique", - "columns": [ - "email" - ], + "columns": ["email"], "isUnique": true } }, @@ -3454,9 +3251,7 @@ "indexes": { "verification_identifier_idx": { "name": "verification_identifier_idx", - "columns": [ - "identifier" - ], + "columns": ["identifier"], "isUnique": false } }, @@ -3715,39 +3510,27 @@ "indexes": { "versions_collection_seq_uq": { "name": "versions_collection_seq_uq", - "columns": [ - "collection_id", - "seq" - ], + "columns": ["collection_id", "seq"], "isUnique": true }, "versions_collection_semver_uq": { "name": "versions_collection_semver_uq", - "columns": [ - "collection_id", - "semver" - ], + "columns": ["collection_id", "semver"], "isUnique": true }, "versions_hash_idx": { "name": "versions_hash_idx", - "columns": [ - "hash" - ], + "columns": ["hash"], "isUnique": false }, "versions_published_idx": { "name": "versions_published_idx", - "columns": [ - "published_at" - ], + "columns": ["published_at"], "isUnique": false }, "versions_push_session_idx": { "name": "versions_push_session_idx", - "columns": [ - "push_session_id" - ], + "columns": ["push_session_id"], "isUnique": false } }, @@ -3756,12 +3539,8 @@ "name": "versions_collection_id_collections_id_fk", "tableFrom": "versions", "tableTo": "collections", - "columnsFrom": [ - "collection_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -3893,18 +3672,12 @@ "indexes": { "webhook_deliveries_webhook_idx": { "name": "webhook_deliveries_webhook_idx", - "columns": [ - "webhook_id", - "created_at" - ], + "columns": ["webhook_id", "created_at"], "isUnique": false }, "webhook_deliveries_pending_idx": { "name": "webhook_deliveries_pending_idx", - "columns": [ - "status", - "next_attempt_at" - ], + "columns": ["status", "next_attempt_at"], "isUnique": false } }, @@ -3913,12 +3686,8 @@ "name": "webhook_deliveries_webhook_id_collection_webhooks_id_fk", "tableFrom": "webhook_deliveries", "tableTo": "collection_webhooks", - "columnsFrom": [ - "webhook_id" - ], - "columnsTo": [ - "id" - ], + "columnsFrom": ["webhook_id"], + "columnsTo": ["id"], "onDelete": "cascade", "onUpdate": "no action" } @@ -3938,4 +3707,4 @@ "internal": { "indexes": {} } -} \ No newline at end of file +} diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index 23fec22..1af082d 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -213,4 +213,4 @@ "breakpoints": true } ] -} \ No newline at end of file +}