diff --git a/packages/protocol/src/repo/layouts/keyed.ts b/packages/protocol/src/repo/layouts/keyed.ts index 4900259..d540290 100644 --- a/packages/protocol/src/repo/layouts/keyed.ts +++ b/packages/protocol/src/repo/layouts/keyed.ts @@ -49,6 +49,16 @@ import { layerKey, filterKey, } from './kit.js' +import { + LEVEL_RATIO, + levelConfig, + type LevelShard, + LevelShards, + levelRaw, + levelTargets, + MAX_LEVEL, + shardKey, +} from './levels.js' import { type KeyRange, LiveHashes } from './reclaim.js' import { type LiveSet, @@ -113,6 +123,10 @@ export interface MergePlan { const mergeKey = (plan: string, name: string) => `merges/${plan}/${name}` +/** Whether a level lists the same shards in both (nothing changed it between). */ +const sameShards = (a: readonly LevelShard[] | undefined, b: readonly LevelShard[] | undefined) => + (a ?? []).map((s) => s.id).join() === (b ?? []).map((s) => s.id).join() + export interface MaintainOptions { /** * Run a merge too large for one job as range jobs (the server queues @@ -142,6 +156,8 @@ const MAX_PACK_TIER = 3 export const UPLOAD_PACK_FINAL_RAW = 1024 * 1024 /** Commits whose uploads' packs a manifest remembers listing (`Manifest.adopted`). */ const ADOPTED_KEPT = 1000 +/** Present in a leveled collection's prefix (`KeyedLayout.markLeveled`). */ +const LEVELED_KEY = 'manifest/f/leveled.json' /** How long a replaced object stays readable (readers holding an older manifest). */ export const layoutConfig = { @@ -164,6 +180,11 @@ export const layoutConfig = { * calls stay well inside what `maintainCalls` leaves (tests set it low). */ mergeInputs: 128, + /** + * Level a collection's packs when its first write finds it empty (levels.ts); one + * holding packs already stays tiered. The server switches it on. + */ + levelNew: false, } /** @@ -198,10 +219,13 @@ export class KeyedLayout implements RecordStore { readonly packs: PackReader readonly index: LayeredIndex readonly layers: LayerReader + /** A leveled collection's level listings (levels.ts). */ + readonly levels: LevelShards /** Every storage call this layout makes goes through `store`, counted here. */ readonly #io = { calls: 0 } readonly store: Store + #leveled: { v: Promise; retry: number } | null = null readonly name = 'f' @@ -216,6 +240,7 @@ export class KeyedLayout implements RecordStore { this.packs = new PackReader(store, lru, scope) this.index = new LayeredIndex(store, lru, scope, this.manifests) this.layers = this.index.reader + this.levels = new LevelShards(store) } /** Whether no pack merge will take `p` again: it then lives in the registry, not the manifest. */ @@ -223,10 +248,68 @@ export class KeyedLayout implements RecordStore { return p.tier >= MAX_PACK_TIER || p.bytes >= SMALL_PACK * 8 ** p.tier } - /** Every pack: the manifest's and the registry's (reclaim, forks). */ + /** + * Whether this collection's packs are leveled (levels.ts): marked once (`markLeveled`), + * by its first write while `layoutConfig.levelNew` is on, or by a fork of one that is. + * A collection holding packs when leveling was switched on stays tiered. Remembered + * once known. + */ + leveled(): Promise { + if (!this.#leveled || Date.now() > this.#leveled.retry) { + const known = { v: Promise.resolve(false), retry: Infinity } + known.v = this.#isLeveled(known) + this.#leveled = known + } + return this.#leveled.v + } + + async #isLeveled(known: { retry: number }): Promise { + if (await this.store.head(LEVELED_KEY)) return true + if (!layoutConfig.levelNew) { + // Read again later: leveling may be switched on meanwhile (the next write decides). + known.retry = Date.now() + 5000 + return false + } + const m = await this.manifests.latest() + const empty = !m.layers.length && !m.packs.length && !(await this.registry.read()).keys.length + if (empty) { + await this.store.put(LEVELED_KEY, '{}', { ifAbsent: true }) + return true + } + // Another writer may have marked it just as this one looked. + return (await this.store.head(LEVELED_KEY)) !== null + } + + /** Make this collection leveled, before anything is written to it. */ + async markLeveled(): Promise { + await this.store.put(LEVELED_KEY, '{}', { ifAbsent: true }) + this.#leveled = { v: Promise.resolve(true), retry: Infinity } + } + + /** + * Whether `p` is listed in the registry rather than the manifest or a level: in a + * leveled collection only history packs are (every other is merged again). + */ + #final(p: PackInfo, leveled: boolean): boolean { + return leveled ? !!p.history : KeyedLayout.final(p) + } + + /** Every pack of levels 1… , by level. */ + async #levelPacks(m: Manifest): Promise> { + const out = new Map() + for (const [l, shards] of Object.entries(m.levels ?? {})) + if (shards.length) out.set(Number(l), await this.levels.packs(shards)) + return out + } + + /** Every pack: the manifest's, the levels' and the registry's (reclaim, forks). */ async allPacks(): Promise { const m = await this.manifests.latest() - return [...m.packs, ...(await this.#registered(m)).packs] + return [ + ...m.packs, + ...[...(await this.#levelPacks(m)).values()].flat(), + ...(await this.#registered(m)).packs, + ] } /** @@ -333,6 +416,7 @@ export class KeyedLayout implements RecordStore { async adoptPacks(name: string, packs: readonly PackInfo[]): Promise { if (!packs.length) return if ((await this.manifests.latest()).adopted?.includes(name)) return + if (await this.leveled()) return this.#adoptLeveled(name, packs) const large = (p: PackInfo) => p.raw >= UPLOAD_PACK_FINAL_RAW || KeyedLayout.final(p) // A fixed key: written once even if this stops before the manifest says so. await this.registry.add(packs.filter(large), `upload-${name}`) @@ -346,6 +430,42 @@ export class KeyedLayout implements RecordStore { }) } + /** + * Leveled: an upload's packs go to level 0, to be merged down. But a load sent in + * key order (packs that never overlap) into an empty collection is already what + * the bottom level is: listed there as written, never rewritten to get there (a + * Crossref-sized load, 1 TB, would otherwise go down every level). + */ + async #adoptLeveled(name: string, packs: readonly PackInfo[]): Promise { + const sorted = [...packs].sort((a, b) => compareUtf8(a.lo ?? '', b.lo ?? '')) + const disjoint = sorted.every( + (p, i) => p.lo && p.hi && (i === 0 || compareUtf8(p.lo, sorted[i - 1]!.hi!) > 0), + ) + const empty = (m: Manifest) => + !m.packs.length && !Object.values(m.levels ?? {}).some((s) => s.length) + if (disjoint && empty(await this.manifests.latest())) { + const shards = await this.levels.write(sorted.map((p) => ({ ...p, level: MAX_LEVEL }))) + let placed = false + await this.manifests.update((m) => { + if (m.adopted?.includes(name) || !empty(m)) return null + m.levels = { ...m.levels, [MAX_LEVEL]: shards } + m.adopted = [...(m.adopted ?? []), name].slice(-ADOPTED_KEPT) + placed = true + return m + }) + if (placed) return + // Something landed first: level 0 after all (or a retry already listed them). + for (const s of shards) await this.store.delete(shardKey(s.id)) + } + await this.manifests.update((m) => { + if (m.adopted?.includes(name)) return null + const have = new Set(m.packs.map((p) => p.id)) + m.packs.push(...packs.filter((p) => !have.has(p.id))) + m.adopted = [...(m.adopted ?? []), name].slice(-ADOPTED_KEPT) + return m + }) + } + /** * Delete packs an upload wrote for commit `name`, but none the collection lists: * a pack `adoptPacks` named (in the manifest, its retired files, or the registry @@ -357,6 +477,7 @@ export class KeyedLayout implements RecordStore { const registered = await this.store.get(this.registry.keyFor(`upload-${name}`)) if (registered) for (const p of JSON.parse(await registered.text()) as PackInfo[]) listed.add(p.id) + for (const ps of (await this.#levelPacks(m)).values()) for (const p of ps) listed.add(p.id) const gone = ids.filter((id) => !listed.has(id) && !listed.has(packKey(id))) for (let i = 0; i < gone.length; i += 16) await Promise.all( @@ -395,9 +516,10 @@ export class KeyedLayout implements RecordStore { // landing throughout, layers alone could keep a job busy and packs pile up. A // job stops well inside its lease and a Worker's subrequests; the next carries on. const until = Date.now() + layoutConfig.maintainBudgetMs + const leveled = await this.leveled() for (;;) { const layers = await this.#mergeLayers(opts.split) - const packs = await this.#mergePacks() + const packs = leveled ? await this.#mergeLevels() : await this.#mergePacks() if (!layers && !packs) break if (Date.now() >= until || spent() >= layoutConfig.maintainCalls) { more = true @@ -598,12 +720,207 @@ export class KeyedLayout implements RecordStore { return this.#swap({ oldPacks: inputs.map((p) => p.id), newLayers: files, newPacks: packs }) } + /** + * Leveled: one merge down, from the level most over its target (levels.ts). Level 0's + * oldest packs merge into the base level; a pack of a level below it merges into + * the next, picking the one whose overlap there is smallest for its size (RocksDB's + * kMinOverlappingRatio: each merge rewrites as little as it can). Both sides go + * through as streams into the next level's packs, cut at 16 MiB and at types, and a + * new layer points the index at where each record now is. + * + * A record id found with several hashes keeps the newest (level 0's newest pack, + * then the upper level) in the levels; the older hashes go to this month's history + * packs. That's a guess at which is current: a revert brings back an old hash, then + * read from history. Only locality depends on it, since the index says where every + * copy is. + */ + async #mergeLevels(): Promise { + const m = await this.manifests.latest() + const { base, targets } = levelTargets(m.levels) + const zero = m.packs.filter((p) => !p.history) + const zeroRaw = zero.reduce((n, p) => n + p.raw, 0) + // Level 0 merges once it's a tenth of the level it merges into: while a collection + // is small its base is the bottom, and a few small commits shouldn't rewrite it all. + const zeroTrigger = Math.max(levelConfig.level0Raw, levelRaw(m.levels, base) / LEVEL_RATIO) + let from = -1 + let score = 1 + if (zero.length >= levelConfig.level0Packs || zeroRaw >= zeroTrigger) { + from = 0 + score = Math.max(zero.length / levelConfig.level0Packs, zeroRaw / zeroTrigger) + } + for (let l = 1; l < MAX_LEVEL; l++) { + const raw = levelRaw(m.levels, l) + if (!raw) continue + // A level above the base has no room: what it holds goes down first. + const s = targets[l]! > 0 ? raw / targets[l]! : Infinity + if (s > score) { + from = l + score = s + } + } + if (from < 0) return false + let upper: PackInfo[] + let target: number + if (from === 0) { + // Oldest first, so what stays in level 0 is newer than what went down. + upper = [] + let raw = 0 + for (const p of zero) { + if (upper.length && raw + p.raw > levelConfig.level0MergeRaw) break + upper.push(p) + raw += p.raw + } + target = base + } else { + target = from + 1 + const here = await this.levels.packs(m.levels?.[from] ?? []) + const below = await this.levels.packs(m.levels?.[target] ?? []) + // Each pack's overlap below, by a walk of both in key order. + let best: PackInfo | null = null + let bestRatio = Infinity + let j = 0 + for (const p of here) { + while (j < below.length && compareUtf8(below[j]!.hi!, p.lo!) < 0) j++ + let over = 0 + for (let k = j; k < below.length && compareUtf8(below[k]!.lo!, p.hi!) <= 0; k++) + over += below[k]!.raw + const ratio = (over + 1) / Math.max(1, p.raw) + if (ratio < bestRatio) { + best = p + bestRatio = ratio + } + } + upper = [best!] + } + const lo = upper.reduce((a, p) => (compareUtf8(p.lo!, a) < 0 ? p.lo! : a), upper[0]!.lo!) + const hi = upper.reduce((a, p) => (compareUtf8(p.hi!, a) > 0 ? p.hi! : a), upper[0]!.hi!) + const lower = await this.levels.overlapping(m.levels?.[target] ?? [], lo, hi) + // Newest first: level 0's newest pack, …, its oldest, then the level below. + const ranked = [...(from === 0 ? [...upper].reverse() : upper), ...lower] + const reader = this.packs + const read = mergeRead(ranked.length) + const streams = ranked.map((p, rank) => + (async function* () { + for await (const { key, line } of reader.all(p.id, read)) yield { key, line, rank } + })(), + ) + const month = new Date().toISOString().slice(0, 7) + const pw = new PackWriter(this.store, { level: target, packRaw: levelConfig.packRaw }) + const hw = new PackWriter(this.store, { history: month, packRaw: levelConfig.packRaw }) + const lw = new RunWriter(this.store, 0) + let group: { key: string; line: string; rank: number }[] = [] + let n = 0 + const flush = async () => { + if (!group.length) return + let cur = group[0]! + for (const x of group) if (x.rank < cur.rank) cur = x + let last = '' + for (const x of group) { + // A copy of a key already written (the same record twice) is one record. + if (x.key === last) continue + if (x.key === cur.key) { + if (x !== cur) continue + lw.add(x.key, pw.append(x.key, x.line)) + } else lw.add(x.key, hw.append(x.key, x.line)) + last = x.key + } + group = [] + if (++n % 64 === 0) await Promise.all([pw.ready(), hw.ready(), lw.ready()]) + } + for await (const x of mergeAll(streams)) { + const id = x.key.slice(0, x.key.lastIndexOf('\u0000')) + if (group.length) { + const g = group[0]!.key + if (g.slice(0, g.lastIndexOf('\u0000')) !== id) await flush() + } + group.push(x) + } + await flush() + const packs = await pw.finish() + const history = await hw.finish() + const files = await lw.finish() + return this.#swapLevels({ + zero: from === 0 ? upper.map((p) => p.id) : [], + from: from > 0 ? { level: from, ids: upper.map((p) => p.id) } : null, + target, + lower: lower.map((p) => p.id), + packs, + history, + layers: files, + }) + } + + /** + * A leveled merge's result in one manifest change: its inputs out of level 0 or + * their level, its output in the target level, its history packs in the registry, + * and its layer the newest. Only if the levels it changed are as it read them and + * level 0's inputs are still listed; otherwise everything it wrote goes. + */ + async #swapLevels(o: { + zero: string[] + from: { level: number; ids: string[] } | null + target: number + lower: string[] + packs: PackInfo[] + history: PackInfo[] + layers: LayerInfo[] + }): Promise { + const m0 = await this.manifests.latest() + const changed = new Map>>() + if (o.from) + changed.set( + o.from.level, + await this.levels.replace(m0.levels?.[o.from.level] ?? [], new Set(o.from.ids), []), + ) + changed.set( + o.target, + await this.levels.replace(m0.levels?.[o.target] ?? [], new Set(o.lower), o.packs), + ) + const registered = await this.registry.add(o.history) + const zero = new Set(o.zero) + let lost = false + await this.manifests.update((m) => { + const have = new Set(m.packs.map((p) => p.id)) + if ( + ![...zero].every((id) => have.has(id)) || + [...changed.keys()].some((l) => !sameShards(m.levels?.[l], m0.levels?.[l])) + ) { + lost = true + return null + } + const now = Date.now() + m.packs = m.packs.filter((p) => !zero.has(p.id)) + m.levels = { ...m.levels } + for (const [l, c] of changed) m.levels[l] = c.shards + const seq = topSeq(m) + 1 + for (const l of o.layers) m.layers.push({ ...l, seq }) + for (const id of [...o.zero, ...(o.from?.ids ?? []), ...o.lower]) + m.retired.push({ key: packKey(id), at: now }, { key: packDirKey(id), at: now }) + for (const c of changed.values()) + for (const id of c.replaced) m.retired.push({ key: shardKey(id), at: now }) + m.stats.levelMerges = (m.stats.levelMerges ?? 0) + 1 + return m + }) + if (!lost) return true + const gone = [ + ...o.layers.flatMap((l) => [layerKey(l.id), layerDirKey(l.id), layerFilterKey(l.id)]), + ...[...o.packs, ...o.history].flatMap((p) => [packKey(p.id), packDirKey(p.id)]), + ...[...changed.values()].flatMap((c) => c.written.map(shardKey)), + ...(registered ? [registered] : []), + ] + for (let i = 0; i < gone.length; i += 16) + await Promise.all(gone.slice(i, i + 16).map((k) => this.store.delete(k))) + return false + } + /** * Replace layers and packs in one manifest change: only if every input the * manifest lists is still there (another job may have merged them first); * otherwise drop what was written. New packs no merge will take are listed in * the registry, the rest in the manifest; `unlisted` are old packs already in - * neither (reclaim took them out of the registry), retired with the rest. + * neither (reclaim took them out of the registry), retired with the rest. In a + * leveled collection old packs a level lists come out of its shards (only if the + * level is as read), and new packs go to level 0 but for history packs. */ async #swap(o: { oldLayers?: string[] @@ -620,15 +937,29 @@ export class KeyedLayout implements RecordStore { const oldPacks = new Set(o.oldPacks ?? []) const newLayers = o.newLayers ?? [] const newPacks = o.newPacks ?? [] - const listed = newPacks.filter((p) => !KeyedLayout.final(p)) - const registered = await this.registry.add(newPacks.filter(KeyedLayout.final)) + const leveled = await this.leveled() + const listed = newPacks.filter((p) => !this.#final(p, leveled)) + const registered = await this.registry.add(newPacks.filter((p) => this.#final(p, leveled))) + const m0 = leveled ? await this.manifests.latest() : null + const changed = new Map>>() + const inLevels = new Set() + if (m0) { + const gone = new Set([...oldPacks, ...(o.unlisted ?? [])]) + for (const [l, ps] of await this.#levelPacks(m0)) { + const ids = new Set(ps.filter((p) => gone.has(p.id)).map((p) => p.id)) + if (!ids.size) continue + for (const id of ids) inLevels.add(id) + changed.set(l, await this.levels.replace(m0.levels![l]!, ids, [])) + } + } let lost = false await this.manifests.update((m) => { const haveL = new Set(m.layers.map((l) => l.id)) const haveP = new Set(m.packs.map((p) => p.id)) if ( ![...oldLayers].every((id) => haveL.has(id)) || - ![...oldPacks].every((id) => haveP.has(id)) + ![...oldPacks].every((id) => haveP.has(id) || inLevels.has(id)) || + [...changed.keys()].some((l) => !sameShards(m.levels?.[l], m0!.levels?.[l])) ) { lost = true return null @@ -636,6 +967,13 @@ export class KeyedLayout implements RecordStore { const now = Date.now() m.layers = m.layers.filter((l) => !oldLayers.has(l.id)) m.packs = m.packs.filter((p) => !oldPacks.has(p.id)) + if (changed.size) { + m.levels = { ...m.levels } + for (const [l, c] of changed) { + m.levels[l] = c.shards + for (const id of c.replaced) m.retired.push({ key: shardKey(id), at: now }) + } + } // A pack merge's layer is the newest: its places win over the old ones. const seq = topSeq(m) + 1 for (const l of newLayers) m.layers.push(o.keepSeq ? l : { ...l, seq }) @@ -660,6 +998,8 @@ export class KeyedLayout implements RecordStore { ]) for (const p of newPacks) await Promise.all([this.store.delete(packKey(p.id)), this.store.delete(packDirKey(p.id))]) + for (const c of changed.values()) + await Promise.all(c.written.map((id) => this.store.delete(shardKey(id)))) if (registered) await this.store.delete(registered) return false } @@ -869,10 +1209,14 @@ export class KeyedLayout implements RecordStore { throw new TooManyRunsError(`Reclaim: ${n} index runs, more than ${RECLAIM_MAX_RUNS}`) const ranges = await this.#cutRanges(m.layers, rangeEntries) const registry = await this.#registered(m) + const levelPacks = [...(await this.#levelPacks(m)).values()].flat() const plan: ReclaimPlan = { id: newId(), layers: m.layers, - packs: [...m.packs, ...registry.packs].map((p) => ({ id: p.id, records: p.records })), + packs: [...m.packs, ...levelPacks, ...registry.packs].map((p) => ({ + id: p.id, + records: p.records, + })), registry: registry.keys, ranges, level: Math.max(0, ...m.layers.map((l) => l.level)), @@ -1095,10 +1439,11 @@ export class KeyedLayout implements RecordStore { throw new Error('forkInto: F needs both collections in their own prefixes') const m = await this.manifests.latest() const registered = (await this.#registered(m)).packs + const levelPacks = await this.#levelPacks(m) // Each file's home: where the parent read it from (its own parent's), else the parent. const mine = await from.inherited() const inherited: Inherited = { packs: {}, layers: {} } - for (const p of [...m.packs, ...registered]) + for (const p of [...m.packs, ...registered, ...[...levelPacks.values()].flat()]) inherited.packs[p.id] = mine.packs[p.id] ?? from.home for (const l of m.layers) inherited.layers[l.id] = mine.layers[l.id] ?? from.home const homes = new Set([...Object.values(inherited.packs), ...Object.values(inherited.layers)]) @@ -1111,9 +1456,17 @@ export class KeyedLayout implements RecordStore { await to.put(INHERITED_KEY, body) to.forget() await target.registry.add(registered) + // A leveled parent's fork is leveled, its levels listing the parent's packs (in + // shards of its own: they change as the fork's merges rewrite its levels). + const levels: Record = {} + if (await this.leveled()) { + await target.markLeveled() + for (const [l, ps] of levelPacks) levels[l] = await target.levels.write(ps) + } // The parent's layers keep their order but number below anything the fork writes. const top = Math.max(0, ...m.layers.map((l) => l.seq)) await target.manifests.update((n) => { + if (Object.keys(levels).length) n.levels = levels n.packs.push(...m.packs) n.layers.push(...m.layers.map((l) => ({ ...l, seq: l.seq - top - 1 }))) return n @@ -1150,11 +1503,16 @@ export class KeyedLayout implements RecordStore { { key: at(h, layerFilterKey(l.id)), optional: true }, ) } - for (const p of [...m.packs, ...registry.packs]) { + const levelPacks = [...(await this.#levelPacks(m)).values()].flat() + for (const p of [...m.packs, ...levelPacks, ...registry.packs]) { const h = inh.packs[p.id] files.push({ key: at(h, packKey(p.id)), bytes: p.bytes }, { key: at(h, packDirKey(p.id)) }) } for (const k of registry.keys) files.push({ key: at(undefined, k) }) + // A leveled collection's mark and its levels' shards: the copy's own, written once. + for (const shards of Object.values(m.levels ?? {})) + for (const s of shards) files.push({ key: at(undefined, shardKey(s.id)) }) + if (await this.leveled()) files.push({ key: at(undefined, LEVELED_KEY) }) if (Object.keys(inh.packs).length || Object.keys(inh.layers).length) files.push({ key: at(undefined, 'inherited.json') }) const dir = `runs/${home.home}/manifest/${this.name}` @@ -1402,9 +1760,12 @@ class KeyedWriter implements RecordWriter { this.#entries = [] const files = await lw.finish() // Listed before the manifest names their index: a pack the manifest never got - // (a failed update) is one reclaim finds with nothing live, and drops. - await this.layout.registry.add(packs.filter(KeyedLayout.final)) - const run = { layers: files, packs: packs.filter((p) => !KeyedLayout.final(p)) } + // (a failed update) is one reclaim finds with nothing live, and drops. A leveled + // collection's go to level 0, every one: merges take them down. + const leveled = await this.layout.leveled() + const final = (p: PackInfo) => !leveled && KeyedLayout.final(p) + await this.layout.registry.add(packs.filter(final)) + const run = { layers: files, packs: packs.filter((p) => !final(p)) } // Held: the caller names them later, with the rest of its commit (appendRuns). if (this.opts.hold) this.#held.push(run) else await this.layout.manifests.append(run.layers, run.packs) diff --git a/packages/protocol/src/repo/layouts/kit.ts b/packages/protocol/src/repo/layouts/kit.ts index b87da13..61d91d3 100644 --- a/packages/protocol/src/repo/layouts/kit.ts +++ b/packages/protocol/src/repo/layouts/kit.ts @@ -17,6 +17,7 @@ import { compareUtf8 } from '../../utf8.js' import { gunzipText, gzip } from '../gzip.js' import type { Lru } from '../lru.js' import { putJoined, type Store } from '../types.js' +import type { Levels } from './levels.js' export const BLOCK_RAW = 64 * 1024 export const PACK_RAW = 16 * 1024 * 1024 @@ -87,6 +88,13 @@ export interface PackInfo { * spans types and merges take one type's packs. Absent on packs written before. */ type?: string + /** The pack's first and last keys: a level's packs never overlap (KeyedLayout's levels). */ + lo?: string + hi?: string + /** A leveled collection's level the pack is in: 0 (or absent) for a commit's or upload's. */ + level?: number + /** A pack of replaced versions a leveled merge moved out, by month written (`2026-10`). */ + history?: string } /** The record type a key leads with (recordKey); '' for a key without one. */ @@ -113,13 +121,22 @@ export class PackWriter { #records = 0 #lastKey: string | null = null #type = '' + #lo = '' + #hi = '' readonly done: PackInfo[] = [] readonly #writing = new Set>() #error: unknown = null constructor( readonly store: Store, - readonly opts: { tier?: number; hooks?: PackHooks } = {}, + readonly opts: { + tier?: number + hooks?: PackHooks + level?: number + history?: string + /** Raw bytes a pack closes at (PACK_RAW). */ + packRaw?: number + } = {}, ) {} /** @@ -132,13 +149,19 @@ export class PackWriter { this.#close() this.#type = type this.#lastKey = key - if (!this.#id) this.#id = newId() + if (!this.#id) { + this.#id = newId() + this.#lo = key + } + this.#hi = key this.#lines.push(`${key}\t${line}`) const at = loc(this.#id, this.#blocks.length, this.#lines.length - 1) this.#raw += key.length + line.length + 2 this.#records++ if (this.#raw >= BLOCK_RAW) this.#seal() - if (this.#packRaw >= PACK_RAW) this.#close() + // A size given counts the block filling too (a small one closes within a block). + const limit = this.opts.packRaw + if (limit ? this.#packRaw + this.#raw >= limit : this.#packRaw >= PACK_RAW) this.#close() return at } @@ -175,6 +198,8 @@ export class PackWriter { records: this.#records, raw: this.#packRaw, type: this.#type, + lo: this.#lo, + hi: this.#hi, } this.#id = '' this.#blocks = [] @@ -194,6 +219,8 @@ export class PackWriter { records: number raw: number type: string + lo: string + hi: string }) { const parts = await Promise.all(job.blocks) const offsets = [0] @@ -208,6 +235,10 @@ export class PackWriter { bytes: offsets.at(-1)!, raw: job.raw, ...(job.type ? { type: job.type } : {}), + lo: job.lo, + hi: job.hi, + ...(this.opts.level ? { level: this.opts.level } : {}), + ...(this.opts.history ? { history: this.opts.history } : {}), } this.done.push(pack) await this.opts.hooks?.after?.(pack) @@ -852,6 +883,11 @@ export interface Manifest { * retry lists none again, not even one a merge has since taken and swept. */ adopted?: string[] + /** + * A leveled collection's levels 1… (levels.ts), by level: the shards listing each + * one's packs. `packs` above is level 0. + */ + levels?: Levels } const empty = (): Manifest => ({ seq: 0, layers: [], packs: [], retired: [], stats: {} }) diff --git a/packages/protocol/src/repo/layouts/levels.ts b/packages/protocol/src/repo/layouts/levels.ts new file mode 100644 index 0000000..7474a06 --- /dev/null +++ b/packages/protocol/src/repo/layouts/levels.ts @@ -0,0 +1,206 @@ +/** + * A leveled collection's levels (leveled-packs-design.md): below level 0 (the packs + * commits and uploads write, listed in the manifest as ever), levels 1…MAX_LEVEL + * each hold packs that never overlap, in key order, within one type. The bottom + * level holds most of the collection; each level above is 1/T the size of the one + * below, so a record is rewritten about T times a level on its way down, and a leaf + * of the current version reads about one pack a level, however many days of + * changes went in (Crossref's dailies over three years, simulated: 28 GETs a leaf + * at the median where tiered packs took 255). + * + * A level's packs are listed in shards: write-once objects of up to `shardPacks` + * packs each, in key order, which the manifest names with their key ranges. The + * bottom level of a Crossref-sized collection is tens of thousands of packs, too + * many for the manifest every reader reads; a merge rewrites only the shards its + * key range touches. Readers never read them: records are found through the index. + */ +import { compareUtf8 } from '../../utf8.js' +import type { Store } from '../types.js' +import { newId, PACK_RAW, type PackInfo } from './kit.js' + +/** Levels below level 0. The bottom one holds most of a collection (RocksDB's seven, 0–6). */ +export const MAX_LEVEL = 6 +/** Each level's size against the one below it. */ +export const LEVEL_RATIO = 10 + +/** Sizes (tests set them low). */ +export const levelConfig = { + /** + * The smallest a level's target gets: level 0 merges into the highest level whose + * target is at least this, skipping smaller ones (RocksDB's dynamic level sizes). + * About half a day of Crossref's changes: three levels above a 1 TB bottom. + */ + minRaw: 256 * 1024 * 1024, + /** + * Level 0 merges down once it holds this many packs, or `level0Raw` raw bytes (or a + * tenth of the level it merges into, if more). A leaf's recent changes are in level + * 0's packs, one read each, so their number is a read cost. + */ + level0Packs: 16, + level0Raw: 64 * 1024 * 1024, + /** Raw bytes of level-0 packs one merge takes, oldest first. */ + level0MergeRaw: 256 * 1024 * 1024, + /** Raw bytes a merge's packs close at. */ + packRaw: PACK_RAW, + /** Packs a shard lists at most. */ + shardPacks: 512, +} + +/** A shard of a level's listing, as the manifest names it. */ +export interface LevelShard { + id: string + lo: string + hi: string + packs: number + /** Raw and stored bytes of its packs. */ + raw: number + bytes: number +} + +export type Levels = Record + +export const shardKey = (id: string) => `manifest/f/levels/${id}.json` + +/** Whether [aLo, aHi] and [bLo, bHi] share a key. */ +export const overlaps = (aLo: string, aHi: string, bLo: string, bHi: string) => + compareUtf8(aLo, bHi) <= 0 && compareUtf8(bLo, aHi) <= 0 + +const byLo = (a: PackInfo, b: PackInfo) => compareUtf8(a.lo!, b.lo!) + +/** Raw bytes a level holds. */ +export const levelRaw = (levels: Levels | undefined, level: number) => + (levels?.[level] ?? []).reduce((n, s) => n + s.raw, 0) + +/** + * Each level's target size from the bottom up, and the base level, where level 0 + * merges: the highest whose target is at least `levelConfig.minRaw`, and never the bottom, + * so level 0 never rewrites the whole collection (the level above it takes level + * 0's merges and goes down a pack at a time). Levels above the base have no room + * (target 0). + */ +export function levelTargets(levels: Levels | undefined): { base: number; targets: number[] } { + const targets = Array.from({ length: MAX_LEVEL + 1 }).fill(0) + let t = levelRaw(levels, MAX_LEVEL) + targets[MAX_LEVEL] = Infinity + let base = MAX_LEVEL - 1 + targets[base] = t / LEVEL_RATIO + t /= LEVEL_RATIO + for (let l = MAX_LEVEL - 2; l >= 1; l--) { + t /= LEVEL_RATIO + if (t < levelConfig.minRaw) break + targets[l] = t + base = l + } + return { base, targets } +} + +/** The shards a leveled collection's levels are listed in (each read once: they never change). */ +export class LevelShards { + readonly #read = new Map>() + + constructor(readonly store: Store) {} + + read(id: string): Promise { + let p = this.#read.get(id) + if (!p) { + p = this.store.get(shardKey(id)).then(async (o) => { + if (!o) throw new Error(`Missing level shard ${id}`) + return JSON.parse(await o.text()) as PackInfo[] + }) + p.catch(() => this.#read.delete(id)) + this.#read.set(id, p) + } + return p + } + + /** Every pack of `shards`, in key order. */ + async packs(shards: readonly LevelShard[]): Promise { + const out: PackInfo[] = [] + for (let i = 0; i < shards.length; i += 16) + for (const ps of await Promise.all(shards.slice(i, i + 16).map((s) => this.read(s.id)))) + out.push(...ps) + return out + } + + /** The packs of `shards` that hold a key in [lo, hi], in key order. */ + async overlapping(shards: readonly LevelShard[], lo: string, hi: string): Promise { + const touched = shards.filter((s) => overlaps(s.lo, s.hi, lo, hi)) + return (await this.packs(touched)).filter((p) => overlaps(p.lo!, p.hi!, lo, hi)) + } + + /** `packs` (in key order, none overlapping) as new shards, written. */ + async write(packs: readonly PackInfo[]): Promise { + const out: LevelShard[] = [] + for (let i = 0; i < packs.length; i += levelConfig.shardPacks) { + const ps = packs.slice(i, i + levelConfig.shardPacks) + const id = newId() + this.#read.set(id, Promise.resolve(ps)) + await this.store.put(shardKey(id), JSON.stringify(ps), { ifAbsent: true }) + out.push({ + id, + lo: ps[0]!.lo!, + hi: ps.at(-1)!.hi!, + packs: ps.length, + raw: ps.reduce((n, p) => n + p.raw, 0), + bytes: ps.reduce((n, p) => n + p.bytes, 0), + }) + } + return out + } + + /** + * A level's shards with the packs `remove` gone and `add` in. Only the shards + * holding a removed pack, or whose range the added ones fall in or between, are + * written again: the result, the shards written, and the ones they replace. + * Throws if the level would overlap. + */ + async replace( + shards: readonly LevelShard[], + remove: ReadonlySet, + add: readonly PackInfo[], + ): Promise<{ shards: LevelShard[]; written: string[]; replaced: string[] }> { + const added = [...add].sort(byLo) + // Shards touched: those holding a removed pack (found by reading the ones whose + // range could), and those the added packs' range reaches. + const touched = new Set() + for (let i = 0; i < shards.length; i++) { + const s = shards[i]! + if (added.length && overlaps(s.lo, s.hi, added[0]!.lo!, added.at(-1)!.hi!)) touched.add(i) + } + if (remove.size) + for (let i = 0; i < shards.length; i++) { + if (touched.has(i)) continue + const ps = await this.read(shards[i]!.id) + if (ps.some((p) => remove.has(p.id))) touched.add(i) + } + // Added packs that fall between shards: the shards on either side join, so the + // rewritten stretch stays contiguous. + if (added.length && !touched.size) { + const at = shards.findIndex((s) => compareUtf8(s.lo, added[0]!.lo!) > 0) + if (at > 0) touched.add(at - 1) + else if (at === 0) touched.add(0) + else if (shards.length) touched.add(shards.length - 1) + } + if (!touched.size && !added.length) return { shards: [...shards], written: [], replaced: [] } + const idx = [...touched].sort((a, b) => a - b) + const from = idx[0] ?? shards.length + const to = idx.length ? idx.at(-1)! : shards.length - 1 + const kept = (await this.packs(shards.slice(from, to + 1))).filter((p) => !remove.has(p.id)) + const packs = [...kept, ...added].sort(byLo) + for (let i = 1; i < packs.length; i++) + if (compareUtf8(packs[i]!.lo!, packs[i - 1]!.hi!) <= 0) + throw new Error(`Level overlap: pack ${packs[i]!.id} starts inside ${packs[i - 1]!.id}`) + const fresh = await this.write(packs) + const before = shards.slice(0, from) + const after = shards.slice(to + 1) + if (before.length && fresh.length && compareUtf8(fresh[0]!.lo, before.at(-1)!.hi) <= 0) + throw new Error('Level overlap at a shard boundary') + if (after.length && fresh.length && compareUtf8(fresh.at(-1)!.hi, after[0]!.lo) >= 0) + throw new Error('Level overlap at a shard boundary') + return { + shards: [...before, ...fresh, ...after], + written: fresh.map((s) => s.id), + replaced: shards.slice(from, to + 1).map((s) => s.id), + } + } +} diff --git a/packages/protocol/test/repo/leveled.test.ts b/packages/protocol/test/repo/leveled.test.ts new file mode 100644 index 0000000..b598500 --- /dev/null +++ b/packages/protocol/test/repo/leveled.test.ts @@ -0,0 +1,242 @@ +/** + * Layout F's leveled packs (layouts/levels.ts): merges take level 0 down to the base + * and each level into the next, every level's packs apart in key order, replaced + * versions in history packs, and every record readable through the index throughout. + */ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { + headConfig, + HomeStore, + KeyedLayout, + layoutConfig, + memoryStore, + type PackInfo, + type Store, +} from '../../src/index.js' +import { levelConfig, MAX_LEVEL } from '../../src/repo/layouts/levels.js' +import { repoLru } from '../../src/repo/repo.js' +import { compareUtf8 } from '../../src/utf8.js' + +const saved = { level: { ...levelConfig }, layout: { ...layoutConfig } } +beforeEach(() => { + headConfig.intervalMs = 1 + layoutConfig.graceMs = 0 + // Small sizes, so a few hundred records make several levels of several packs. + Object.assign(levelConfig, { + minRaw: 1_000, + level0Packs: 4, + level0Raw: 4_000, + level0MergeRaw: 8_000, + packRaw: 3_000, + shardPacks: 3, + }) +}) +afterEach(() => { + headConfig.intervalMs = 1000 + Object.assign(levelConfig, saved.level) + Object.assign(layoutConfig, saved.layout) +}) + +const layout = (store: Store = memoryStore(), home = 'c1') => + new KeyedLayout(new HomeStore(store, home), repoLru(8 << 20), home) + +const leveledLayout = async (store?: Store, home?: string) => { + const l = layout(store, home) + await l.markLeveled() + return l +} + +const ids = Array.from( + { length: 400 }, + (_, i) => `10.${1000 + (i % 7)}/w${String(i).padStart(4, '0')}`, +).sort() +const line = (id: string, v: number) => `${id} v${v} ${'x'.repeat(80)}` + +/** Every level's packs, checked apart and in key order; and the level each says it's in. */ +async function levelPacks(l: KeyedLayout) { + const m = await l.manifests.latest() + const out = new Map() + for (const [lv, shards] of Object.entries(m.levels ?? {})) { + const ps = await l.levels.packs(shards) + for (let i = 1; i < ps.length; i++) + expect(compareUtf8(ps[i]!.lo!, ps[i - 1]!.hi!), `level ${lv} overlaps`).toBeGreaterThan(0) + for (const p of ps) expect(p.level).toBe(Number(lv)) + for (const s of shards) { + const mine = await l.levels.read(s.id) + expect(s.lo).toBe(mine[0]!.lo) + expect(s.hi).toBe(mine.at(-1)!.hi) + } + if (ps.length) out.set(Number(lv), ps) + } + return { m, levels: out } +} + +/** Commits of a few records each, some ids new, some changed; their versions. */ +async function commits(l: KeyedLayout, n: number, seed = 1) { + let s = seed + const rnd = () => (s = (s * 1103515245 + 12345) % 2 ** 31) / 2 ** 31 + const current = new Map() + const all: [string, number][] = [] + for (let c = 0; c < n; c++) { + const w = l.writer() + const picked = new Set() + while (picked.size < 20) picked.add(ids[Math.floor(rnd() * ids.length)]!) + for (const id of [...picked].sort()) { + const v = (current.get(id) ?? 0) + 1 + current.set(id, v) + all.push([id, v]) + w.add('T', { key: id, hash: `h${v}`, size: 10 }, line(id, v)) + } + await w.finish() + while (await l.maintain()); + await l.maintain() + } + return { current, all } +} + +async function readAll(l: KeyedLayout, versions: Iterable<[string, number]>) { + const vs = [...versions].sort((a, b) => compareUtf8(a[0], b[0]) || a[1] - b[1]) + const got = await l.leafLines({ + hash: 'leaf', + type: 'T', + entries: vs.map(([id, v]) => ({ key: id, hash: `h${v}`, size: 10 })), + }) + expect(got).toEqual(vs.map(([id, v]) => line(id, v))) +} + +describe('leveled packs', () => { + it('merge down level by level, each level apart, every record readable', async () => { + const l = await leveledLayout() + const { current, all } = await commits(l, 40) + const { m, levels } = await levelPacks(l) + // Data went below level 0, into more than one level, and a level holds several packs. + expect(levels.size).toBeGreaterThan(1) + expect([...levels.values()].some((ps) => ps.length > 1)).toBe(true) + expect(m.stats.levelMerges).toBeGreaterThan(0) + expect(m.packs.length).toBeLessThan(levelConfig.level0Packs) + // The current versions, and every older one, through the index. + await readAll(l, current) + await readAll(l, all) + }) + + it('move replaced versions to history packs, out of the levels', async () => { + const l = await leveledLayout() + const { current } = await commits(l, 40) + const { levels } = await levelPacks(l) + const history = (await l.allPacks()).filter((p) => p.history) + expect(history.length).toBeGreaterThan(0) + expect(history.every((p) => p.history === new Date().toISOString().slice(0, 7))).toBe(true) + // The levels hold about one version an id: the newest, or one a merge hasn't met yet. + const inLevels = [...levels.values()].flat().reduce((n, p) => n + p.records, 0) + expect(inLevels).toBeLessThan(current.size * 1.5) + }) + + it('take a load sent in key order as the bottom level, as written', async () => { + const l = await leveledLayout() + const w = l.uploadWriter() + for (const id of ids) w.append({ type: 'T', id, hash: 'h1', line: line(id, 1) }) + // Sent in two parts, so the load is two packs that don't overlap. + const packs = [...(await w.finish())] + const w2 = l.uploadWriter() + for (const id of ids) w2.append({ type: 'U', id, hash: 'h1', line: line(id, 1) }) + packs.push(...(await w2.finish())) + expect(packs).toHaveLength(2) + await l.adoptPacks('load', packs) + const { m, levels } = await levelPacks(l) + expect(m.packs).toEqual([]) + expect( + levels + .get(MAX_LEVEL)! + .map((p) => p.id) + .sort(), + ).toEqual(packs.map((p) => p.id).sort()) + // A retry lists nothing again. + await l.adoptPacks('load', packs) + expect((await levelPacks(l)).levels.get(MAX_LEVEL)!.length).toBe(packs.length) + // An upload out of order into a collection holding data goes to level 0. + const u = l.uploadWriter() + for (const id of [ids[5]!, ids[1]!]) u.append({ type: 'T', id, hash: 'h2', line: line(id, 2) }) + const more = await u.finish() + await l.adoptPacks('more', more) + expect((await l.manifests.latest()).packs.map((p) => p.id).sort()).toEqual( + more.map((p) => p.id).sort(), + ) + }) + + it('keep every record readable while commits land during merges', async () => { + const l = await leveledLayout() + const current = new Map() + for (let c = 0; c < 30; c++) { + const w = l.writer() + const batch = ids.slice((c * 13) % 380, ((c * 13) % 380) + 12) + for (const id of batch) { + const v = (current.get(id) ?? 0) + 1 + current.set(id, v) + w.add('T', { key: id, hash: `h${v}`, size: 10 }, line(id, v)) + } + await Promise.all([w.finish(), l.maintain(), l.maintain()]) + } + while (await l.maintain()); + await levelPacks(l) + await readAll(l, current) + }) + + it('reclaim drops what nothing keeps, from the levels and history alike', async () => { + const l = await leveledLayout() + const { current, all } = await commits(l, 40) + const before = (await l.allPacks()).length + // Keep the current versions only: older ones (mostly in history) are dropped. + const stats = await l.reclaim({ + records: [...current].map(([id, v]) => `T\u0000${id}\u0000h${v}`), + } as never) + expect(stats.recordsDropped).toBe(all.length - current.size) + while (await l.maintain()); + await levelPacks(l) + await readAll(l, current) + expect((await l.allPacks()).length).toBeLessThan(before) + }) + + it('fork as leveled, the fork reading its parent and its merges leaving the parent alone', async () => { + const root = memoryStore() + const parent = await leveledLayout(root, 'p') + const { current } = await commits(parent, 20) + const fork = layout(root, 'f') + await parent.forkInto(fork) + expect(await fork.leveled()).toBe(true) + await readAll(fork, current) + // The fork changes and merges; the parent reads as before. + const more = await commits(fork, 20, 7) + await levelPacks(fork) + await readAll(parent, current) + const merged = new Map(current) + for (const [id, v] of more.current) merged.set(id, Math.max(v, merged.get(id) ?? 0)) + expect((await levelPacks(parent)).levels.size).toBeGreaterThan(0) + }) + + it('level an empty collection at its first write once switched on, but not one holding packs', async () => { + const root = memoryStore() + const older = layout(root, 'old') + await commits(older, 3) + layoutConfig.levelNew = true + try { + // Opened again (a new isolate): it held packs before leveling, so it stays tiered. + expect(await layout(root, 'old').leveled()).toBe(false) + const fresh = layout(root, 'new') + const { current } = await commits(fresh, 30) + expect(await layout(root, 'new').leveled()).toBe(true) + expect((await levelPacks(fresh)).levels.size).toBeGreaterThan(0) + await readAll(fresh, current) + } finally { + layoutConfig.levelNew = false + } + }) + + it('leave a collection that is not leveled tiered, as before', async () => { + const l = layout() + await commits(l, 10) + const m = await l.manifests.latest() + expect(m.levels).toBeUndefined() + expect(await l.leveled()).toBe(false) + }) +}) diff --git a/packages/server/drizzle/0028_push_pack_ranges.sql b/packages/server/drizzle/0028_push_pack_ranges.sql new file mode 100644 index 0000000..ed0512e --- /dev/null +++ b/packages/server/drizzle/0028_push_pack_ranges.sql @@ -0,0 +1,2 @@ +ALTER TABLE `push_packs` ADD `lo_key` text;--> statement-breakpoint +ALTER TABLE `push_packs` ADD `hi_key` text; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0028_snapshot.json b/packages/server/drizzle/meta/0028_snapshot.json new file mode 100644 index 0000000..d668937 --- /dev/null +++ b/packages/server/drizzle/meta/0028_snapshot.json @@ -0,0 +1,3703 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "715b9c5b-f4b6-4fa6-8d5a-b7a936a3e6d9", + "prevId": "6fed31a6-5812-4a14-a6b3-b5bc8df442b5", + "tables": { + "abuse_reports": { + "name": "abuse_reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "contact": { + "name": "contact", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reporter_id": { + "name": "reporter_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "resolved_by": { + "name": "resolved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "abuse_reports_status_idx": { + "name": "abuse_reports_status_idx", + "columns": ["status", "created_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 86400000 + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 10 + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_request": { + "name": "last_request", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": ["key"], + "isUnique": false + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": ["reference_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_collections": { + "name": "ark_collections", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "ark_id": { + "name": "ark_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "custom_url": { + "name": "custom_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_collections_ark_id_unique": { + "name": "ark_collections_ark_id_unique", + "columns": ["ark_id"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_collections_collection_id_collections_id_fk": { + "name": "ark_collections_collection_id_collections_id_fk", + "tableFrom": "ark_collections", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_record_types": { + "name": "ark_record_types", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirect_url_field": { + "name": "redirect_url_field", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "ark_record_types_collection_id_collections_id_fk": { + "name": "ark_record_types_collection_id_collections_id_fk", + "tableFrom": "ark_record_types", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "ark_record_types_collection_id_record_type_pk": { + "columns": ["collection_id", "record_type"], + "name": "ark_record_types_collection_id_record_type_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_shoulders": { + "name": "ark_shoulders", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "shoulder": { + "name": "shoulder", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_shoulders_shoulder_unique": { + "name": "ark_shoulders_shoulder_unique", + "columns": ["shoulder"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_shoulders_organization_id_organization_id_fk": { + "name": "ark_shoulders_organization_id_organization_id_fk", + "tableFrom": "ark_shoulders", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cleanup_runs": { + "name": "cleanup_runs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "step": { + "name": "step", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dry_run": { + "name": "dry_run", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "requested_by": { + "name": "requested_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mark_run_id": { + "name": "mark_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lease": { + "name": "lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stats": { + "name": "stats", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "cleanup_runs_created_idx": { + "name": "cleanup_runs_created_idx", + "columns": ["created_at"], + "isUnique": false + }, + "cleanup_runs_step_idx": { + "name": "cleanup_runs_step_idx", + "columns": ["step", "status"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_tombstones": { + "name": "collection_tombstones", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "versions": { + "name": "versions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_by": { + "name": "deleted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_layout": { + "name": "record_layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'today'" + }, + "records_dropped_at": { + "name": "records_dropped_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_webhooks": { + "name": "collection_webhooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bump_filter": { + "name": "bump_filter", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_delivery_at": { + "name": "last_delivery_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "collection_webhooks_collection_idx": { + "name": "collection_webhooks_collection_idx", + "columns": ["collection_id"], + "isUnique": false + } + }, + "foreignKeys": { + "collection_webhooks_collection_id_collections_id_fk": { + "name": "collection_webhooks_collection_id_collections_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collections": { + "name": "collections", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "head_version_id": { + "name": "head_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_salt": { + "name": "private_salt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_files_root": { + "name": "public_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "version_count": { + "name": "version_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "history_bytes": { + "name": "history_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_push_at": { + "name": "last_push_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_started_at": { + "name": "reconcile_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_state": { + "name": "reconcile_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_seq": { + "name": "reconciled_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "reconciled_files_root": { + "name": "reconciled_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_layout": { + "name": "record_layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'today'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "collections_org_slug_uq": { + "name": "collections_org_slug_uq", + "columns": ["organization_id", "slug"], + "isUnique": true + }, + "collections_public_updated_idx": { + "name": "collections_public_updated_idx", + "columns": ["public", "updated_at"], + "isUnique": false + } + }, + "foreignKeys": { + "collections_organization_id_organization_id_fk": { + "name": "collections_organization_id_organization_id_fk", + "tableFrom": "collections", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "commit_units": { + "name": "commit_units", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "plan_id": { + "name": "plan_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ord": { + "name": "ord", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "after": { + "name": "after", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "through": { + "name": "through", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "gap": { + "name": "gap", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "slices_key": { + "name": "slices_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_key": { + "name": "output_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "queued_at": { + "name": "queued_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "commit_units_plan_idx": { + "name": "commit_units_plan_idx", + "columns": ["plan_id", "set", "type", "ord"], + "isUnique": false + }, + "commit_units_status_idx": { + "name": "commit_units_status_idx", + "columns": ["session_id", "status", "queued_at"], + "isUnique": false + } + }, + "foreignKeys": { + "commit_units_session_id_push_sessions_id_fk": { + "name": "commit_units_session_id_push_sessions_id_fk", + "tableFrom": "commit_units", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "denylist": { + "name": "denylist", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "file_uploads": { + "name": "file_uploads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "multipart_upload_id": { + "name": "multipart_upload_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "file_uploads_hash_idx": { + "name": "file_uploads_hash_idx", + "columns": ["hash"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "files": { + "name": "files", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "forks": { + "name": "forks", + "columns": { + "child_collection_id": { + "name": "child_collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "parent_collection_id": { + "name": "parent_collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_seq": { + "name": "parent_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'public'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "forks_child_collection_id_collections_id_fk": { + "name": "forks_child_collection_id_collections_id_fk", + "tableFrom": "forks", + "tableTo": "collections", + "columnsFrom": ["child_collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instance_settings": { + "name": "instance_settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invitation": { + "name": "invitation", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "jobs": { + "name": "jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "run_at": { + "name": "run_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locked_until": { + "name": "locked_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "jobs_ready_idx": { + "name": "jobs_ready_idx", + "columns": ["status", "run_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "member": { + "name": "member", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + }, + "member_user_id_idx": { + "name": "member_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organization": { + "name": "organization", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ark_naan": { + "name": "ark_naan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kf_org_id": { + "name": "kf_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "columns": ["slug"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "placements": { + "name": "placements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location_id": { + "name": "location_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "synced_seq": { + "name": "synced_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until": { + "name": "lease_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "placements_one_primary_uq": { + "name": "placements_one_primary_uq", + "columns": ["collection_id"], + "isUnique": true, + "where": "\"placements\".\"role\" = 'primary' AND \"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_collection_location_uq": { + "name": "placements_collection_location_uq", + "columns": ["collection_id", "location_id"], + "isUnique": true, + "where": "\"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_org_location_uq": { + "name": "placements_org_location_uq", + "columns": ["organization_id", "location_id"], + "isUnique": true, + "where": "\"placements\".\"organization_id\" IS NOT NULL" + }, + "placements_location_idx": { + "name": "placements_location_idx", + "columns": ["location_id"], + "isUnique": false + } + }, + "foreignKeys": { + "placements_collection_id_collections_id_fk": { + "name": "placements_collection_id_collections_id_fk", + "tableFrom": "placements", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_organization_id_organization_id_fk": { + "name": "placements_organization_id_organization_id_fk", + "tableFrom": "placements", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_location_id_storage_locations_id_fk": { + "name": "placements_location_id_storage_locations_id_fk", + "tableFrom": "placements", + "tableTo": "storage_locations", + "columnsFrom": ["location_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_packs": { + "name": "push_packs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pack_id": { + "name": "pack_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_set": { + "name": "record_set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "records": { + "name": "records", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw": { + "name": "raw", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lo_key": { + "name": "lo_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hi_key": { + "name": "hi_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "push_packs_session_id_push_sessions_id_fk": { + "name": "push_packs_session_id_push_sessions_id_fk", + "tableFrom": "push_packs", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_packs_session_id_pack_id_pk": { + "columns": ["session_id", "pack_id"], + "name": "push_packs_session_id_pack_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_runs": { + "name": "push_runs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "first_key": { + "name": "first_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_key": { + "name": "last_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "merging_into": { + "name": "merging_into", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "push_runs_tier_idx": { + "name": "push_runs_tier_idx", + "columns": ["session_id", "tier", "merging_into"], + "isUnique": false + } + }, + "foreignKeys": { + "push_runs_session_id_push_sessions_id_fk": { + "name": "push_runs_session_id_push_sessions_id_fk", + "tableFrom": "push_runs", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_runs_session_id_seq_pk": { + "columns": ["session_id", "seq"], + "name": "push_runs_session_id_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_sessions": { + "name": "push_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_version_id": { + "name": "base_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "strip_unknown_fields": { + "name": "strip_unknown_fields", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "records_received": { + "name": "records_received", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "runs": { + "name": "runs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "result": { + "name": "result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finalize_started_at": { + "name": "finalize_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_started": { + "name": "commit_started", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "in_key_order": { + "name": "in_key_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "overlapping_uploads": { + "name": "overlapping_uploads", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cleaned_at": { + "name": "cleaned_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_phase": { + "name": "commit_phase", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "progress_at": { + "name": "progress_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stalls": { + "name": "stalls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "commit_plan": { + "name": "commit_plan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "assembly_lease": { + "name": "assembly_lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "push_sessions_collection_idx": { + "name": "push_sessions_collection_idx", + "columns": ["collection_id"], + "isUnique": false + }, + "push_sessions_expires_idx": { + "name": "push_sessions_expires_idx", + "columns": ["status", "expires_at"], + "isUnique": false + }, + "push_sessions_user_idx": { + "name": "push_sessions_user_idx", + "columns": ["user_id", "status"], + "isUnique": false + } + }, + "foreignKeys": { + "push_sessions_collection_id_collections_id_fk": { + "name": "push_sessions_collection_id_collections_id_fk", + "tableFrom": "push_sessions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compaction_parts": { + "name": "ref_compaction_parts", + "columns": { + "compaction_id": { + "name": "compaction_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "part": { + "name": "part", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_compaction_parts_compaction_id_part_pk": { + "columns": ["compaction_id", "part"], + "name": "ref_compaction_parts_compaction_id_part_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compactions": { + "name": "ref_compactions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_runs": { + "name": "input_runs", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output_run": { + "name": "output_run", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parts": { + "name": "parts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix_length": { + "name": "prefix_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'running'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_index_units": { + "name": "ref_index_units", + "columns": { + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unit": { + "name": "unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "events": { + "name": "events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_index_units_version_id_unit_pk": { + "columns": ["version_id", "unit"], + "name": "ref_index_units_version_id_unit_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_segments": { + "name": "ref_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_hash": { + "name": "first_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_hash": { + "name": "last_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'live'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ref_segments_range_idx": { + "name": "ref_segments_range_idx", + "columns": ["state", "first_hash", "last_hash"], + "isUnique": false + }, + "ref_segments_run_idx": { + "name": "ref_segments_run_idx", + "columns": ["run_id"], + "isUnique": false + }, + "ref_segments_tier_idx": { + "name": "ref_segments_tier_idx", + "columns": ["tier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_labels": { + "name": "schema_labels", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "schema_labels_label_idx": { + "name": "schema_labels_label_idx", + "columns": ["label"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "schema_labels_schema_hash_label_pk": { + "columns": ["schema_hash", "label"], + "name": "schema_labels_schema_hash_label_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_usage": { + "name": "schema_usage", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type_slug": { + "name": "type_slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_seq": { + "name": "from_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "to_seq": { + "name": "to_seq", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "schema_usage_hash_idx": { + "name": "schema_usage_hash_idx", + "columns": ["schema_hash"], + "isUnique": false + } + }, + "foreignKeys": { + "schema_usage_collection_id_collections_id_fk": { + "name": "schema_usage_collection_id_collections_id_fk", + "tableFrom": "schema_usage", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "schema_usage_collection_id_type_slug_set_from_seq_pk": { + "columns": ["collection_id", "type_slug", "set", "from_seq"], + "name": "schema_usage_collection_id_type_slug_set_from_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schemas": { + "name": "schemas", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": ["token"], + "isUnique": true + }, + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_fence": { + "name": "storage_fence", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "epoch": { + "name": "epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "window_until": { + "name": "window_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "window_run_id": { + "name": "window_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_locations": { + "name": "storage_locations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucket": { + "name": "bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "credentials": { + "name": "credentials", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unverified'" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checked_at": { + "name": "checked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "storage_locations_organization_id_organization_id_fk": { + "name": "storage_locations_organization_id_organization_id_fk", + "tableFrom": "storage_locations", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "usage_rollups": { + "name": "usage_rollups", + "columns": { + "day": { + "name": "day", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metric": { + "name": "metric", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "amount": { + "name": "amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "usage_rollups_account_idx": { + "name": "usage_rollups_account_idx", + "columns": ["account_id", "day"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "usage_rollups_day_account_id_collection_id_metric_pk": { + "columns": ["day", "account_id", "collection_id", "metric"], + "name": "usage_rollups_day_account_id_collection_id_metric_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": ["email"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": ["identifier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "versions": { + "name": "versions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "major": { + "name": "major", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "minor": { + "name": "minor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "patch": { + "name": "patch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pushed_by": { + "name": "pushed_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_record_count": { + "name": "public_record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_file_count": { + "name": "public_file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "public_total_bytes": { + "name": "public_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "type_counts": { + "name": "type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_type_counts": { + "name": "public_type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "has_private": { + "name": "has_private", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "public_refs_root": { + "name": "public_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_refs_root": { + "name": "private_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refs_indexed": { + "name": "refs_indexed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "changes": { + "name": "changes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "push_session_id": { + "name": "push_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "versions_collection_seq_uq": { + "name": "versions_collection_seq_uq", + "columns": ["collection_id", "seq"], + "isUnique": true + }, + "versions_collection_semver_uq": { + "name": "versions_collection_semver_uq", + "columns": ["collection_id", "semver"], + "isUnique": true + }, + "versions_hash_idx": { + "name": "versions_hash_idx", + "columns": ["hash"], + "isUnique": false + }, + "versions_published_idx": { + "name": "versions_published_idx", + "columns": ["published_at"], + "isUnique": false + }, + "versions_push_session_idx": { + "name": "versions_push_session_idx", + "columns": ["push_session_id"], + "isUnique": false + } + }, + "foreignKeys": { + "versions_collection_id_collections_id_fk": { + "name": "versions_collection_id_collections_id_fk", + "tableFrom": "versions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhook_deliveries": { + "name": "webhook_deliveries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "webhook_id": { + "name": "webhook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bump_type": { + "name": "bump_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'version.created'" + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "delivered_at": { + "name": "delivered_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhook_deliveries_webhook_idx": { + "name": "webhook_deliveries_webhook_idx", + "columns": ["webhook_id", "created_at"], + "isUnique": false + }, + "webhook_deliveries_pending_idx": { + "name": "webhook_deliveries_pending_idx", + "columns": ["status", "next_attempt_at"], + "isUnique": false + } + }, + "foreignKeys": { + "webhook_deliveries_webhook_id_collection_webhooks_id_fk": { + "name": "webhook_deliveries_webhook_id_collection_webhooks_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collection_webhooks", + "columnsFrom": ["webhook_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index ca816c2..11a6e3d 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -197,6 +197,13 @@ "when": 1792605564163, "tag": "0027_push_pack_types", "breakpoints": true + }, + { + "idx": 28, + "version": "6", + "when": 1792606671953, + "tag": "0028_push_pack_ranges", + "breakpoints": true } ] } diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index bb4a59b..6ec7161 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -701,6 +701,13 @@ export const pushPacks = sqliteTable( raw: integer('raw'), /** The record type the pack holds (PackInfo.type): merges take one type's packs. */ recordType: text('record_type'), + /** + * Its first and last keys (PackInfo.lo, hi), as JSON strings (a key's NULs would + * end SQLite text): a leveled collection takes a sorted load's packs, none + * overlapping, as its bottom level. + */ + loKey: text('lo_key'), + hiKey: text('hi_key'), createdAt: createdAt(), }, (t) => [primaryKey({ columns: [t.sessionId, t.packId] })], diff --git a/packages/server/src/push/packs.ts b/packages/server/src/push/packs.ts index 8fcaafa..c3473d5 100644 --- a/packages/server/src/push/packs.ts +++ b/packages/server/src/push/packs.ts @@ -36,6 +36,9 @@ const SETS = ['public', 'private'] as const /** An upload marks an entry private exactly when it goes to the private set (deltaChanges). */ const setOf = (e: RunEntry): RecordSet => (e.p ? 'private' : 'public') +/** A record key as SQLite text: JSON, since keys hold NULs and text stops at one. */ +const keyText = (key: string | undefined) => (key === undefined ? undefined : JSON.stringify(key)) + /** Each pack recorded before it's written, and its stats once it is. */ const hooksFor = (ports: Ports, sessionId: string, set: RecordSet): PackHooks => ({ before: async (packId) => { @@ -44,7 +47,14 @@ const hooksFor = (ports: Ports, sessionId: string, set: RecordSet): PackHooks => after: async (pack) => { await ports.db .update(schema.pushPacks) - .set({ records: pack.records, bytes: pack.bytes, raw: pack.raw, recordType: pack.type }) + .set({ + records: pack.records, + bytes: pack.bytes, + raw: pack.raw, + recordType: pack.type, + loKey: keyText(pack.lo), + hiKey: keyText(pack.hi), + }) .where(and(eq(schema.pushPacks.sessionId, sessionId), eq(schema.pushPacks.packId, pack.id))) }, }) @@ -133,6 +143,9 @@ export async function adoptSessionPacks(ports: Ports, repo: Repo, sessionId: str bytes: r.bytes!, raw: r.raw!, ...(r.recordType ? { type: r.recordType } : {}), + ...(r.loKey && r.hiKey + ? { lo: JSON.parse(r.loKey) as string, hi: JSON.parse(r.hiKey) as string } + : {}), })) if (packs.length) await repo.records[set].adoptPacks!(sessionId, packs) } diff --git a/packages/server/src/stores.ts b/packages/server/src/stores.ts index daf1340..fd920ea 100644 --- a/packages/server/src/stores.ts +++ b/packages/server/src/stores.ts @@ -9,6 +9,7 @@ import { type LayoutName, type Cache, + layoutConfig, PrefixedStore, type PresigningStore, Repo, @@ -42,6 +43,11 @@ export interface PlatformStorage { * parent's. */ newCollections?: LayoutName + /** + * Layout F: level a collection's packs (leveled-packs-design.md) when its first write + * finds it empty. A collection holding packs already stays tiered. + */ + leveledPacks?: boolean } /** Primary placements change rarely, a collection's layout only by a conversion; remember them briefly per isolate. */ @@ -60,6 +66,7 @@ export function createStores(db: Db, cache: Cache, platform: PlatformStorage): S /** F's, by location and collection; a Map's order is its recency (a hit moves to the end). */ const runRepos = new Map() const files = platform.files ?? presigning(platform.bucket) + layoutConfig.levelNew = platform.leveledPacks === true const forLocation = async (locationId: string): Promise => { const known = repos.get(locationId) diff --git a/packages/server/src/worker.ts b/packages/server/src/worker.ts index bcfff27..f83f129 100644 --- a/packages/server/src/worker.ts +++ b/packages/server/src/worker.ts @@ -66,6 +66,8 @@ export interface Env { AUTH_INTERNAL_API_KEY?: string REPO_PREFIX?: string INTERNAL_PREFIX?: string + /** '1': layout F levels new collections' packs (stores.ts `leveledPacks`). */ + LEVELED_PACKS?: string /** * Public assets (org logos): a world-readable bucket served at ASSETS_BASE_URL. * Reached through the PUBLIC_ASSETS binding when there is one, otherwise @@ -136,6 +138,7 @@ function makePorts(env: Env, ctx: ExecutionContext, req?: Request): Ports { files: s3, repoPrefix: env.REPO_PREFIX ?? 'repo', internalPrefix: env.INTERNAL_PREFIX ?? 'internal', + leveledPacks: env.LEVELED_PACKS === '1', // The bulk Worker's isolates run jobs, whose reads rarely repeat. ...(env.ROLE === 'bulk' ? { memoryBytes: 8 * 1024 * 1024 } : {}), }), diff --git a/packages/server/test/cleanup.test.ts b/packages/server/test/cleanup.test.ts index 594b2a3..d70ea84 100644 --- a/packages/server/test/cleanup.test.ts +++ b/packages/server/test/cleanup.test.ts @@ -1,6 +1,6 @@ import { createHash } from 'node:crypto' -import { fsck, newSalt, openRepo } from '@underlay/protocol' +import { fsck, layoutConfig, newSalt, openRepo } from '@underlay/protocol' import { eq, sql } from 'drizzle-orm' import { afterAll, beforeEach, describe, expect, it } from 'vitest' @@ -308,7 +308,8 @@ describe('storage cleanup, steps 2 and 3: mark and sweep', () => { const fresh = await h.collection('new') const puts = h.bucket.puts await push(h, user, 'new', records, 'the pdf') - expect(h.bucket.puts - puts).toBeLessThan(15) // mostly reused + // Mostly reused (leveled, the new collection's mark is one more). + expect(h.bucket.puts - puts).toBeLessThan(layoutConfig.levelNew ? 17 : 15) ageAll(h) const sweep = await run(h, 'sweep') diff --git a/packages/server/test/harness.ts b/packages/server/test/harness.ts index e9dad08..ef16ba4 100644 --- a/packages/server/test/harness.ts +++ b/packages/server/test/harness.ts @@ -89,6 +89,7 @@ export async function harness(opts: { newCollections?: LayoutName } = {}): Promi repoPrefix: 'repo', internalPrefix: 'internal', newCollections: opts.newCollections ?? testLayout, + leveledPacks: process.env.UNDERLAY_TEST_LEVELED === '1', }), cache, signer: async () => signer, diff --git a/packages/server/test/manage.test.ts b/packages/server/test/manage.test.ts index 06f36ca..63ef4f3 100644 --- a/packages/server/test/manage.test.ts +++ b/packages/server/test/manage.test.ts @@ -1,4 +1,4 @@ -import { readHead, verifyLog } from '@underlay/protocol' +import { layoutConfig, readHead, verifyLog } from '@underlay/protocol' import { eq } from 'drizzle-orm' import { afterAll, describe, expect, it } from 'vitest' @@ -136,9 +136,10 @@ describe('collection management', () => { version: { recordCount: 1 }, }) // Root, log entry, head.json; layout F adds the fork's list of its parent's files, a - // marker in the parent, and the fork's manifest and its head (the public set's alone). + // marker in the parent, and the fork's manifest and its head (the public set's alone); + // leveled, the fork's leveled mark and its level's shard. expect(h.bucket.puts - putsBefore).toBeLessThanOrEqual( - h.ports.stores.newCollections === 'f' ? 8 : 4, + h.ports.stores.newCollections === 'f' ? (layoutConfig.levelNew ? 10 : 8) : 4, ) const [child] = await h.ports.db .select() diff --git a/packages/server/test/upload-packs.test.ts b/packages/server/test/upload-packs.test.ts index 58b0fc9..3fe0fdc 100644 --- a/packages/server/test/upload-packs.test.ts +++ b/packages/server/test/upload-packs.test.ts @@ -6,7 +6,14 @@ */ import { createHash } from 'node:crypto' -import { countTree, iterate, type RecordSet, RepoSource, recordTree } from '@underlay/protocol' +import { + countTree, + iterate, + layoutConfig, + type RecordSet, + RepoSource, + recordTree, +} from '@underlay/protocol' import { eq } from 'drizzle-orm' import { afterAll, afterEach, describe, expect, it } from 'vitest' @@ -274,7 +281,29 @@ describe.runIf(process.env.UNDERLAY_TEST_LAYOUT === 'f')('uploads into layout F' ) expect(await commit(h, user, 'l', sid)).toBe(201) const registry = `repo/runs/${c.id}/public/manifest/f/packs/upload-${sid}.json` - expect(h.bucket.objects.has(registry)).toBe(true) + // Leveled, the first load (into an empty collection, none of its packs overlapping) + // is the bottom level instead, as written. + expect(h.bucket.objects.has(registry)).toBe(!layoutConfig.levelNew) + const repo = await h.ports.stores.forCollection(c.id) + const layout = repo.records!.public as unknown as { + manifests: { + latest(): Promise<{ + packs: { id: string; tier: number; type?: string }[] + levels?: Record + }> + } + levels: { packs(s: { id: string }[]): Promise<{ lo?: string; hi?: string }[]> } + } + if (layoutConfig.levelNew) { + const bottom = (await layout.manifests.latest()).levels?.[6] + expect(bottom).toHaveLength(1) + // Its key range whole, NULs and all, through the database. + const [p] = await layout.levels.packs(bottom!) + expect([p!.lo!.split('\u0000').slice(0, 2), p!.hi!.split('\u0000').slice(0, 2)]).toEqual([ + ['Doc', id(0)], + ['Doc', id(599)], + ]) + } // Small pushes: their packs go in the manifest, and merges re-point their records. for (let j = 0; j < 10; j++) { const s = await open(h, user, 'l') @@ -284,15 +313,11 @@ describe.runIf(process.env.UNDERLAY_TEST_LAYOUT === 'f')('uploads into layout F' const { out } = await headRecords(h, c.id) expect(out).toHaveLength(610) expect(out.filter((r) => /changed|small/.test(r))).toHaveLength(20) - const repo = await h.ports.stores.forCollection(c.id) - const layout = repo.records!.public as unknown as { - manifests: { - latest(): Promise<{ packs: { id: string; tier: number; type?: string }[] }> - } - } const m = await layout.manifests.latest() - // Every small pack listed, or merged into one above them, and each says its type. - expect(m.packs.some((p) => p.tier > 0) || m.packs.length >= 10).toBe(true) + // Every small pack listed, or merged into one above them (leveled: all ten in level + // 0, under its 16-pack trigger), and each says its type. + if (layoutConfig.levelNew) expect(m.packs).toHaveLength(10) + else expect(m.packs.some((p) => p.tier > 0) || m.packs.length >= 10).toBe(true) expect(m.packs.every((p) => p.type === 'Doc')).toBe(true) expect(new Set(m.packs.map((p) => p.id)).size).toBe(m.packs.length) })