From 1d3cfa4e92373bc5f89ace253587fb196df42409 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 20 Oct 2026 19:51:10 +0200 Subject: [PATCH 1/6] Layout F: uploads write records straight into packs; runs carry their places --- packages/protocol/src/index.ts | 4 + packages/protocol/src/repo/file-sets.ts | 13 +- packages/protocol/src/repo/layouts/index.ts | 2 + packages/protocol/src/repo/layouts/keyed.ts | 107 +- packages/protocol/src/repo/layouts/kit.ts | 32 +- packages/protocol/src/repo/layouts/types.ts | 36 +- packages/protocol/src/repo/repo.ts | 11 +- packages/protocol/test/repo/layouts.test.ts | 92 + packages/server/drizzle/0026_push_packs.sql | 13 + .../server/drizzle/meta/0026_snapshot.json | 3666 +++++++++++++++++ packages/server/drizzle/meta/_journal.json | 7 + packages/server/src/api/push.ts | 2 + packages/server/src/cleanup/internal.ts | 14 +- packages/server/src/db/schema.ts | 30 + packages/server/src/push/changes.ts | 14 +- packages/server/src/push/delta.ts | 24 +- packages/server/src/push/packs.ts | 157 + packages/server/src/push/runs.ts | 6 +- packages/server/src/push/session.ts | 5 + packages/server/src/versions/commit.ts | 4 + packages/server/test/cleanup.test.ts | 76 +- packages/server/test/stream-push.test.ts | 5 +- packages/server/test/upload-packs.test.ts | 365 ++ 23 files changed, 4620 insertions(+), 65 deletions(-) create mode 100644 packages/server/drizzle/0026_push_packs.sql create mode 100644 packages/server/drizzle/meta/0026_snapshot.json create mode 100644 packages/server/src/push/packs.ts create mode 100644 packages/server/test/upload-packs.test.ts diff --git a/packages/protocol/src/index.ts b/packages/protocol/src/index.ts index 9b78817f..704c036b 100644 --- a/packages/protocol/src/index.ts +++ b/packages/protocol/src/index.ts @@ -20,9 +20,13 @@ export { type LiveSet, type MergePlan, type PackInfo, + packIdsFor, + type PackRecord, + type Placed, MissingRecordError, type ReclaimPlan, recordKey, + UPLOAD_PACK_FINAL_RAW, type RecordSet, type RecordSets, setHome, diff --git a/packages/protocol/src/repo/file-sets.ts b/packages/protocol/src/repo/file-sets.ts index 854c290a..9816e708 100644 --- a/packages/protocol/src/repo/file-sets.ts +++ b/packages/protocol/src/repo/file-sets.ts @@ -38,6 +38,7 @@ import { type SetObject, type TreeSummary, } from '../format.js' +import type { Placed } from './layouts/types.js' import { outOfLineHash, type Repo, RepoSink, RepoSource } from './repo.js' export type SetName = 'public' | 'private' @@ -103,8 +104,16 @@ export class FileRefDelta { /** Feed one record change of a type in a set (mergeTree's onChange). */ record(set: SetName, before: RecordEntry | null, after: RecordEntry | null, type: string): void { - if (before && !this.noOldRefs.has(set)) this.#body(set, type, before.body, -1) - if (after) this.#body(set, type, after.body, +1) + if (before && !this.noOldRefs.has(set)) this.#entry(set, type, before, -1) + if (after) this.#entry(set, type, after, +1) + } + + #entry(set: SetName, type: string, e: RecordEntry, by: number) { + // Written into a pack at upload: its references were read from it then. + const p = e as Placed + if (p.body === undefined && p.packed !== undefined) { + for (const h of p.fileRefs ?? []) this.add(set, type, h, by) + } else this.#body(set, type, e.body, by) } /** diff --git a/packages/protocol/src/repo/layouts/index.ts b/packages/protocol/src/repo/layouts/index.ts index d6e153db..b7c1e21b 100644 --- a/packages/protocol/src/repo/layouts/index.ts +++ b/packages/protocol/src/repo/layouts/index.ts @@ -11,7 +11,9 @@ export { layoutConfig, type MergePlan, MissingRecordError, + packIdsFor, type ReclaimPlan, + UPLOAD_PACK_FINAL_RAW, } from './keyed.js' export { headConfig, type LayerInfo, type PackInfo, pendingConfig } from './kit.js' export * from './types.js' diff --git a/packages/protocol/src/repo/layouts/keyed.ts b/packages/protocol/src/repo/layouts/keyed.ts index ff47313b..7d32f4cf 100644 --- a/packages/protocol/src/repo/layouts/keyed.ts +++ b/packages/protocol/src/repo/layouts/keyed.ts @@ -36,6 +36,7 @@ import { newId, type PackInfo, PackReader, + PACK_RAW, PackWriter, packDirKey, packKey, @@ -47,7 +48,13 @@ import { filterKey, } from './kit.js' import { type KeyRange, LiveHashes } from './reclaim.js' -import { type LiveSet, recordKey, type RecordStore, type RecordWriter } from './types.js' +import { + type LiveSet, + type PackRecord, + recordKey, + type RecordStore, + type RecordWriter, +} from './types.js' /** Index entries a reclaim range job works on: its live set is 8 bytes an entry. */ export const RECLAIM_RANGE_ENTRIES = 250_000 @@ -113,6 +120,27 @@ const MAX_RUNS = 10 /** Packs under this many stored bytes, times 8 per tier, are merged eight at a time. */ const SMALL_PACK = 4 * 1024 * 1024 const MAX_PACK_TIER = 3 +/** + * An upload's pack this large (raw) is listed with the packs no merge takes: a bulk + * load's batches (8 MB each) are written once, where they stay. Merging them was + * the external sort again, one pack at a time on the set's one maintain lane. + * Smaller ones (a small push's) are merged like a commit's. + */ +export const UPLOAD_PACK_FINAL_RAW = 1024 * 1024 +/** Commits whose uploads' packs a manifest remembers listing (`Manifest.adopted`). */ +const ADOPTED_KEPT = 1000 + +/** + * Ids for the packs `writePacks` makes of these records, recorded before they're + * written (so an upload that dies leaves nothing unknown). A pack closes only once + * it holds PACK_RAW (PackWriter, the same count of a line), so this many at most. + */ +export function packIdsFor(records: readonly PackRecord[]): string[] { + let raw = 0 + for (const r of records) raw += r.id.length + r.hash.length + r.line.length + 3 + return Array.from({ length: Math.floor(raw / PACK_RAW) + 1 }, newId) +} + /** How long a replaced object stays readable (readers holding an older manifest). */ export const layoutConfig = { graceMs: 60_000, @@ -272,6 +300,78 @@ export class KeyedLayout implements RecordStore { return new KeyedWriter(this) } + /** + * An upload's records into packs of their own, in key order, outside the index and + * the manifest until a commit names them (adoptPacks). Their places, in the order + * given, go in the push's runs, so a commit indexes the records without moving them. + */ + async writePacks( + records: readonly PackRecord[], + ids: readonly string[], + ): Promise<{ places: string[]; packs: PackInfo[] }> { + let next = 0 + const pw = new PackWriter(this.store, { + ids: () => { + const id = ids[next++] + if (id === undefined) throw new Error('writePacks: the records need more packs than ids') + return id + }, + }) + const keys = records.map((r) => recordKey(r.id, r.hash)) + const order = keys.map((_, i) => i).sort((a, b) => compareUtf8(keys[a]!, keys[b]!)) + const places: string[] = Array.from({ length: records.length }) + let n = 0 + for (const i of order) { + places[i] = pw.append(keys[i]!, records[i]!.line) + if (++n % 1024 === 0) await pw.ready() + } + return { places, packs: await pw.finish() } + } + + /** + * List packs uploads wrote, once the commit's index names their records (after its + * runs are appended: a merge re-points a pack's records in a newer layer, which a + * commit's layer folded in later would otherwise override with the old places). + * Large ones go in the registry, small ones in the manifest for merges. Once only + * a `name`: a retry finds it in `adopted` and lists nothing. + */ + async adoptPacks(name: string, packs: readonly PackInfo[]): Promise { + if (!packs.length) return + if ((await this.manifests.latest()).adopted?.includes(name)) return + const large = (p: PackInfo) => p.raw >= UPLOAD_PACK_FINAL_RAW || KeyedLayout.final(p) + // A fixed key: written once even if this stops before the manifest says so. + await this.registry.add(packs.filter(large), `upload-${name}`) + const small = packs.filter((p) => !large(p)) + await this.manifests.update((m) => { + if (m.adopted?.includes(name)) return null + const have = new Set(m.packs.map((p) => p.id)) + m.packs.push(...small.filter((p) => !have.has(p.id))) + m.adopted = [...(m.adopted ?? []), name].slice(-ADOPTED_KEPT) + return m + }) + } + + /** + * Delete packs an upload wrote for a commit `name` that never named them (an + * abandoned push). A commit that named them and then didn't publish (a conflict, + * then the session failed) leaves them: the manifest or registry lists them, so + * they go only by reclaim. False when kept. + */ + async dropPacks(name: string, ids: readonly string[]): Promise { + const m = await this.manifests.latest() + if (m.adopted?.includes(name)) return false + if (await this.store.head(this.registry.keyFor(`upload-${name}`))) return false + const listed = new Set([...m.packs.map((p) => p.id), ...m.retired.map((r) => r.key)]) + const gone = ids.filter((id) => !listed.has(id) && !listed.has(packKey(id))) + for (let i = 0; i < gone.length; i += 16) + await Promise.all( + gone + .slice(i, i + 16) + .flatMap((id) => [this.store.delete(packKey(id)), this.store.delete(packDirKey(id))]), + ) + return true + } + /** A commit's runs its units held back (RecordWriter.finish), as one pending object. */ appendRuns(runs: PendingRun[]): Promise { return this.manifests.appendRuns(runs) @@ -1229,6 +1329,11 @@ class KeyedWriter implements RecordWriter { return at } + addPlaced(entry: RecordEntry, place: string): string { + this.#entries.push([recordKey(entry.key, entry.hash), place]) + return place + } + async finish(opts: { hold?: boolean } = {}): Promise { const packs = await this.#packs.finish() if (!this.#entries.length) return null diff --git a/packages/protocol/src/repo/layouts/kit.ts b/packages/protocol/src/repo/layouts/kit.ts index bb9ae831..9933b000 100644 --- a/packages/protocol/src/repo/layouts/kit.ts +++ b/packages/protocol/src/repo/layouts/kit.ts @@ -77,14 +77,15 @@ export class PackWriter { constructor( readonly store: Store, - readonly opts: { tier?: number } = {}, + /** `ids` names each new pack (an upload's, recorded before they're written); else random. */ + readonly opts: { tier?: number; ids?: () => string } = {}, ) {} /** Add a record line under `key`; returns its place. A key below the last starts a new pack. */ append(key: string, line: string): string { if (this.#lastKey !== null && compareUtf8(key, this.#lastKey) < 0) this.#close() this.#lastKey = key - if (!this.#id) this.#id = newId() + if (!this.#id) this.#id = this.opts.ids ? this.opts.ids() : newId() this.#lines.push(`${key}\t${line}`) const at = loc(this.#id, this.#blocks.length, this.#lines.length - 1) this.#raw += key.length + line.length + 2 @@ -758,6 +759,11 @@ export interface Manifest { * delete failed must never be folded again once its layers are merged away. */ folded?: string[] + /** + * The latest commits whose uploads' packs were listed (KeyedLayout.adoptPacks): a + * retry lists none again, not even one a merge has since taken and swept. + */ + adopted?: string[] } const empty = (): Manifest => ({ seq: 0, layers: [], packs: [], retired: [], stats: {} }) @@ -812,14 +818,22 @@ export class PackRegistry { return `manifest/${this.name}/packs/` } - /** List `packs` in a new object; its key (null for none). */ - async add(packs: readonly PackInfo[]): Promise { + /** + * List `packs` in a new object; its key (null for none). `name` makes the key + * fixed, so listing the same packs again (a retried commit) writes nothing. + */ + async add(packs: readonly PackInfo[], name: string = newId()): Promise { if (!packs.length) return null - const key = `${this.#prefix}${newId()}.json` + const key = this.keyFor(name) await this.store.put(key, JSON.stringify(packs), { ifAbsent: true }) return key } + /** The key of the registry object `name`. */ + keyFor(name: string): string { + return `${this.#prefix}${name}.json` + } + async read(): Promise<{ keys: string[]; packs: PackInfo[] }> { const keys: string[] = [] let cursor: string | undefined @@ -828,11 +842,13 @@ export class PackRegistry { keys.push(...page.keys) cursor = page.cursor } while (cursor) - const packs: PackInfo[] = [] + // A pack listed twice (a retried commit naming an upload's packs after a reclaim + // replaced their object) is one pack. + const packs = new Map() for (let i = 0; i < keys.length; i += 16) for (const o of await Promise.all(keys.slice(i, i + 16).map((k) => this.store.get(k)))) - if (o) packs.push(...(JSON.parse(await o.text()) as PackInfo[])) - return { keys, packs } + if (o) for (const p of JSON.parse(await o.text()) as PackInfo[]) packs.set(p.id, p) + return { keys, packs: [...packs.values()] } } /** Replace the objects `keys` with one listing `packs`. */ diff --git a/packages/protocol/src/repo/layouts/types.ts b/packages/protocol/src/repo/layouts/types.ts index 8fc847bb..b181d0bc 100644 --- a/packages/protocol/src/repo/layouts/types.ts +++ b/packages/protocol/src/repo/layouts/types.ts @@ -5,13 +5,24 @@ * holds. Neither changes a node, a root or any hash. */ import type { RecordEntry } from '../../tree/node.js' -import type { PendingRun } from './kit.js' +import type { PackInfo, PendingRun } from './kit.js' export const LAYOUTS = ['today', 'f'] as const export type LayoutName = (typeof LAYOUTS)[number] -/** A record entry as layout F carries it through a build: with its place once stored. */ -export type Placed = RecordEntry & { loc?: string } +/** + * A record entry as layout F carries it through a build: with its place once stored. + * A record written into a pack at upload comes without its body: `packed` is its + * place there and `fileRefs` the files it references, read from it at upload. + */ +export type Placed = RecordEntry & { loc?: string; packed?: string; fileRefs?: string[] } + +/** A record an upload writes into a pack: its id, hash and canonical line. */ +export interface PackRecord { + id: string + hash: string + line: string +} /** Records still in use, from a mark over every retained version (storage reclaim). */ export interface LiveSet { @@ -57,11 +68,30 @@ export interface RecordStore { appendRuns?(runs: HeldRun[]): Promise /** Give a fork its records: it lists the parent's files and reads them where they are. */ forkInto?(target: RecordStore): Promise> + /** + * Write an upload's records into packs of their own, named by `ids` (more than it + * needs; see `packIdsFor`), outside the index and the manifest: their places, in + * the order given, and the packs written. + */ + writePacks?( + records: readonly PackRecord[], + ids: readonly string[], + ): Promise<{ places: string[]; packs: PackInfo[] }> + /** + * List packs uploads wrote, once the commit's index names their records: large + * ones in the registry, small ones in the manifest for merges. `name` is the + * commit's, so a retry lists nothing twice. + */ + adoptPacks?(name: string, packs: readonly PackInfo[]): Promise + /** Delete packs uploads wrote for commit `name`, unless it named them (false then). */ + dropPacks?(name: string, ids: readonly string[]): Promise } export interface RecordWriter { /** Store one record line (a spilled or new entry): its place. */ add(entry: RecordEntry, line: string): string + /** Index a record an upload already wrote into a pack, at `place`; returns it. */ + addPlaced(entry: RecordEntry, place: string): string /** * Write what's held: packs, the index layer, then the manifest. With `hold`, the * manifest isn't told: the run is returned, for the commit to name with its others. diff --git a/packages/protocol/src/repo/repo.ts b/packages/protocol/src/repo/repo.ts index c7028659..43a65dc8 100644 --- a/packages/protocol/src/repo/repo.ts +++ b/packages/protocol/src/repo/repo.ts @@ -875,11 +875,16 @@ export class RepoSink implements TreeSink { this.#members.push(this.#member(entries)) } - /** Store a new record in the packed layout, and note its place on the entry. */ + /** + * Store a new record in the packed layout, and note its place on the entry. One an + * upload already wrote into a pack (`packed`) is only indexed there: its bytes + * aren't read or written again. + */ #place(e: E) { const p = e as unknown as Placed - if (p.loc !== undefined || p.body === undefined) return - p.loc = this.#writer!.add(p, p.body) + if (p.loc !== undefined) return + if (p.packed !== undefined) p.loc = this.#writer!.addPlaced(p, p.packed) + else if (p.body !== undefined) p.loc = this.#writer!.add(p, p.body) } leaf(desc: NodeDesc, json: string, entries: readonly E[], spilled: number): void { diff --git a/packages/protocol/test/repo/layouts.test.ts b/packages/protocol/test/repo/layouts.test.ts index c7cae337..7e765c4f 100644 --- a/packages/protocol/test/repo/layouts.test.ts +++ b/packages/protocol/test/repo/layouts.test.ts @@ -25,12 +25,14 @@ import { mergeTree, type NodeSource, openRepo, + packIdsFor, type RecordEntry, recordCanonical, RepoSink, recordTree, sha256Hex, treeObjects, + UPLOAD_PACK_FINAL_RAW, utf8ByteLength, } from '../../src/index.js' import { @@ -595,6 +597,96 @@ describe("a commit's held runs", () => { }) }) +describe("an upload's packs", () => { + const rec = (i: number, v = 1) => ({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 }) + const line = (i: number, v = 1) => `line ${v}.${i}` + const upload = (from: number, to: number, v = 1) => + Array.from({ length: to - from }, (_, j) => { + // Out of key order, as a batch arrives. + const i = to - 1 - j + return { id: ids[i]!, hash: `h${v}.${i}`, line: line(i, v) } + }) + + it('are indexed where they lie, and merges re-point them once listed', async () => { + const saved = layoutConfig.graceMs + layoutConfig.graceMs = 0 + try { + const root = memoryStore() + const layout = fLayout(root) + const packsNow = async () => (await root.list('runs/c1/packs/')).keys.length + // Ten small uploads, each into a pack of its own; their commits index them. + for (let u = 0; u < 10; u++) { + const recs = upload(u * 100, u * 100 + 100) + const ids_ = packIdsFor(recs) + expect(ids_).toHaveLength(1) + const { places, packs } = await layout.writePacks(recs, ids_) + expect(packs.map((p) => p.id)).toEqual(ids_) + const before = await packsNow() + const w = layout.writer() + recs.forEach((r, i) => w.addPlaced(rec(u * 100 + (99 - i)), places[i]!)) + await w.finish() + // The commit wrote no pack: only its index. + expect(await packsNow()).toBe(before) + await layout.adoptPacks(`s${u}`, packs) + } + const all = Array.from({ length: 1000 }, (_, i) => i) + const read = () => + layout.leafLines( + 'leaf', + all.map((i) => rec(i)), + ) + expect(await read()).toEqual(all.map((i) => line(i))) + // Small packs: listed in the manifest, merged, their records re-pointed. + expect((await layout.manifests.latest()).packs).toHaveLength(10) + await layout.maintain() + await new Promise((r) => setTimeout(r, 5)) + await layout.maintain() + const m = await layout.manifests.latest() + expect(m.packs.length).toBeLessThan(10) + expect(await read()).toEqual(all.map((i) => line(i))) + } finally { + layoutConfig.graceMs = saved + } + }) + + it('are listed once however often a commit retries, even after a merge took them', async () => { + const root = memoryStore() + const layout = fLayout(root) + const recs = upload(0, 50) + const { packs } = await layout.writePacks(recs, packIdsFor(recs)) + await layout.adoptPacks('s1', packs) + await layout.adoptPacks('s1', packs) + expect((await layout.manifests.latest()).packs.map((p) => p.id)).toEqual([packs[0]!.id]) + // As if a merge took it: a retry must not list it again. + await layout.manifests.update((m) => ({ ...m, packs: [] })) + await layout.adoptPacks('s1', packs) + expect((await layout.manifests.latest()).packs).toEqual([]) + + // A large one goes in the registry, once. + const large = { ...packs[0]!, raw: UPLOAD_PACK_FINAL_RAW } + await layout.adoptPacks('s2', [large]) + await layout.adoptPacks('s2', [large]) + expect((await layout.registry.read()).packs).toEqual([large]) + }) + + it('are dropped for a push that never listed them, and kept once it did', async () => { + const root = memoryStore() + const layout = fLayout(root) + const named = upload(0, 20) + const a = await layout.writePacks(named, packIdsFor(named)) + await layout.adoptPacks('kept', a.packs) + const left = upload(20, 40) + const b = await layout.writePacks(left, packIdsFor(left)) + const files = async () => (await root.list('runs/c1/packs/')).keys + expect(await files()).toHaveLength(4) + expect(await layout.dropPacks('kept', [a.packs[0]!.id])).toBe(false) + expect(await layout.dropPacks('left', [b.packs[0]!.id, 'neverwritten'])).toBe(true) + expect((await files()).sort()).toEqual( + [`runs/c1/packs/${a.packs[0]!.id}.dir`, `runs/c1/packs/${a.packs[0]!.id}.pk`].sort(), + ) + }) +}) + describe('the maintain lease', () => { it('is left alone when another job took it over', async () => { // A job that ran past its lease, which another job then took: releasing it at diff --git a/packages/server/drizzle/0026_push_packs.sql b/packages/server/drizzle/0026_push_packs.sql new file mode 100644 index 00000000..70d99ec2 --- /dev/null +++ b/packages/server/drizzle/0026_push_packs.sql @@ -0,0 +1,13 @@ +CREATE TABLE `push_packs` ( + `session_id` text NOT NULL, + `pack_id` text NOT NULL, + `record_set` text NOT NULL, + `records` integer, + `bytes` integer, + `raw` integer, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + PRIMARY KEY(`session_id`, `pack_id`), + FOREIGN KEY (`session_id`) REFERENCES `push_sessions`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +ALTER TABLE `push_sessions` ADD `commit_started` integer DEFAULT false NOT NULL; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0026_snapshot.json b/packages/server/drizzle/meta/0026_snapshot.json new file mode 100644 index 00000000..5a88d9a7 --- /dev/null +++ b/packages/server/drizzle/meta/0026_snapshot.json @@ -0,0 +1,3666 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "da1a4cfe-5e4b-4fb0-9240-470d050217f9", + "prevId": "42b4a563-0b06-417e-a83f-2216ffba5eb8", + "tables": { + "abuse_reports": { + "name": "abuse_reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "contact": { + "name": "contact", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reporter_id": { + "name": "reporter_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "resolved_by": { + "name": "resolved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "abuse_reports_status_idx": { + "name": "abuse_reports_status_idx", + "columns": ["status", "created_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 86400000 + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 10 + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_request": { + "name": "last_request", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": ["key"], + "isUnique": false + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": ["reference_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_collections": { + "name": "ark_collections", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "ark_id": { + "name": "ark_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "custom_url": { + "name": "custom_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_collections_ark_id_unique": { + "name": "ark_collections_ark_id_unique", + "columns": ["ark_id"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_collections_collection_id_collections_id_fk": { + "name": "ark_collections_collection_id_collections_id_fk", + "tableFrom": "ark_collections", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_record_types": { + "name": "ark_record_types", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirect_url_field": { + "name": "redirect_url_field", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "ark_record_types_collection_id_collections_id_fk": { + "name": "ark_record_types_collection_id_collections_id_fk", + "tableFrom": "ark_record_types", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "ark_record_types_collection_id_record_type_pk": { + "columns": ["collection_id", "record_type"], + "name": "ark_record_types_collection_id_record_type_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_shoulders": { + "name": "ark_shoulders", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "shoulder": { + "name": "shoulder", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_shoulders_shoulder_unique": { + "name": "ark_shoulders_shoulder_unique", + "columns": ["shoulder"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_shoulders_organization_id_organization_id_fk": { + "name": "ark_shoulders_organization_id_organization_id_fk", + "tableFrom": "ark_shoulders", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cleanup_runs": { + "name": "cleanup_runs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "step": { + "name": "step", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dry_run": { + "name": "dry_run", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "requested_by": { + "name": "requested_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mark_run_id": { + "name": "mark_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lease": { + "name": "lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stats": { + "name": "stats", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "cleanup_runs_created_idx": { + "name": "cleanup_runs_created_idx", + "columns": ["created_at"], + "isUnique": false + }, + "cleanup_runs_step_idx": { + "name": "cleanup_runs_step_idx", + "columns": ["step", "status"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_tombstones": { + "name": "collection_tombstones", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "versions": { + "name": "versions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_by": { + "name": "deleted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_layout": { + "name": "record_layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'today'" + }, + "records_dropped_at": { + "name": "records_dropped_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_webhooks": { + "name": "collection_webhooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bump_filter": { + "name": "bump_filter", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_delivery_at": { + "name": "last_delivery_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "collection_webhooks_collection_idx": { + "name": "collection_webhooks_collection_idx", + "columns": ["collection_id"], + "isUnique": false + } + }, + "foreignKeys": { + "collection_webhooks_collection_id_collections_id_fk": { + "name": "collection_webhooks_collection_id_collections_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collections": { + "name": "collections", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "head_version_id": { + "name": "head_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_salt": { + "name": "private_salt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_files_root": { + "name": "public_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "version_count": { + "name": "version_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "history_bytes": { + "name": "history_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_push_at": { + "name": "last_push_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_started_at": { + "name": "reconcile_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_state": { + "name": "reconcile_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_seq": { + "name": "reconciled_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "reconciled_files_root": { + "name": "reconciled_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_layout": { + "name": "record_layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'today'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "collections_org_slug_uq": { + "name": "collections_org_slug_uq", + "columns": ["organization_id", "slug"], + "isUnique": true + }, + "collections_public_updated_idx": { + "name": "collections_public_updated_idx", + "columns": ["public", "updated_at"], + "isUnique": false + } + }, + "foreignKeys": { + "collections_organization_id_organization_id_fk": { + "name": "collections_organization_id_organization_id_fk", + "tableFrom": "collections", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "commit_units": { + "name": "commit_units", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "plan_id": { + "name": "plan_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ord": { + "name": "ord", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "after": { + "name": "after", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "through": { + "name": "through", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "gap": { + "name": "gap", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "slices_key": { + "name": "slices_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_key": { + "name": "output_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "queued_at": { + "name": "queued_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "commit_units_plan_idx": { + "name": "commit_units_plan_idx", + "columns": ["plan_id", "set", "type", "ord"], + "isUnique": false + }, + "commit_units_status_idx": { + "name": "commit_units_status_idx", + "columns": ["session_id", "status", "queued_at"], + "isUnique": false + } + }, + "foreignKeys": { + "commit_units_session_id_push_sessions_id_fk": { + "name": "commit_units_session_id_push_sessions_id_fk", + "tableFrom": "commit_units", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "denylist": { + "name": "denylist", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "file_uploads": { + "name": "file_uploads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "multipart_upload_id": { + "name": "multipart_upload_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "file_uploads_hash_idx": { + "name": "file_uploads_hash_idx", + "columns": ["hash"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "files": { + "name": "files", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "forks": { + "name": "forks", + "columns": { + "child_collection_id": { + "name": "child_collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "parent_collection_id": { + "name": "parent_collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_seq": { + "name": "parent_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'public'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "forks_child_collection_id_collections_id_fk": { + "name": "forks_child_collection_id_collections_id_fk", + "tableFrom": "forks", + "tableTo": "collections", + "columnsFrom": ["child_collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instance_settings": { + "name": "instance_settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invitation": { + "name": "invitation", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "jobs": { + "name": "jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "run_at": { + "name": "run_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locked_until": { + "name": "locked_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "jobs_ready_idx": { + "name": "jobs_ready_idx", + "columns": ["status", "run_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "member": { + "name": "member", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + }, + "member_user_id_idx": { + "name": "member_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organization": { + "name": "organization", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ark_naan": { + "name": "ark_naan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kf_org_id": { + "name": "kf_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "columns": ["slug"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "placements": { + "name": "placements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location_id": { + "name": "location_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "synced_seq": { + "name": "synced_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until": { + "name": "lease_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "placements_one_primary_uq": { + "name": "placements_one_primary_uq", + "columns": ["collection_id"], + "isUnique": true, + "where": "\"placements\".\"role\" = 'primary' AND \"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_collection_location_uq": { + "name": "placements_collection_location_uq", + "columns": ["collection_id", "location_id"], + "isUnique": true, + "where": "\"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_org_location_uq": { + "name": "placements_org_location_uq", + "columns": ["organization_id", "location_id"], + "isUnique": true, + "where": "\"placements\".\"organization_id\" IS NOT NULL" + }, + "placements_location_idx": { + "name": "placements_location_idx", + "columns": ["location_id"], + "isUnique": false + } + }, + "foreignKeys": { + "placements_collection_id_collections_id_fk": { + "name": "placements_collection_id_collections_id_fk", + "tableFrom": "placements", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_organization_id_organization_id_fk": { + "name": "placements_organization_id_organization_id_fk", + "tableFrom": "placements", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_location_id_storage_locations_id_fk": { + "name": "placements_location_id_storage_locations_id_fk", + "tableFrom": "placements", + "tableTo": "storage_locations", + "columnsFrom": ["location_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_packs": { + "name": "push_packs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pack_id": { + "name": "pack_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_set": { + "name": "record_set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "records": { + "name": "records", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw": { + "name": "raw", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "push_packs_session_id_push_sessions_id_fk": { + "name": "push_packs_session_id_push_sessions_id_fk", + "tableFrom": "push_packs", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_packs_session_id_pack_id_pk": { + "columns": ["session_id", "pack_id"], + "name": "push_packs_session_id_pack_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_runs": { + "name": "push_runs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "first_key": { + "name": "first_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_key": { + "name": "last_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "merging_into": { + "name": "merging_into", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "push_runs_tier_idx": { + "name": "push_runs_tier_idx", + "columns": ["session_id", "tier", "merging_into"], + "isUnique": false + } + }, + "foreignKeys": { + "push_runs_session_id_push_sessions_id_fk": { + "name": "push_runs_session_id_push_sessions_id_fk", + "tableFrom": "push_runs", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_runs_session_id_seq_pk": { + "columns": ["session_id", "seq"], + "name": "push_runs_session_id_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_sessions": { + "name": "push_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_version_id": { + "name": "base_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "strip_unknown_fields": { + "name": "strip_unknown_fields", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "records_received": { + "name": "records_received", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "runs": { + "name": "runs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "result": { + "name": "result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finalize_started_at": { + "name": "finalize_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_started": { + "name": "commit_started", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "cleaned_at": { + "name": "cleaned_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_phase": { + "name": "commit_phase", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "progress_at": { + "name": "progress_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stalls": { + "name": "stalls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "commit_plan": { + "name": "commit_plan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "assembly_lease": { + "name": "assembly_lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "push_sessions_collection_idx": { + "name": "push_sessions_collection_idx", + "columns": ["collection_id"], + "isUnique": false + }, + "push_sessions_expires_idx": { + "name": "push_sessions_expires_idx", + "columns": ["status", "expires_at"], + "isUnique": false + }, + "push_sessions_user_idx": { + "name": "push_sessions_user_idx", + "columns": ["user_id", "status"], + "isUnique": false + } + }, + "foreignKeys": { + "push_sessions_collection_id_collections_id_fk": { + "name": "push_sessions_collection_id_collections_id_fk", + "tableFrom": "push_sessions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compaction_parts": { + "name": "ref_compaction_parts", + "columns": { + "compaction_id": { + "name": "compaction_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "part": { + "name": "part", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_compaction_parts_compaction_id_part_pk": { + "columns": ["compaction_id", "part"], + "name": "ref_compaction_parts_compaction_id_part_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compactions": { + "name": "ref_compactions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_runs": { + "name": "input_runs", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output_run": { + "name": "output_run", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parts": { + "name": "parts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix_length": { + "name": "prefix_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'running'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_index_units": { + "name": "ref_index_units", + "columns": { + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unit": { + "name": "unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "events": { + "name": "events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_index_units_version_id_unit_pk": { + "columns": ["version_id", "unit"], + "name": "ref_index_units_version_id_unit_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_segments": { + "name": "ref_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_hash": { + "name": "first_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_hash": { + "name": "last_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'live'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ref_segments_range_idx": { + "name": "ref_segments_range_idx", + "columns": ["state", "first_hash", "last_hash"], + "isUnique": false + }, + "ref_segments_run_idx": { + "name": "ref_segments_run_idx", + "columns": ["run_id"], + "isUnique": false + }, + "ref_segments_tier_idx": { + "name": "ref_segments_tier_idx", + "columns": ["tier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_labels": { + "name": "schema_labels", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "schema_labels_label_idx": { + "name": "schema_labels_label_idx", + "columns": ["label"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "schema_labels_schema_hash_label_pk": { + "columns": ["schema_hash", "label"], + "name": "schema_labels_schema_hash_label_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_usage": { + "name": "schema_usage", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type_slug": { + "name": "type_slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_seq": { + "name": "from_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "to_seq": { + "name": "to_seq", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "schema_usage_hash_idx": { + "name": "schema_usage_hash_idx", + "columns": ["schema_hash"], + "isUnique": false + } + }, + "foreignKeys": { + "schema_usage_collection_id_collections_id_fk": { + "name": "schema_usage_collection_id_collections_id_fk", + "tableFrom": "schema_usage", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "schema_usage_collection_id_type_slug_set_from_seq_pk": { + "columns": ["collection_id", "type_slug", "set", "from_seq"], + "name": "schema_usage_collection_id_type_slug_set_from_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schemas": { + "name": "schemas", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": ["token"], + "isUnique": true + }, + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_fence": { + "name": "storage_fence", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "epoch": { + "name": "epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "window_until": { + "name": "window_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "window_run_id": { + "name": "window_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_locations": { + "name": "storage_locations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucket": { + "name": "bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "credentials": { + "name": "credentials", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unverified'" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checked_at": { + "name": "checked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "storage_locations_organization_id_organization_id_fk": { + "name": "storage_locations_organization_id_organization_id_fk", + "tableFrom": "storage_locations", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "usage_rollups": { + "name": "usage_rollups", + "columns": { + "day": { + "name": "day", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metric": { + "name": "metric", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "amount": { + "name": "amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "usage_rollups_account_idx": { + "name": "usage_rollups_account_idx", + "columns": ["account_id", "day"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "usage_rollups_day_account_id_collection_id_metric_pk": { + "columns": ["day", "account_id", "collection_id", "metric"], + "name": "usage_rollups_day_account_id_collection_id_metric_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": ["email"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": ["identifier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "versions": { + "name": "versions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "major": { + "name": "major", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "minor": { + "name": "minor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "patch": { + "name": "patch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pushed_by": { + "name": "pushed_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_record_count": { + "name": "public_record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_file_count": { + "name": "public_file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "public_total_bytes": { + "name": "public_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "type_counts": { + "name": "type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_type_counts": { + "name": "public_type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "has_private": { + "name": "has_private", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "public_refs_root": { + "name": "public_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_refs_root": { + "name": "private_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refs_indexed": { + "name": "refs_indexed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "changes": { + "name": "changes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "push_session_id": { + "name": "push_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "versions_collection_seq_uq": { + "name": "versions_collection_seq_uq", + "columns": ["collection_id", "seq"], + "isUnique": true + }, + "versions_collection_semver_uq": { + "name": "versions_collection_semver_uq", + "columns": ["collection_id", "semver"], + "isUnique": true + }, + "versions_hash_idx": { + "name": "versions_hash_idx", + "columns": ["hash"], + "isUnique": false + }, + "versions_published_idx": { + "name": "versions_published_idx", + "columns": ["published_at"], + "isUnique": false + }, + "versions_push_session_idx": { + "name": "versions_push_session_idx", + "columns": ["push_session_id"], + "isUnique": false + } + }, + "foreignKeys": { + "versions_collection_id_collections_id_fk": { + "name": "versions_collection_id_collections_id_fk", + "tableFrom": "versions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhook_deliveries": { + "name": "webhook_deliveries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "webhook_id": { + "name": "webhook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bump_type": { + "name": "bump_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'version.created'" + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "delivered_at": { + "name": "delivered_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhook_deliveries_webhook_idx": { + "name": "webhook_deliveries_webhook_idx", + "columns": ["webhook_id", "created_at"], + "isUnique": false + }, + "webhook_deliveries_pending_idx": { + "name": "webhook_deliveries_pending_idx", + "columns": ["status", "next_attempt_at"], + "isUnique": false + } + }, + "foreignKeys": { + "webhook_deliveries_webhook_id_collection_webhooks_id_fk": { + "name": "webhook_deliveries_webhook_id_collection_webhooks_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collection_webhooks", + "columnsFrom": ["webhook_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index f8fd9100..225cdc62 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -183,6 +183,13 @@ "when": 1792506002064, "tag": "0025_record_layout", "breakpoints": true + }, + { + "idx": 26, + "version": "6", + "when": 1792518549719, + "tag": "0026_push_packs", + "breakpoints": true } ] } diff --git a/packages/server/src/api/push.ts b/packages/server/src/api/push.ts index 3e654f85..adceb665 100644 --- a/packages/server/src/api/push.ts +++ b/packages/server/src/api/push.ts @@ -139,6 +139,7 @@ async function commitRoute(c: Context, session: SessionRow) { // uploads (still in storage) and a fresh plan. await transition(ports, session.id, 'failed', 'committing', { finalizeStartedAt: new Date(), + commitStarted: true, error: null, commitPlan: null, commitPhase: null, @@ -151,6 +152,7 @@ async function commitRoute(c: Context, session: SessionRow) { : // The error of a commit refused for missing files is cleared on the next try. await transition(ports, session.id, 'open', 'committing', { finalizeStartedAt: new Date(), + commitStarted: true, error: null, ...rebased, }) diff --git a/packages/server/src/cleanup/internal.ts b/packages/server/src/cleanup/internal.ts index 067749f6..eff5b6da 100644 --- a/packages/server/src/cleanup/internal.ts +++ b/packages/server/src/cleanup/internal.ts @@ -19,6 +19,7 @@ import { and, eq, inArray, isNull, lt, sql } from 'drizzle-orm' import { chunks } from '../db/chunks.js' import * as schema from '../db/schema.js' import type { Ports } from '../ports.js' +import { settleSessionPacks } from '../push/packs.js' import { cleanupConfig, count, emptyStats, problem } from './config.js' const TERMINAL: schema.SessionStatus[] = ['committed', 'failed', 'expired'] @@ -67,7 +68,11 @@ export async function cleanInternal( // Push sessions. const sessions = await db - .select({ id: schema.pushSessions.id }) + .select({ + id: schema.pushSessions.id, + collectionId: schema.pushSessions.collectionId, + commitStarted: schema.pushSessions.commitStarted, + }) .from(schema.pushSessions) .where( and( @@ -89,7 +94,12 @@ export async function cleanInternal( count(stats, 'sessions', gone.bytes, gone.objects) // A session bigger than what's left of the budget finishes next time. if (!gone.complete) more = true - if (dryRun || !gone.complete) continue + if (!gone.complete) continue + // Layout F: packs the uploads wrote into the collection (push/packs.ts). + const packs = await settleSessionPacks(ports, s, dryRun) + budget -= packs.objects + if (packs.objects) count(stats, 'upload packs', packs.bytes, packs.objects) + if (dryRun) continue const [runs, units] = await Promise.all([ db .delete(schema.pushRuns) diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index 095f757a..39d87f99 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -607,6 +607,12 @@ export const pushSessions = sqliteTable( result: json>('result'), error: json<{ statusCode: number; error: string; [k: string]: unknown }>('error'), finalizeStartedAt: ts('finalize_started_at'), + /** + * A commit of this session has started, ever (unlike `finalizeStartedAt`, never + * cleared): its index may name records in the packs its uploads wrote + * (push/packs.ts), so cleanup leaves those to reclaim instead of deleting them. + */ + commitStarted: bool('commit_started').notNull().default(false), /** When storage cleanup deleted the session's objects (sessions//) and run rows. */ cleanedAt: ts('cleaned_at'), /** @@ -662,6 +668,30 @@ export const pushRuns = sqliteTable( ], ) +/** + * A pack an upload wrote straight into a layout F collection (push/delta.ts): its + * records' places go in the session's runs, so the commit indexes them without + * moving them. A row is written before its pack; the stats once the pack is + * written (null: it never was). The commit lists the packs in the collection's + * manifest or registry; storage cleanup deletes those of a push that never did. + */ +export const pushPacks = sqliteTable( + 'push_packs', + { + sessionId: text('session_id') + .notNull() + .references(() => pushSessions.id, { onDelete: 'cascade' }), + packId: text('pack_id').notNull(), + set: text('record_set', { enum: ['public', 'private'] }).notNull(), + /** PackInfo, once written. */ + records: integer('records'), + bytes: integer('bytes'), + raw: integer('raw'), + createdAt: createdAt(), + }, + (t) => [primaryKey({ columns: [t.sessionId, t.packId] })], +) + export type CommitUnitStatus = 'pending' | 'done' | 'failed' | 'superseded' /** diff --git a/packages/server/src/push/changes.ts b/packages/server/src/push/changes.ts index 06c6971a..3b52863b 100644 --- a/packages/server/src/push/changes.ts +++ b/packages/server/src/push/changes.ts @@ -9,6 +9,7 @@ import { keys, outOfLineHash, outOfLinePointer, + type Placed, type RecordEntry, type Repo, type SetName, @@ -64,12 +65,13 @@ export async function takeStaged( export const isPrivateSchema = (s: Record) => s.private === true -export const toRecordEntry = (e: RunEntry): RecordEntry => ({ - key: e.k, - hash: e.h!, - size: e.s!, - body: e.b!, -}) +/** A run entry as the commit takes it: with its body, or its place in an upload's pack. */ +export function toRecordEntry(e: RunEntry): RecordEntry { + if (e.l === undefined) return { key: e.k, hash: e.h!, size: e.s!, body: e.b! } + const placed: Placed = { key: e.k, hash: e.h!, size: e.s!, packed: e.l } + if (e.r) placed.fileRefs = e.r + return placed +} /** Which sets of a type the base has a tree in. */ export interface BaseSets { diff --git a/packages/server/src/push/delta.ts b/packages/server/src/push/delta.ts index ae7107be..153b3b98 100644 --- a/packages/server/src/push/delta.ts +++ b/packages/server/src/push/delta.ts @@ -12,6 +12,7 @@ import { checkTypeSlug, compileSchema, emptySet, + fileRefs, InputRuleError, OUT_OF_LINE_BYTES, parseRecordLine, @@ -30,6 +31,7 @@ import type { Ports } from '../ports.js' import { type BaseVersion, commitVersion, type TypeInput } from '../versions/commit.js' import { deltaChanges, isPrivateSchema, stageOutOfLine } from './changes.js' import type { SessionCommitResult } from './outcome.js' +import { packEntries, packStats } from './packs.js' import { planParallel, SerialTooLargeError } from './parallel.js' import { type RunEntry, type RunIndex, writeRun } from './runs.js' import { @@ -80,7 +82,8 @@ export async function prepareRecords( sessionId: string, inputs: SessionInputs, input: string | AsyncIterable, - opts: { stripUnknownFields: boolean }, + /** `packed`: the records go into packs (layout F), so large ones aren't staged apart. */ + opts: { stripUnknownFields: boolean; packed?: boolean }, ): Promise< { entries: RunEntry[] } | { errors: LineError[]; total: number } | { tooManyLines: true } > { @@ -161,10 +164,13 @@ export async function prepareRecords( const hash = sha256Hex(canonical) const size = utf8ByteLength(canonical) const body = - size > OUT_OF_LINE_BYTES + size > OUT_OF_LINE_BYTES && !opts.packed ? await stageOutOfLine(internal, sessionId, hash, canonical) : canonical const isPrivate = isPrivateSchema(typeSchema) || rec.private === true + // A packed record's body leaves its entry (push/packs.ts): the commit counts its + // file references from these. + const refs = opts.packed && canonical.includes('"$file"') ? fileRefs(data) : [] const entry: RunEntry = { t: rec.type, k: rec.id, @@ -172,6 +178,7 @@ export async function prepareRecords( s: size, b: body, ...(isPrivate ? { p: true } : {}), + ...(refs.length ? { r: refs } : {}), } entries.push(entry) } @@ -184,8 +191,11 @@ export async function ingestRecords( input: string | AsyncIterable, ): Promise { const inputs = await loadInputs(ports, session.id) + // Layout F: the records go straight into the collection's packs. + const records = (await ports.stores.forCollection(session.collectionId)).records const prepared = await prepareRecords(ports, session.id, inputs, input, { stripUnknownFields: session.stripUnknownFields, + packed: records !== null, }) if ('tooManyLines' in prepared) { return { ok: false, status: 413, error: `At most ${MAX_BATCH_LINES} records per batch` } @@ -202,8 +212,16 @@ export async function ingestRecords( if (prepared.entries.length === 0) return { ok: false, status: 400, error: 'Empty batch' } const seq = await nextRunSeq(ports, session.id) if (seq === null) return { ok: false, status: 409, error: 'Session is not open' } + const packs = records ? await packEntries(ports, session.id, records, prepared.entries) : [] const index = await writeRun(ports.stores.internal, session.id, seq, prepared.entries) - await recordRun(ports, session.id, 'records', index, { records: prepared.entries.length }) + await recordRun( + ports, + session.id, + 'records', + index, + { records: prepared.entries.length }, + packStats(ports, session.id, packs), + ) return { ok: true, received: prepared.entries.length } } diff --git a/packages/server/src/push/packs.ts b/packages/server/src/push/packs.ts new file mode 100644 index 00000000..37e507b2 --- /dev/null +++ b/packages/server/src/push/packs.ts @@ -0,0 +1,157 @@ +/** + * Uploads into a layout F collection write their records straight into packs of + * the collection's, where they stay. Each record's run entry then carries its + * place (`l`) instead of its body, so compaction moves entries of about 150 bytes + * instead of whole records (a 30M-record Crossref session spent 10.5 hours + * compacting 206 GB, 2026-10-19), and commit units index the records without + * reading or writing them again. + * + * A pack is outside the collection's index and manifest until a commit names it + * (`adoptSessionPacks`, just before the version publishes). Each is recorded in + * `push_packs` before it's written, so storage cleanup can delete the packs of a + * push that never published (`dropSessionPacks`). + */ +import { + packIdsFor, + type PackInfo, + type RecordSet, + type RecordSets, + type Repo, +} from '@underlay/protocol' +import { and, eq, isNotNull } from 'drizzle-orm' + +import * as schema from '../db/schema.js' +import type { Ports } from '../ports.js' +import { type RunEntry, runKey } from './runs.js' + +const SETS = ['public', 'private'] as const + +/** A pack an upload wrote, for its row's stats. */ +export interface UploadPack { + set: RecordSet + pack: PackInfo +} + +/** + * Write a batch's records into packs, a set each, and give their entries places + * instead of bodies. Within the batch the last entry of a key wins (writeRun), + * so only those are written. + */ +export async function packEntries( + ports: Ports, + sessionId: string, + records: RecordSets, + entries: RunEntry[], +): Promise { + const last = new Map() + for (const e of entries) last.set(runKey(e), e) + const out: UploadPack[] = [] + for (const set of SETS) { + // An upload marks an entry private exactly when it goes to the private set (deltaChanges). + const group = [...last.values()].filter((e) => (e.p === true) === (set === 'private')) + if (!group.length) continue + const recs = group.map((e) => ({ id: e.k, hash: e.h!, line: e.b! })) + const ids = packIdsFor(recs) + // Recorded before they're written: an upload that dies leaves no pack cleanup can't find. + await ports.db + .insert(schema.pushPacks) + .values(ids.map((packId) => ({ sessionId, packId, set }))) + const { places, packs } = await records[set].writePacks!(recs, ids) + group.forEach((e, i) => { + e.l = places[i]! + delete e.b + }) + out.push(...packs.map((pack) => ({ set, pack }))) + } + return out +} + +/** The statements that record written packs' stats (run with the upload's run row). */ +export const packStats = (ports: Ports, sessionId: string, packs: UploadPack[]) => + packs.map(({ pack }) => + ports.db + .update(schema.pushPacks) + .set({ records: pack.records, bytes: pack.bytes, raw: pack.raw }) + .where(and(eq(schema.pushPacks.sessionId, sessionId), eq(schema.pushPacks.packId, pack.id))), + ) + +/** + * List a session's packs in the collection's manifest or registry. Called once the + * commit's index names their records, before its version publishes; a retry lists + * nothing twice (KeyedLayout.adoptPacks). + */ +export async function adoptSessionPacks(ports: Ports, repo: Repo, sessionId: string) { + if (!repo.records) return + const rows = await ports.db + .select() + .from(schema.pushPacks) + .where(and(eq(schema.pushPacks.sessionId, sessionId), isNotNull(schema.pushPacks.records))) + for (const set of SETS) { + const packs: PackInfo[] = rows + .filter((r) => r.set === set) + .map((r) => ({ id: r.packId, tier: 0, records: r.records!, bytes: r.bytes!, raw: r.raw! })) + if (packs.length) await repo.records[set].adoptPacks!(sessionId, packs) + } +} + +/** + * At storage cleanup, settle a finished session's packs, then drop its rows: + * - a version names the session: its commit listed them, nothing to do; + * - no commit of it ever started: nothing indexes their records, so they're deleted; + * - a commit started and never published: its index may name their records (one the + * collection already held, say, whose newest place is now such a pack), so they're + * listed instead, and reclaim drops what no version uses. + * Returns the objects and bytes deleted (a pack is two objects), counting packs that + * were written. A collection whose repository won't open (deleted) is left to the + * sweep, which drops its files. + */ +export async function settleSessionPacks( + ports: Ports, + session: { id: string; collectionId: string; commitStarted: boolean }, + dryRun: boolean, +): Promise<{ objects: number; bytes: number }> { + const gone = { objects: 0, bytes: 0 } + const rows = await ports.db + .select() + .from(schema.pushPacks) + .where(eq(schema.pushPacks.sessionId, session.id)) + if (!rows.length) return gone + const [published] = await ports.db + .select({ id: schema.versions.id }) + .from(schema.versions) + .where(eq(schema.versions.pushSessionId, session.id)) + .limit(1) + let repo: Repo | null = null + try { + repo = await ports.stores.forCollection(session.collectionId) + } catch { + repo = null + } + if (!published && repo?.records) { + if (session.commitStarted) { + if (!dryRun) await adoptSessionPacks(ports, repo, session.id) + } else { + for (const set of SETS) { + const mine = rows.filter((r) => r.set === set) + if (!mine.length) continue + // Refused if a commit listed them after all (KeyedLayout.dropPacks). + const dropped = + dryRun || + (await repo.records[set].dropPacks!( + session.id, + mine.map((r) => r.packId), + )) + if (!dropped) continue + // A row without stats is a pack never written, or one whose upload died writing it. + for (const r of mine) + if (r.records !== null) { + gone.objects += 2 + gone.bytes += r.bytes ?? 0 + } + } + } + } + if (!dryRun) + await ports.db.delete(schema.pushPacks).where(eq(schema.pushPacks.sessionId, session.id)) + return gone +} diff --git a/packages/server/src/push/runs.ts b/packages/server/src/push/runs.ts index 657a6e22..889cc74a 100644 --- a/packages/server/src/push/runs.ts +++ b/packages/server/src/push/runs.ts @@ -62,8 +62,12 @@ export interface RunEntry { s?: number /** Pushed as private. */ p?: boolean - /** The canonical record (records runs). */ + /** The canonical record (records runs), unless it's in a pack (`l`). */ b?: string + /** Layout F: the record's place in a pack the upload wrote (push/packs.ts). */ + l?: string + /** With `l`: the files the record references, read at upload. */ + r?: string[] /** A delete (delta deletes runs). */ x?: boolean /** Precedence: the seq of the upload this entry came from (compacted runs only). */ diff --git a/packages/server/src/push/session.ts b/packages/server/src/push/session.ts index 118ac9f1..f9f286eb 100644 --- a/packages/server/src/push/session.ts +++ b/packages/server/src/push/session.ts @@ -44,6 +44,8 @@ export interface SessionInputs { export type SessionRow = typeof schema.pushSessions.$inferSelect +type BatchStatement = Parameters[0][number] + const inputsKey = (id: string) => `sessions/${id}/inputs.json` export async function createSession( @@ -113,6 +115,8 @@ export async function recordRun( kind: 'records' | 'deletes', index: RunIndex, counters: { records?: number }, + /** More statements for the same batch (packs.ts: the upload's packs' stats). */ + extra: BatchStatement[] = [], ): Promise { const count = runCount(index) await ports.db.batch([ @@ -133,6 +137,7 @@ export async function recordRun( recordsReceived: sql`${schema.pushSessions.recordsReceived} + ${counters.records ?? 0}`, }) .where(eq(schema.pushSessions.id, sessionId)), + ...extra, ]) await maybeCompact(ports, sessionId, index.tier ?? 0) } diff --git a/packages/server/src/versions/commit.ts b/packages/server/src/versions/commit.ts index fa2851ec..bd6b775a 100644 --- a/packages/server/src/versions/commit.ts +++ b/packages/server/src/versions/commit.ts @@ -44,6 +44,7 @@ import { eq } from 'drizzle-orm' import { FenceError, writeFence } from '../cleanup/fence.js' import * as schema from '../db/schema.js' import type { Ports } from '../ports.js' +import { adoptSessionPacks } from '../push/packs.js' import { publishCollectionInfo } from './collection-info.js' import { collectionFileSizes, fileSizes } from './file-refs.js' import { publishVersion, type SchemaUsageChange } from './publish.js' @@ -152,6 +153,9 @@ export async function commitVersion(ports: Ports, input: CommitInput): Promise { expect(h.bucket.objects.has(`repo/files/${sha('the pdf')}`)).toBe(true) }) - it('keeps a large record a push uploaded while the sweep ran', async () => { - const h = await harness() - const user = await h.member() - // Over OUT_OF_LINE_BYTES: stored as its own records/ object. - const big = { id: 'big', type: 'Doc', data: { title: 'x'.repeat(70_000) } } - await h.collection('old') - await push(h, user, 'old', [big]) - const recordKeys = () => keys(h, 'repo/records/') - expect(recordKeys()).toHaveLength(1) - await h.request('/api/collections/org/old', { method: 'DELETE', user }) - await pastGrace(h) - await run(h, 'mark') - - // A new push uploads the same record and stays open while the sweep runs. - const fresh = await h.collection('new') - const base = '/api/collections/org/new' - const open = await h.request(`${base}/push`, { - method: 'POST', - user, - json: { schemas: { Doc } }, - }) - const sid = ((await open.json()) as { session_id: string }).session_id - await h.request(`${base}/push/${sid}/records`, { method: 'POST', user, ndjson: [big] }) - // Staged in the session's area, not written to the repository yet. - expect(keys(h, `internal/sessions/${sid}/records/`)).toHaveLength(1) - ageAll(h) - const sweep = await run(h, 'sweep') - expect(sweep.stats!.deleted.records).toMatchObject({ objects: 1 }) - expect(recordKeys()).toEqual([]) + // Today's layout: on layout F an upload writes a large record into the collection's + // packs instead (push/packs.ts), and cleanup deletes those only for a push that never + // published (upload-packs.test.ts). + it.skipIf(process.env.UNDERLAY_TEST_LAYOUT === 'f')( + 'keeps a large record a push uploaded while the sweep ran', + async () => { + const h = await harness() + const user = await h.member() + // Over OUT_OF_LINE_BYTES: stored as its own records/ object. + const big = { id: 'big', type: 'Doc', data: { title: 'x'.repeat(70_000) } } + await h.collection('old') + await push(h, user, 'old', [big]) + const recordKeys = () => keys(h, 'repo/records/') + expect(recordKeys()).toHaveLength(1) + await h.request('/api/collections/org/old', { method: 'DELETE', user }) + await pastGrace(h) + await run(h, 'mark') - // The commit puts it back, under its fence. - const commit = await h.request(`${base}/push/${sid}/commit`, { method: 'POST', user }) - expect(commit.status).toBe(201) - expect(recordKeys()).toHaveLength(1) - await healthy(h, fresh.id) - }) + // A new push uploads the same record and stays open while the sweep runs. + const fresh = await h.collection('new') + const base = '/api/collections/org/new' + const open = await h.request(`${base}/push`, { + method: 'POST', + user, + json: { schemas: { Doc } }, + }) + const sid = ((await open.json()) as { session_id: string }).session_id + await h.request(`${base}/push/${sid}/records`, { method: 'POST', user, ndjson: [big] }) + // Staged in the session's area, not written to the repository yet. + expect(keys(h, `internal/sessions/${sid}/records/`)).toHaveLength(1) + ageAll(h) + const sweep = await run(h, 'sweep') + expect(sweep.stats!.deleted.records).toMatchObject({ objects: 1 }) + expect(recordKeys()).toEqual([]) + + // The commit puts it back, under its fence. + const commit = await h.request(`${base}/push/${sid}/commit`, { method: 'POST', user }) + expect(commit.status).toBe(201) + expect(recordKeys()).toHaveLength(1) + await healthy(h, fresh.id) + }, + ) it('gives the same result in many small jobs, shards and windows', async () => { const saved = { ...cleanupConfig } diff --git a/packages/server/test/stream-push.test.ts b/packages/server/test/stream-push.test.ts index e1565e63..54bea238 100644 --- a/packages/server/test/stream-push.test.ts +++ b/packages/server/test/stream-push.test.ts @@ -220,7 +220,10 @@ describe('one version from many uploads', () => { }) const status = async () => json(await h.request(`${path}/push/${sid}`, { user })) expect((await status()).progress).toMatchObject({ runs: 6, phase: null, units: null }) - expect((await status()).progress.run_bytes).toBeGreaterThan(6 * 10 * 512) + // Runs hold the records, or on layout F their places in the uploads' packs. + if (process.env.UNDERLAY_TEST_LAYOUT === 'f') + expect((await status()).progress.run_bytes).toBeLessThan(6 * 10 * 512) + else expect((await status()).progress.run_bytes).toBeGreaterThan(6 * 10 * 512) Object.assign(compactConfig, { commitRuns: 2 }) const res = await h.request(`${path}/push/${sid}/commit`, { method: 'POST', user }) // Too many runs for one commit: it compacts first, and answers 202. diff --git a/packages/server/test/upload-packs.test.ts b/packages/server/test/upload-packs.test.ts new file mode 100644 index 00000000..22f3063f --- /dev/null +++ b/packages/server/test/upload-packs.test.ts @@ -0,0 +1,365 @@ +/** + * Layout F uploads write their records straight into the collection's packs + * (push/packs.ts): runs carry places instead of bodies, a commit indexes the + * records without writing them again, its packs are listed once before the version + * publishes, and cleanup deletes the packs of a push that never did. + */ +import { createHash } from 'node:crypto' + +import { countTree, iterate, type RecordSet, RepoSource, recordTree } from '@underlay/protocol' +import { eq } from 'drizzle-orm' +import { afterAll, afterEach, describe, expect, it } from 'vitest' + +import { cleanupConfig, getRun } from '../src/cleanup/config.js' +import { startRun } from '../src/cleanup/runs.js' +import * as schema from '../src/db/schema.js' +import { compactConfig } from '../src/push/compact.js' +import { parallelConfig } from '../src/push/parallel.js' +import { mergeRuns, type RunEntry } from '../src/push/runs.js' +import { sessionRuns } from '../src/push/session.js' +import { cleanup, type Harness, harness } from './harness.js' + +afterAll(cleanup) + +const defaults = { parallel: { ...parallelConfig }, compact: { ...compactConfig } } +afterEach(() => { + Object.assign(parallelConfig, defaults.parallel) + Object.assign(compactConfig, defaults.compact) +}) + +const sha = (s: string) => createHash('sha256').update(s).digest('hex') +const Doc = { type: 'object', properties: { title: { type: 'string' }, pdf: { type: 'object' } } } +const doc = (id: string, title: string, extra: Record = {}) => ({ + id, + type: 'Doc', + data: { title, ...extra }, +}) +const id = (i: number) => `d${String(i).padStart(5, '0')}` +const DAY = 24 * 60 * 60 * 1000 + +async function json(res: Response) { + return (await res.json()) as any +} + +async function open(h: Harness, user: string, slug: string) { + const res = await h.request(`/api/collections/org/${slug}/push`, { + method: 'POST', + user, + json: { schemas: { Doc } }, + }) + expect(res.status).toBe(200) + return (await json(res)).session_id as string +} + +async function upload(h: Harness, user: string, slug: string, sid: string, batch: object[]) { + const r = await h.request(`/api/collections/org/${slug}/push/${sid}/records`, { + method: 'POST', + user, + ndjson: batch, + }) + expect(r.status).toBe(200) +} + +async function commit(h: Harness, user: string, slug: string, sid: string) { + const res = await h.request(`/api/collections/org/${slug}/push/${sid}/commit`, { + method: 'POST', + user, + }) + await h.drain() + return res.status +} + +/** Pack files of a collection's set in the bucket. */ +const packKeys = (h: Harness, collectionId: string, set: RecordSet = 'public') => + [...h.bucket.objects.keys()].filter((k) => + k.startsWith(`repo/runs/${collectionId}/${set}/packs/`), + ) + +/** Every record of the head version: `set id title`. */ +async function headRecords(h: Harness, collectionId: string) { + const [c] = await h.ports.db + .select() + .from(schema.collections) + .where(eq(schema.collections.id, collectionId)) + const [v] = await h.ports.db + .select() + .from(schema.versions) + .where(eq(schema.versions.id, c!.headVersionId!)) + const repo = await h.ports.stores.forCollection(collectionId) + const root = await repo.root(v!.hash) + const out: string[] = [] + const sets: [RecordSet, string | null][] = [['public', root.public.types.Doc?.root ?? null]] + if (root.private) + sets.push(['private', (await repo.privateSet(root.private)).types.Doc?.root ?? null]) + for (const [set, tree] of sets) + for await (const e of iterate(new RepoSource(recordTree, repo, { set }), tree, { + payloads: true, + })) { + const body = JSON.parse(e.body!) as { data: { title: string } } + out.push(`${set} ${e.key} ${body.data.title}`) + } + return { v: v!, out } +} + +async function sessionEntries(h: Harness, sid: string) { + const out: RunEntry[] = [] + for await (const e of mergeRuns(h.ports.stores.internal, sid, await sessionRuns(h.ports, sid))) + out.push(e) + return out +} + +describe.runIf(process.env.UNDERLAY_TEST_LAYOUT === 'f')('uploads into layout F', () => { + it('write records into packs once, and runs carry places, not bodies', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('a', { recordLayout: 'f' }) + const sid = await open(h, user, 'a') + const title = 't'.repeat(2000) + // The last of a key in a batch wins: only it goes in a pack. + await upload(h, user, 'a', sid, [ + ...Array.from({ length: 300 }, (_, i) => doc(id(i), `${title}${i}`)), + doc(id(0), 'replaced'), + ]) + const packs = packKeys(h, c.id) + // A pack is its blocks and their directory. + expect(packs).toHaveLength(2) + const entries = await sessionEntries(h, sid) + expect(entries).toHaveLength(300) + expect(entries.every((e) => e.b === undefined && typeof e.l === 'string')).toBe(true) + // The run holds places: a small part of the records' bytes. + const [run] = await h.ports.db + .select() + .from(schema.pushRuns) + .where(eq(schema.pushRuns.sessionId, sid)) + expect(run!.bytes).toBeLessThan((300 * title.length) / 10) + const rows = await h.ports.db + .select() + .from(schema.pushPacks) + .where(eq(schema.pushPacks.sessionId, sid)) + const written = rows.filter((r) => r.records !== null) + expect(written).toHaveLength(1) + expect(written[0]!.records).toBe(300) + + // The commit indexes them where they are: no record is written again. + expect(await commit(h, user, 'a', sid)).toBe(201) + expect(packKeys(h, c.id)).toEqual(packs) + const { out } = await headRecords(h, c.id) + expect(out).toHaveLength(300) + expect(out[0]).toBe(`public ${id(0)} replaced`) + expect(out[299]).toBe(`public ${id(299)} ${title}299`) + }) + + it('keep large records in packs, without staging them apart', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('big', { recordLayout: 'f' }) + const sid = await open(h, user, 'big') + const big = doc('big', 'x'.repeat(70_000)) + await upload(h, user, 'big', sid, [big, doc('small', 's')]) + const staged = [...h.bucket.objects.keys()].filter((k) => + k.startsWith(`internal/sessions/${sid}/records/`), + ) + expect(staged).toEqual([]) + expect(await commit(h, user, 'big', sid)).toBe(201) + expect([...h.bucket.objects.keys()].filter((k) => k.startsWith('repo/records/'))).toEqual([]) + const { out } = await headRecords(h, c.id) + expect(out).toEqual([`public big ${'x'.repeat(70_000)}`, 'public small s']) + }) + + it('put private records in the private set’s packs', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('p', { recordLayout: 'f' }) + const sid = await open(h, user, 'p') + await upload(h, user, 'p', sid, [ + doc('pub', 'open'), + { ...doc('priv', 'hidden'), private: true }, + ]) + expect(packKeys(h, c.id, 'public')).toHaveLength(2) + expect(packKeys(h, c.id, 'private')).toHaveLength(2) + expect(await commit(h, user, 'p', sid)).toBe(201) + expect((await headRecords(h, c.id)).out).toEqual(['public pub open', 'private priv hidden']) + }) + + it('count the files records reference, read at upload', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('files', { recordLayout: 'f' }) + const file = 'the pdf' + const withFile = doc('d1', 'has a file', { pdf: { $file: `sha256:${sha(file)}` } }) + // Not uploaded: the commit refuses, so it saw the reference. + const missing = await open(h, user, 'files') + await upload(h, user, 'files', missing, [withFile]) + expect(await commit(h, user, 'files', missing)).toBe(422) + + const put = await h.request(`/api/collections/org/files/files/${sha(file)}`, { + method: 'PUT', + user, + body: file, + }) + expect(put.status).toBeLessThan(300) + const sid = await open(h, user, 'files') + await upload(h, user, 'files', sid, [withFile]) + expect(await commit(h, user, 'files', sid)).toBe(201) + // The version's file references: the record's file, from the run entry alone. + const { v } = await headRecords(h, c.id) + expect(v.publicRefsRoot).not.toBeNull() + const repo = await h.ports.stores.forCollection(c.id) + const refs: string[] = [] + for await (const e of iterate(new RepoSource(countTree, repo), v.publicRefsRoot)) + refs.push(`${e.key} ${e.n}`) + expect(refs.some((r) => r.includes(sha(file)) && r.endsWith(' 1'))).toBe(true) + }) + + it('commit in parallel after compaction, the same versions as today’s layout', async () => { + const h = await harness() + const user = await h.member() + const today = await h.collection('today', { recordLayout: 'today' }) + const f = await h.collection('f', { recordLayout: 'f' }) + await h.ports.db + .update(schema.collections) + .set({ privateSalt: today.privateSalt }) + .where(eq(schema.collections.id, f.id)) + // Six uploads into a collection with a base: compacted, then committed in units. + const batches = Array.from({ length: 6 }, (_, j) => + Array.from({ length: 200 }, (_, i) => { + const n = (j * 7919 + i * 104_729) % 2000 + return doc(id(n), `v${j}-${n}`) + }), + ) + for (const slug of ['today', 'f']) { + const first = await open(h, user, slug) + await upload(h, user, slug, first, [doc(id(0), 'base'), doc(id(5000), 'kept')]) + expect(await commit(h, user, slug, first)).toBe(201) + Object.assign(compactConfig, { commitRuns: 2 }) + Object.assign(parallelConfig, { aboveBytes: 0, unitBytes: 32_000, baseLevel: 0 }) + const sid = await open(h, user, slug) + for (const b of batches) await upload(h, user, slug, sid, b) + const before = packKeys(h, f.id).length + expect(await commit(h, user, slug, sid)).toBe(202) + const [s] = await h.ports.db + .select() + .from(schema.pushSessions) + .where(eq(schema.pushSessions.id, sid)) + expect(s!.status).toBe('committed') + // F's units wrote no pack: every record was in one already. + if (slug === 'f') expect(packKeys(h, f.id).length).toBe(before) + Object.assign(compactConfig, defaults.compact) + Object.assign(parallelConfig, defaults.parallel) + } + const a = await headRecords(h, today.id) + const b = await headRecords(h, f.id) + expect(b.v.hash).toBe(a.v.hash) + expect(b.out).toEqual(a.out) + }) + + it('are listed once in the registry, and merged when small', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('l', { recordLayout: 'f' }) + // A batch past UPLOAD_PACK_FINAL_RAW: listed in the registry, never merged. + const sid = await open(h, user, 'l') + await upload( + h, + user, + 'l', + sid, + Array.from({ length: 600 }, (_, i) => doc(id(i), `${'y'.repeat(2000)}${i}`)), + ) + expect(await commit(h, user, 'l', sid)).toBe(201) + const registry = `repo/runs/${c.id}/public/manifest/f/packs/upload-${sid}.json` + expect(h.bucket.objects.has(registry)).toBe(true) + // Small pushes: their packs go in the manifest, and merges re-point their records. + for (let j = 0; j < 10; j++) { + const s = await open(h, user, 'l') + await upload(h, user, 'l', s, [doc(id(1000 + j), `small${j}`), doc(id(j), `changed${j}`)]) + expect(await commit(h, user, 'l', s)).toBe(201) + } + const { out } = await headRecords(h, c.id) + expect(out).toHaveLength(610) + expect(out.filter((r) => /changed|small/.test(r))).toHaveLength(20) + const repo = await h.ports.stores.forCollection(c.id) + const layout = repo.records!.public as unknown as { + manifests: { latest(): Promise<{ packs: { id: string; tier: number }[] }> } + } + const m = await layout.manifests.latest() + // Every small pack listed, or merged into one above them. + expect(m.packs.some((p) => p.tier > 0) || m.packs.length >= 10).toBe(true) + expect(new Set(m.packs.map((p) => p.id)).size).toBe(m.packs.length) + }) + + it('of a push that never publishes are deleted by cleanup; a published one’s are kept', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('gone', { recordLayout: 'f' }) + const kept = await open(h, user, 'gone') + await upload(h, user, 'gone', kept, [doc('k', 'kept')]) + expect(await commit(h, user, 'gone', kept)).toBe(201) + const keptPacks = packKeys(h, c.id) + expect(keptPacks).toHaveLength(2) + + const sid = await open(h, user, 'gone') + await upload(h, user, 'gone', sid, [doc('x', 'abandoned')]) + expect(packKeys(h, c.id)).toHaveLength(4) + const abort = await h.request(`/api/collections/org/gone/push/${sid}`, { + method: 'DELETE', + user, + }) + expect(abort.status).toBeLessThan(300) + // Past their sessions' grace. + await h.ports.db + .update(schema.pushSessions) + .set({ expiresAt: new Date(Date.now() - cleanupConfig.sessionGraceMs - DAY) }) + const dry = await startRun(h.ports, 'internal', { trigger: 'manual', dryRun: true }) + await h.drain() + expect((await getRun(h.ports.db, dry.id))!.stats!.deleted['upload packs']).toMatchObject({ + objects: 2, + }) + expect(packKeys(h, c.id)).toHaveLength(4) + + const r = await startRun(h.ports, 'internal', { trigger: 'manual' }) + await h.drain() + expect((await getRun(h.ports.db, r.id))!.status).toBe('done') + expect(packKeys(h, c.id).sort()).toEqual(keptPacks.sort()) + expect(await h.ports.db.select().from(schema.pushPacks)).toEqual([]) + expect((await headRecords(h, c.id)).out).toEqual(['public k kept']) + }) + + it('of a push whose commit started and never published are listed, not deleted', async () => { + // The failed commit may have indexed their records (the same record as the head's, + // say, whose newest place is then in such a pack): deleting them could break reads. + const h = await harness() + const user = await h.member() + const c = await h.collection('half', { recordLayout: 'f' }) + const first = await open(h, user, 'half') + await upload(h, user, 'half', first, [doc('same', 'unchanged')]) + expect(await commit(h, user, 'half', first)).toBe(201) + + const sid = await open(h, user, 'half') + const file = { pdf: { $file: `sha256:${sha('never uploaded')}` } } + await upload(h, user, 'half', sid, [doc('same', 'unchanged'), doc('new', 'needs a file', file)]) + // Refused for the missing file: the session is open again, then abandoned. + expect(await commit(h, user, 'half', sid)).toBe(422) + const abort = await h.request(`/api/collections/org/half/push/${sid}`, { + method: 'DELETE', + user, + }) + expect(abort.status).toBeLessThan(300) + const packs = packKeys(h, c.id) + await h.ports.db + .update(schema.pushSessions) + .set({ expiresAt: new Date(Date.now() - cleanupConfig.sessionGraceMs - DAY) }) + const r = await startRun(h.ports, 'internal', { trigger: 'manual' }) + await h.drain() + expect((await getRun(h.ports.db, r.id))!.status).toBe('done') + expect(packKeys(h, c.id)).toEqual(packs) + expect(await h.ports.db.select().from(schema.pushPacks)).toEqual([]) + // Listed with the collection's packs, for reclaim. + const repo = await h.ports.stores.forCollection(c.id) + const layout = repo.records!.public as unknown as { + manifests: { latest(): Promise<{ adopted?: string[] }> } + } + expect((await layout.manifests.latest()).adopted).toContain(sid) + expect((await headRecords(h, c.id)).out).toEqual(['public same unchanged']) + }) +}) From d5dd5b9c8f3e44005a79b4314b9240ab5b798747 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 20 Oct 2026 21:36:11 +0200 Subject: [PATCH 2/6] Layout F: key-ordered sessions stream records into whole packs; count overlapping uploads; pack rows from write hooks --- docs/protocol-v2.md | 38 +++-- packages/protocol/src/index.ts | 3 +- packages/protocol/src/repo/layouts/index.ts | 3 +- packages/protocol/src/repo/layouts/keyed.ts | 66 ++++---- packages/protocol/src/repo/layouts/kit.ts | 21 ++- packages/protocol/src/repo/layouts/types.ts | 26 +++- packages/protocol/test/repo/layouts.test.ts | 58 +++++-- packages/server/drizzle/0026_push_packs.sql | 4 +- .../server/drizzle/meta/0026_snapshot.json | 18 ++- packages/server/drizzle/meta/_journal.json | 2 +- packages/server/src/api/push.ts | 22 ++- packages/server/src/db/schema.ts | 12 ++ packages/server/src/push/delta.ts | 64 +++++--- packages/server/src/push/packs.ts | 144 +++++++++++------- packages/server/src/push/session.ts | 37 ++++- packages/server/test/upload-packs.test.ts | 92 ++++++++++- 16 files changed, 444 insertions(+), 166 deletions(-) diff --git a/docs/protocol-v2.md b/docs/protocol-v2.md index 3fd33258..ba33acc1 100644 --- a/docs/protocol-v2.md +++ b/docs/protocol-v2.md @@ -778,23 +778,25 @@ DELETE /push/ abandon the session **Opening a session.** The request body is a JSON object. Every member is OPTIONAL. -| Member | Meaning | -| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `base` | The semver of the version the changes are against. If it is not the collection's head, the server MUST answer 409 with `currentVersion`. `null` or absent: the head at the time of opening | -| `schemas` | The complete type set, slug → schema (Section 5). It replaces the base's type set; a type omitted is removed with its records. Absent: the base's type set | -| `metadata` | Replaces the base's metadata. An object or `null` | -| `metadata_patch` | An object whose top-level members are merged into the base's metadata. Ignored if `metadata` is present. With neither, the base's metadata is kept | -| `files` | `{"add": [fileHash, …], "remove": [fileHash, …]}`: files to declare or remove in addition to those records reference (Section 9) | -| `message` | A string recorded in the log entry | -| `app_id`, `actor_id` | Strings identifying the publishing application and actor | -| `strip_unknown_fields` | A boolean; see **Records** | - -The response is 200 with `{"session_id", "base", "needed_files", "expires_at", "limits"}`: +| Member | Meaning | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `base` | The semver of the version the changes are against. If it is not the collection's head, the server MUST answer 409 with `currentVersion`. `null` or absent: the head at the time of opening | +| `schemas` | The complete type set, slug → schema (Section 5). It replaces the base's type set; a type omitted is removed with its records. Absent: the base's type set | +| `metadata` | Replaces the base's metadata. An object or `null` | +| `metadata_patch` | An object whose top-level members are merged into the base's metadata. Ignored if `metadata` is present. With neither, the base's metadata is kept | +| `files` | `{"add": [fileHash, …], "remove": [fileHash, …]}`: files to declare or remove in addition to those records reference (Section 9) | +| `message` | A string recorded in the log entry | +| `app_id`, `actor_id` | Strings identifying the publishing application and actor | +| `strip_unknown_fields` | A boolean; see **Records** | +| `in_key_order` | A boolean: the client sends each records request's lines in key order (type, then id); see **Records**. A server MAY decline it (the response's `in_key_order` says), and MAY then allow larger requests | + +The response is 200 with `{"session_id", "base", "needed_files", "expires_at", "in_key_order", "limits"}`: - `base` is the semver the session is against, or `null` if the collection has no versions; - `needed_files` lists the declared files the server does not hold for the collection; the client MUST upload them before committing; - `expires_at` is when the session expires; it is extended by each records or deletes request; +- `in_key_order` is whether the server took the session as key-ordered; - `limits` are the server's limits, by which a client MUST size its requests: | Limit | Meaning | @@ -819,6 +821,10 @@ type's schema (Section 5.1), and its type MUST be in the session's type set. identified by its 1-based `line` number among the request's non-empty lines, and MUST store nothing from the request. A server MAY list only the first failures, with `totalErrors` giving their number. +- In a session the server took as `in_key_order`, each request's lines MUST come in key order: + by type, then id, as UTF-8 bytes (equal keys allowed; the later wins). A line before the one + ahead of it fails the request with 422, and nothing from it is stored. Order across requests is + the client's to keep; a server MAY report how many requests overlapped earlier ones' key ranges. **Deletes.** `POST …/deletes` takes NDJSON lines `{"type", "id"}` and answers 200 with `{"received": n}`. Each line MUST satisfy the syntax, duplicate key, unsafe integer and lone surrogate rules, and its `id` and `type` the record id and type slug rules (Section 3); the type MUST be in the session's type set. Deleting a (type, id) the base does @@ -861,7 +867,10 @@ server's storage: a client can upload a whole corpus into one session over many days, and commit it as one version. A server keeps the uploads in storage rather than memory, and a background commit's status MAY carry `progress` (underlay.org: `runs`, `run_bytes`, `runs_compacting`, `phase` of `compacting`, `planning`, `units` or `assembling`, `units` done of -`total`, and `progress_at`). An open session expires only after `session_idle_seconds` without an +`total`, and `progress_at`). underlay.org's status also carries `in_key_order` and +`overlapping_uploads`, the records requests whose key range overlapped an earlier one's: a corpus +sent in key order, with none overlapping, is stored with each record next to its neighbours, and +reads back with the fewest requests. An open session expires only after `session_idle_seconds` without an upload, so a client that stops can pick the same session up again by its id. Each version rewrites the leaves its changes land in, so one large version costs far less than the same records in many small ones. @@ -909,7 +918,8 @@ The reference implementation defines these in `packages/protocol/src/constants.t test vectors repeat them. Server limits (Section 11.4) are not protocol constants. Informative: underlay.org advertises -`open_bytes` 8 MiB, `batch_bytes` 8 MiB, `batch_lines` 10,000, `session_idle_seconds` 43,200, +`open_bytes` 8 MiB, `batch_bytes` 8 MiB (64 MiB in an `in_key_order` session), `batch_lines` +10,000 (50,000 in one), `session_idle_seconds` 43,200, `open_sessions` 20 and `file_bytes` 32 MiB; caps log pages at 1,000 entries, manifest pages at 25,000 records and diff pages at 5,000 records or 4 MiB of records; stores records over 64 KiB out of line; and commits in the background when a session uploaded more than 100,000 records or a diff --git a/packages/protocol/src/index.ts b/packages/protocol/src/index.ts index 5cb5a657..827fc821 100644 --- a/packages/protocol/src/index.ts +++ b/packages/protocol/src/index.ts @@ -20,8 +20,9 @@ export { type LiveSet, type MergePlan, type PackInfo, - packIdsFor, + type PackHooks, type PackRecord, + type UploadWriter, type Placed, MissingRecordError, RECLAIM_MAX_RUNS, diff --git a/packages/protocol/src/repo/layouts/index.ts b/packages/protocol/src/repo/layouts/index.ts index 2b3b6f5d..259f3e76 100644 --- a/packages/protocol/src/repo/layouts/index.ts +++ b/packages/protocol/src/repo/layouts/index.ts @@ -12,12 +12,11 @@ export { type MergePlan, MissingRecordError, RECLAIM_MAX_RUNS, - packIdsFor, type ReclaimPlan, TooManyRunsError, UPLOAD_PACK_FINAL_RAW, } from './keyed.js' -export { headConfig, type LayerInfo, type PackInfo, pendingConfig } from './kit.js' +export { headConfig, type LayerInfo, type PackHooks, type PackInfo, pendingConfig } from './kit.js' export * from './types.js' /** A set's home under the collection's: `runs///` holds its files. */ diff --git a/packages/protocol/src/repo/layouts/keyed.ts b/packages/protocol/src/repo/layouts/keyed.ts index bf191151..91a3ed31 100644 --- a/packages/protocol/src/repo/layouts/keyed.ts +++ b/packages/protocol/src/repo/layouts/keyed.ts @@ -38,7 +38,7 @@ import { newId, type PackInfo, PackReader, - PACK_RAW, + type PackHooks, PackWriter, packDirKey, packKey, @@ -57,6 +57,7 @@ import { type RecordStore, type RecordWriter, type TypedLeaf, + type UploadWriter, } from './types.js' /** Index entries a reclaim range job works on: its live set is 8 bytes an entry. */ @@ -142,18 +143,6 @@ export const UPLOAD_PACK_FINAL_RAW = 1024 * 1024 /** Commits whose uploads' packs a manifest remembers listing (`Manifest.adopted`). */ const ADOPTED_KEPT = 1000 -/** - * Ids for the packs `writePacks` makes of these records, recorded before they're - * written (so an upload that dies leaves nothing unknown). A pack closes only once - * it holds PACK_RAW (PackWriter, the same count of a line), so this many at most. - */ -export function packIdsFor(records: readonly PackRecord[]): string[] { - let raw = 0 - // A line is `\t` and its newline, the key `recordKey`'s: type, id, hash and two NULs. - for (const r of records) raw += r.type.length + r.id.length + r.hash.length + r.line.length + 4 - return Array.from({ length: Math.floor(raw / PACK_RAW) + 1 }, newId) -} - /** How long a replaced object stays readable (readers holding an older manifest). */ export const layoutConfig = { graceMs: 60_000, @@ -302,31 +291,36 @@ export class KeyedLayout implements RecordStore { } /** - * An upload's records into packs of their own, in key order, outside the index and - * the manifest until a commit names them (adoptPacks). Their places, in the order - * given, go in the push's runs, so a commit indexes the records without moving them. + * A writer of an upload's records into packs of its own, outside the index and the + * manifest until a commit names them (adoptPacks). Their places go in the push's + * runs, so a commit indexes the records without moving them. Records arriving in + * key order (an upload sent sorted) stream straight into whole packs; one below the + * last starts a new pack. */ + uploadWriter(hooks: PackHooks = {}): UploadWriter { + const pw = new PackWriter(this.store, { hooks }) + return { + append: (r) => pw.append(recordKey(r.type, r.id, r.hash), r.line), + ready: () => pw.ready(), + finish: () => pw.finish(), + } + } + + /** An upload's records into packs, sorted first: their places, in the order given. */ async writePacks( records: readonly PackRecord[], - ids: readonly string[], + hooks: PackHooks = {}, ): Promise<{ places: string[]; packs: PackInfo[] }> { - let next = 0 - const pw = new PackWriter(this.store, { - ids: () => { - const id = ids[next++] - if (id === undefined) throw new Error('writePacks: the records need more packs than ids') - return id - }, - }) + const w = this.uploadWriter(hooks) const keys = records.map((r) => recordKey(r.type, r.id, r.hash)) const order = keys.map((_, i) => i).sort((a, b) => compareUtf8(keys[a]!, keys[b]!)) const places: string[] = Array.from({ length: records.length }) let n = 0 for (const i of order) { - places[i] = pw.append(keys[i]!, records[i]!.line) - if (++n % 1024 === 0) await pw.ready() + places[i] = w.append(records[i]!) + if (++n % 1024 === 0) await w.ready() } - return { places, packs: await pw.finish() } + return { places, packs: await w.finish() } } /** @@ -353,16 +347,16 @@ export class KeyedLayout implements RecordStore { } /** - * Delete packs an upload wrote for a commit `name` that never named them (an - * abandoned push). A commit that named them and then didn't publish (a conflict, - * then the session failed) leaves them: the manifest or registry lists them, so - * they go only by reclaim. False when kept. + * Delete packs an upload wrote for commit `name`, but none the collection lists: + * a pack `adoptPacks` named (in the manifest, its retired files, or the registry + * object it wrote) goes only by reclaim. The ids deleted. */ - async dropPacks(name: string, ids: readonly string[]): Promise { + async dropPacks(name: string, ids: readonly string[]): Promise { const m = await this.manifests.latest() - if (m.adopted?.includes(name)) return false - if (await this.store.head(this.registry.keyFor(`upload-${name}`))) return false const listed = new Set([...m.packs.map((p) => p.id), ...m.retired.map((r) => r.key)]) + const registered = await this.store.get(this.registry.keyFor(`upload-${name}`)) + if (registered) + for (const p of JSON.parse(await registered.text()) as PackInfo[]) listed.add(p.id) const gone = ids.filter((id) => !listed.has(id) && !listed.has(packKey(id))) for (let i = 0; i < gone.length; i += 16) await Promise.all( @@ -370,7 +364,7 @@ export class KeyedLayout implements RecordStore { .slice(i, i + 16) .flatMap((id) => [this.store.delete(packKey(id)), this.store.delete(packDirKey(id))]), ) - return true + return gone } /** A commit's runs its units held back (RecordWriter.finish), as one pending object. */ diff --git a/packages/protocol/src/repo/layouts/kit.ts b/packages/protocol/src/repo/layouts/kit.ts index df5366c9..15919e66 100644 --- a/packages/protocol/src/repo/layouts/kit.ts +++ b/packages/protocol/src/repo/layouts/kit.ts @@ -76,6 +76,15 @@ export interface PackInfo { raw: number } +/** + * Called around each pack's write. An upload records a pack before it's written, so + * cleanup can find it if the upload dies, and its stats once it is. + */ +export interface PackHooks { + before?(id: string): Promise + after?(pack: PackInfo): Promise +} + export class PackWriter { #id = '' #blocks: Promise[] = [] @@ -90,15 +99,14 @@ export class PackWriter { constructor( readonly store: Store, - /** `ids` names each new pack (an upload's, recorded before they're written); else random. */ - readonly opts: { tier?: number; ids?: () => string } = {}, + readonly opts: { tier?: number; hooks?: PackHooks } = {}, ) {} /** Add a record line under `key`; returns its place. A key below the last starts a new pack. */ append(key: string, line: string): string { if (this.#lastKey !== null && compareUtf8(key, this.#lastKey) < 0) this.#close() this.#lastKey = key - if (!this.#id) this.#id = this.opts.ids ? this.opts.ids() : newId() + if (!this.#id) this.#id = newId() this.#lines.push(`${key}\t${line}`) const at = loc(this.#id, this.#blocks.length, this.#lines.length - 1) this.#raw += key.length + line.length + 2 @@ -152,15 +160,18 @@ export class PackWriter { const parts = await Promise.all(job.blocks) const offsets = [0] for (const p of parts) offsets.push(offsets.at(-1)! + p.byteLength) + await this.opts.hooks?.before?.(job.id) await putJoined(this.store, packKey(job.id), parts, { ifAbsent: true }) await this.store.put(packDirKey(job.id), JSON.stringify(offsets), { ifAbsent: true }) - this.done.push({ + const pack = { id: job.id, tier: this.opts.tier ?? 0, records: job.records, bytes: offsets.at(-1)!, raw: job.raw, - }) + } + this.done.push(pack) + await this.opts.hooks?.after?.(pack) } /** Close the current pack and wait for every write. */ diff --git a/packages/protocol/src/repo/layouts/types.ts b/packages/protocol/src/repo/layouts/types.ts index 7ece86e4..55c78b28 100644 --- a/packages/protocol/src/repo/layouts/types.ts +++ b/packages/protocol/src/repo/layouts/types.ts @@ -5,7 +5,7 @@ * holds. Neither changes a node, a root or any hash. */ import type { RecordEntry } from '../../tree/node.js' -import type { PackInfo, PendingRun } from './kit.js' +import type { PackHooks, PackInfo, PendingRun } from './kit.js' export const LAYOUTS = ['today', 'f'] as const export type LayoutName = (typeof LAYOUTS)[number] @@ -25,6 +25,16 @@ export interface PackRecord { line: string } +/** Writes an upload's records into packs (RecordStore.uploadWriter). */ +export interface UploadWriter { + /** Write one record: its place. */ + append(record: PackRecord): string + /** Wait while too much is held (call between records): memory stays bounded. */ + ready(): Promise + /** Write the last pack and wait for every write: the packs written. */ + finish(): Promise +} + /** Records still in use, from a mark over every retained version (storage reclaim). */ export interface LiveSet { /** `recordKey(type, id, hash)` of every record in a retained version. */ @@ -73,13 +83,15 @@ export interface RecordStore { /** Give a fork its records: it lists the parent's files and reads them where they are. */ forkInto?(target: RecordStore): Promise> /** - * Write an upload's records into packs of their own, named by `ids` (more than it - * needs; see `packIdsFor`), outside the index and the manifest: their places, in - * the order given, and the packs written. + * A writer of an upload's records into packs of its own, outside the index and the + * manifest until a commit names them (`adoptPacks`). Records in key order fill whole + * packs; one below the last starts a new pack. */ + uploadWriter?(hooks?: PackHooks): UploadWriter + /** An upload's records into packs, sorted first: their places, in the order given. */ writePacks?( records: readonly PackRecord[], - ids: readonly string[], + hooks?: PackHooks, ): Promise<{ places: string[]; packs: PackInfo[] }> /** * List packs uploads wrote, once the commit's index names their records: large @@ -87,8 +99,8 @@ export interface RecordStore { * commit's, so a retry lists nothing twice. */ adoptPacks?(name: string, packs: readonly PackInfo[]): Promise - /** Delete packs uploads wrote for commit `name`, unless it named them (false then). */ - dropPacks?(name: string, ids: readonly string[]): Promise + /** Delete packs uploads wrote for commit `name`, but none the collection lists: the ids deleted. */ + dropPacks?(name: string, ids: readonly string[]): Promise } export interface RecordWriter { diff --git a/packages/protocol/test/repo/layouts.test.ts b/packages/protocol/test/repo/layouts.test.ts index db5a43a7..6fa35320 100644 --- a/packages/protocol/test/repo/layouts.test.ts +++ b/packages/protocol/test/repo/layouts.test.ts @@ -25,7 +25,6 @@ import { mergeTree, type NodeSource, openRepo, - packIdsFor, type RecordEntry, recordCanonical, RECLAIM_MAX_RUNS, @@ -648,10 +647,17 @@ describe("an upload's packs", () => { // Ten small uploads, each into a pack of its own; their commits index them. for (let u = 0; u < 10; u++) { const recs = upload(u * 100, u * 100 + 100) - const ids_ = packIdsFor(recs) - expect(ids_).toHaveLength(1) - const { places, packs } = await layout.writePacks(recs, ids_) - expect(packs.map((p) => p.id)).toEqual(ids_) + // Each pack is announced before it's written, and reported once it is. + const seen: string[] = [] + const { places, packs } = await layout.writePacks(recs, { + before: async (id) => { + seen.push(`before ${id}`) + expect(await root.head(`runs/c1/packs/${id}.pk`)).toBeNull() + }, + after: async (p) => void seen.push(`after ${p.id} ${p.records}`), + }) + expect(packs).toHaveLength(1) + expect(seen).toEqual([`before ${packs[0]!.id}`, `after ${packs[0]!.id} 100`]) const before = await packsNow() const w = layout.writer() recs.forEach((_, i) => w.addPlaced('T', rec(u * 100 + (99 - i)), places[i]!)) @@ -677,6 +683,33 @@ describe("an upload's packs", () => { } }) + it('stream into whole packs when they arrive in key order', async () => { + // A sorted upload's records go straight into packs as they arrive: 16 MB raw + // each, not one pack a request. + const layout = fLayout(memoryStore()) + const big = (i: number) => ({ + type: 'T', + id: `k${String(i).padStart(5, '0')}`, + hash: `h${i}`, + line: `${i}:${'x'.repeat(20_000)}`, + }) + const sorted = layout.uploadWriter() + const places: string[] = [] + for (let i = 0; i < 1000; i++) { + places.push(sorted.append(big(i))) + await sorted.ready() + } + const packs = await sorted.finish() + // 20 MB of records: one full pack and the rest. + expect(packs).toHaveLength(2) + expect(packs[0]!.raw).toBeGreaterThanOrEqual(16 * 1024 * 1024) + expect(new Set(places.map((p) => p.split('.')[0])).size).toBe(2) + // Out of order, each step down starts a new pack. + const shuffled = layout.uploadWriter() + for (const i of [5, 6, 1, 2, 9, 3]) shuffled.append(big(i)) + expect(await shuffled.finish()).toHaveLength(3) + }) + it("count toward a commit writer's runs, written as it goes", async () => { // A commit of packed entries alone must not hold every index entry to its end (#63). const saved = layoutConfig.writerRunEntries @@ -684,7 +717,7 @@ describe("an upload's packs", () => { try { const layout = fLayout(memoryStore()) const recs = upload(0, 350) - const { places } = await layout.writePacks(recs, packIdsFor(recs)) + const { places } = await layout.writePacks(recs) const w = layout.writer({ hold: true }) for (let j = 0; j < 350; j++) { w.addPlaced('T', rec(349 - j), places[j]!) @@ -708,7 +741,7 @@ describe("an upload's packs", () => { const root = memoryStore() const layout = fLayout(root) const recs = upload(0, 50) - const { packs } = await layout.writePacks(recs, packIdsFor(recs)) + const { packs } = await layout.writePacks(recs) await layout.adoptPacks('s1', packs) await layout.adoptPacks('s1', packs) expect((await layout.manifests.latest()).packs.map((p) => p.id)).toEqual([packs[0]!.id]) @@ -728,14 +761,17 @@ describe("an upload's packs", () => { const root = memoryStore() const layout = fLayout(root) const named = upload(0, 20) - const a = await layout.writePacks(named, packIdsFor(named)) + const a = await layout.writePacks(named) await layout.adoptPacks('kept', a.packs) const left = upload(20, 40) - const b = await layout.writePacks(left, packIdsFor(left)) + const b = await layout.writePacks(left) const files = async () => (await root.list('runs/c1/packs/')).keys expect(await files()).toHaveLength(4) - expect(await layout.dropPacks('kept', [a.packs[0]!.id])).toBe(false) - expect(await layout.dropPacks('left', [b.packs[0]!.id, 'neverwritten'])).toBe(true) + expect(await layout.dropPacks('kept', [a.packs[0]!.id])).toEqual([]) + expect(await layout.dropPacks('left', [b.packs[0]!.id, 'neverwritten'])).toEqual([ + b.packs[0]!.id, + 'neverwritten', + ]) expect((await files()).sort()).toEqual( [`runs/c1/packs/${a.packs[0]!.id}.dir`, `runs/c1/packs/${a.packs[0]!.id}.pk`].sort(), ) diff --git a/packages/server/drizzle/0026_push_packs.sql b/packages/server/drizzle/0026_push_packs.sql index 70d99ec2..f0d9856c 100644 --- a/packages/server/drizzle/0026_push_packs.sql +++ b/packages/server/drizzle/0026_push_packs.sql @@ -10,4 +10,6 @@ CREATE TABLE `push_packs` ( FOREIGN KEY (`session_id`) REFERENCES `push_sessions`(`id`) ON UPDATE no action ON DELETE cascade ); --> statement-breakpoint -ALTER TABLE `push_sessions` ADD `commit_started` integer DEFAULT false NOT NULL; \ No newline at end of file +ALTER TABLE `push_sessions` ADD `commit_started` integer DEFAULT false NOT NULL;--> statement-breakpoint +ALTER TABLE `push_sessions` ADD `in_key_order` integer DEFAULT false NOT NULL;--> statement-breakpoint +ALTER TABLE `push_sessions` ADD `overlapping_uploads` integer DEFAULT 0 NOT NULL; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0026_snapshot.json b/packages/server/drizzle/meta/0026_snapshot.json index 5a88d9a7..6ababddb 100644 --- a/packages/server/drizzle/meta/0026_snapshot.json +++ b/packages/server/drizzle/meta/0026_snapshot.json @@ -1,7 +1,7 @@ { "version": "6", "dialect": "sqlite", - "id": "da1a4cfe-5e4b-4fb0-9240-470d050217f9", + "id": "b7f7c1c5-2c58-4d38-a78e-dec398756445", "prevId": "42b4a563-0b06-417e-a83f-2216ffba5eb8", "tables": { "abuse_reports": { @@ -2316,6 +2316,22 @@ "autoincrement": false, "default": false }, + "in_key_order": { + "name": "in_key_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "overlapping_uploads": { + "name": "overlapping_uploads", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, "cleaned_at": { "name": "cleaned_at", "type": "integer", diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index 225cdc62..e60d3a5b 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -187,7 +187,7 @@ { "idx": 26, "version": "6", - "when": 1792518549719, + "when": 1792524725308, "tag": "0026_push_packs", "breakpoints": true } diff --git a/packages/server/src/api/push.ts b/packages/server/src/api/push.ts index adceb665..30f1368b 100644 --- a/packages/server/src/api/push.ts +++ b/packages/server/src/api/push.ts @@ -22,9 +22,9 @@ import type { Ports } from '../ports.js' import { headBase, ingestDeletes, + batchLimits, ingestRecords, MAX_BATCH_BYTES, - MAX_BATCH_LINES, } from '../push/delta.js' import { finalizeSession } from '../push/finalize.js' import { unitProgress } from '../push/recover.js' @@ -54,10 +54,10 @@ export const commitConfig = { asyncAbove: 100_000 } * This node's push limits, advertised when a session opens (docs/protocol-v2.md, * section 11.4). Read on each call: tests lower some of them. */ -export const pushLimits = () => ({ +export const pushLimits = (session: Pick = { inKeyOrder: false }) => ({ open_bytes: MAX_OPEN_BYTES, - batch_bytes: MAX_BATCH_BYTES, - batch_lines: MAX_BATCH_LINES, + batch_bytes: batchLimits(session).bytes, + batch_lines: batchLimits(session).lines, session_idle_seconds: SESSION_TTL_MS / 1000, open_sessions: limits.openSessions, file_bytes: SMALL_UPLOAD_BYTES, @@ -248,6 +248,10 @@ async function statusBody(ports: Ports, s: SessionRow) { expires_at: s.expiresAt, created_at: s.createdAt, finalize_started_at: s.finalizeStartedAt, + // Whether the uploads came in key order: an upload that overlapped an earlier + // one's key range leaves records scattered across packs (layout F). + in_key_order: s.inKeyOrder, + overlapping_uploads: s.overlappingUploads, result: s.result ?? null, error: s.error ?? null, } @@ -356,6 +360,8 @@ export function pushRoutes() { appId: str(body.app_id), actorId: str(body.actor_id), stripUnknownFields: body.strip_unknown_fields === true, + // Only layout F packs records at upload: elsewhere it changes nothing. + inKeyOrder: body.in_key_order === true && access.collection.recordLayout === 'f', }, inputs, ) @@ -369,7 +375,8 @@ export function pushRoutes() { inputs.files && 'add' in inputs.files ? inputs.files.add : [], ), expires_at: session.expiresAt, - limits: pushLimits(), + in_key_order: session.inKeyOrder, + limits: pushLimits(session), }) }) @@ -379,12 +386,13 @@ export function pushRoutes() { const session = await ownSession(c, access) if (session instanceof Response) return session // Parsed line by line as the body arrives (v2-scale-review.md S6). + const max = batchLimits(session).bytes let r try { - r = await ingestRecords(c.var.ports, session, readLines(c, MAX_BATCH_BYTES)) + r = await ingestRecords(c.var.ports, session, readLines(c, max)) } catch (err) { if (err instanceof BodyTooLarge) - return jsonError(c, 413, `Batches are limited to ${MAX_BATCH_BYTES} bytes`) + return jsonError(c, 413, `Batches are limited to ${max} bytes`) throw err } return r.ok diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index 39d87f99..f59a0be9 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -613,6 +613,18 @@ export const pushSessions = sqliteTable( * (push/packs.ts), so cleanup leaves those to reclaim instead of deleting them. */ commitStarted: bool('commit_started').notNull().default(false), + /** + * Opened `in_key_order` on a layout F collection: each batch's records come by + * type, then id, and stream straight into packs (push/packs.ts), so batches + * may be larger (push/delta.ts `batchLimits`). + */ + inKeyOrder: bool('in_key_order').notNull().default(false), + /** + * Uploads whose key range overlapped an earlier upload's (or a compaction's) + * when they landed: 0 means the session's uploads came in key order, and its + * packs hold neighbouring records. + */ + overlappingUploads: integer('overlapping_uploads').notNull().default(0), /** When storage cleanup deleted the session's objects (sessions//) and run rows. */ cleanedAt: ts('cleaned_at'), /** diff --git a/packages/server/src/push/delta.ts b/packages/server/src/push/delta.ts index 153b3b98..4f8237d6 100644 --- a/packages/server/src/push/delta.ts +++ b/packages/server/src/push/delta.ts @@ -31,7 +31,7 @@ import type { Ports } from '../ports.js' import { type BaseVersion, commitVersion, type TypeInput } from '../versions/commit.js' import { deltaChanges, isPrivateSchema, stageOutOfLine } from './changes.js' import type { SessionCommitResult } from './outcome.js' -import { packEntries, packStats } from './packs.js' +import { keyOrderPlacer, OutOfKeyOrder, packEntries } from './packs.js' import { planParallel, SerialTooLargeError } from './parallel.js' import { type RunEntry, type RunIndex, writeRun } from './runs.js' import { @@ -49,6 +49,19 @@ import { /** 8 MB: an upload request holds its batch several times over (text, records, run lines). */ export const MAX_BATCH_BYTES = 8 * 1024 * 1024 export const MAX_BATCH_LINES = 10_000 +/** + * A key-ordered session's batches (layout F, `in_key_order`): records stream into + * packs as they arrive, so a request holds only its run entries (about 150 bytes a + * record) and the pack filling. Larger batches mean fewer runs to compact. + */ +export const KEY_ORDER_BATCH_BYTES = 64 * 1024 * 1024 +export const KEY_ORDER_BATCH_LINES = 50_000 + +/** A session's batch limits: larger for one whose records stream into packs. */ +export const batchLimits = (session: Pick) => + session.inKeyOrder + ? { bytes: KEY_ORDER_BATCH_BYTES, lines: KEY_ORDER_BATCH_LINES } + : { bytes: MAX_BATCH_BYTES, lines: MAX_BATCH_LINES } const MAX_REPORTED = 100 export interface LineError { @@ -83,7 +96,13 @@ export async function prepareRecords( inputs: SessionInputs, input: string | AsyncIterable, /** `packed`: the records go into packs (layout F), so large ones aren't staged apart. */ - opts: { stripUnknownFields: boolean; packed?: boolean }, + opts: { + stripUnknownFields: boolean + packed?: boolean + maxLines?: number + /** Each accepted record as it's read (a key-ordered session's packing), with its line number. */ + place?: (e: RunEntry, line: number) => Promise + }, ): Promise< { entries: RunEntry[] } | { errors: LineError[]; total: number } | { tooManyLines: true } > { @@ -115,7 +134,7 @@ export async function prepareRecords( let i = -1 for await (const line of source) { i++ - if (i >= MAX_BATCH_LINES) return { tooManyLines: true } + if (i >= (opts.maxLines ?? MAX_BATCH_LINES)) return { tooManyLines: true } let rec try { rec = parseRecordLine(line) @@ -180,6 +199,8 @@ export async function prepareRecords( ...(isPrivate ? { p: true } : {}), ...(refs.length ? { r: refs } : {}), } + // A batch with an invalid record is refused whole: nothing more is packed. + if (opts.place && total === 0) await opts.place(entry, i + 1) entries.push(entry) } return total > 0 ? { errors, total } : { entries } @@ -191,14 +212,28 @@ export async function ingestRecords( input: string | AsyncIterable, ): Promise { const inputs = await loadInputs(ports, session.id) - // Layout F: the records go straight into the collection's packs. + // Layout F: the records go straight into the collection's packs; a key-ordered + // session's as they arrive. const records = (await ports.stores.forCollection(session.collectionId)).records - const prepared = await prepareRecords(ports, session.id, inputs, input, { - stripUnknownFields: session.stripUnknownFields, - packed: records !== null, - }) + const placer = records && session.inKeyOrder ? keyOrderPlacer(ports, session.id, records) : null + const limits = batchLimits(session) + let prepared + try { + prepared = await prepareRecords(ports, session.id, inputs, input, { + stripUnknownFields: session.stripUnknownFields, + packed: records !== null, + maxLines: limits.lines, + ...(placer ? { place: placer.place } : {}), + }) + } catch (err) { + // Packs already written are reported (their rows' stats), so cleanup settles them. + await placer?.finish().catch(() => {}) + if (err instanceof OutOfKeyOrder) return { ok: false, status: 422, error: err.message } + throw err + } + await placer?.finish() if ('tooManyLines' in prepared) { - return { ok: false, status: 413, error: `At most ${MAX_BATCH_LINES} records per batch` } + return { ok: false, status: 413, error: `At most ${limits.lines} records per batch` } } if ('errors' in prepared) { return { @@ -212,16 +247,9 @@ export async function ingestRecords( if (prepared.entries.length === 0) return { ok: false, status: 400, error: 'Empty batch' } const seq = await nextRunSeq(ports, session.id) if (seq === null) return { ok: false, status: 409, error: 'Session is not open' } - const packs = records ? await packEntries(ports, session.id, records, prepared.entries) : [] + if (records && !placer) await packEntries(ports, session.id, records, prepared.entries) const index = await writeRun(ports.stores.internal, session.id, seq, prepared.entries) - await recordRun( - ports, - session.id, - 'records', - index, - { records: prepared.entries.length }, - packStats(ports, session.id, packs), - ) + await recordRun(ports, session.id, 'records', index, { records: prepared.entries.length }) return { ok: true, received: prepared.entries.length } } diff --git a/packages/server/src/push/packs.ts b/packages/server/src/push/packs.ts index 8fd12631..dd5d8580 100644 --- a/packages/server/src/push/packs.ts +++ b/packages/server/src/push/packs.ts @@ -6,31 +6,48 @@ * compacting 206 GB, 2026-10-19), and commit units index the records without * reading or writing them again. * + * A session opened `in_key_order` (its uploads sorted by type and id) streams each + * record into a pack as it arrives (`keyOrderPlacer`): packs fill to 16 MB, hold + * neighbouring records, and a batch isn't held in memory, so batches can be larger. + * Otherwise a batch is sorted and packed once it's read (`packEntries`), a pack a + * batch, in whatever order the batches come. + * * A pack is outside the collection's index and manifest until a commit names it * (`adoptSessionPacks`, just before the version publishes). Each is recorded in - * `push_packs` before it's written, so storage cleanup can delete the packs of a - * push that never published (`dropSessionPacks`). + * `push_packs` before it's written and its stats once it is, so storage cleanup can + * settle the packs of a push that never published (`settleSessionPacks`). */ import { - packIdsFor, + type PackHooks, type PackInfo, type RecordSet, type RecordSets, type Repo, + type UploadWriter, } from '@underlay/protocol' import { and, eq, isNotNull } from 'drizzle-orm' import * as schema from '../db/schema.js' import type { Ports } from '../ports.js' -import { type RunEntry, runKey } from './runs.js' +import { compareRunKeys, type RunEntry, runKey } from './runs.js' const SETS = ['public', 'private'] as const -/** A pack an upload wrote, for its row's stats. */ -export interface UploadPack { - set: RecordSet - pack: PackInfo -} +/** An upload marks an entry private exactly when it goes to the private set (deltaChanges). */ +const setOf = (e: RunEntry): RecordSet => (e.p ? 'private' : 'public') + +/** Each pack recorded before it's written, and its stats once it is. */ +const hooksFor = (ports: Ports, sessionId: string, set: RecordSet): PackHooks => ({ + before: async (packId) => { + await ports.db.insert(schema.pushPacks).values({ sessionId, packId, set }) + }, + after: async (pack) => { + await ports.db + .update(schema.pushPacks) + .set({ records: pack.records, bytes: pack.bytes, raw: pack.raw }) + .where(and(eq(schema.pushPacks.sessionId, sessionId), eq(schema.pushPacks.packId, pack.id))) + }, +}) /** * Write a batch's records into packs, a set each, and give their entries places @@ -42,38 +59,58 @@ export async function packEntries( sessionId: string, records: RecordSets, entries: RunEntry[], -): Promise { +): Promise { const last = new Map() for (const e of entries) last.set(runKey(e), e) - const out: UploadPack[] = [] for (const set of SETS) { - // An upload marks an entry private exactly when it goes to the private set (deltaChanges). - const group = [...last.values()].filter((e) => (e.p === true) === (set === 'private')) + const group = [...last.values()].filter((e) => setOf(e) === set) if (!group.length) continue const recs = group.map((e) => ({ type: e.t, id: e.k, hash: e.h!, line: e.b! })) - const ids = packIdsFor(recs) - // Recorded before they're written: an upload that dies leaves no pack cleanup can't find. - await ports.db - .insert(schema.pushPacks) - .values(ids.map((packId) => ({ sessionId, packId, set }))) - const { places, packs } = await records[set].writePacks!(recs, ids) + const { places } = await records[set].writePacks!(recs, hooksFor(ports, sessionId, set)) group.forEach((e, i) => { e.l = places[i]! delete e.b }) - out.push(...packs.map((pack) => ({ set, pack }))) } - return out } -/** The statements that record written packs' stats (run with the upload's run row). */ -export const packStats = (ports: Ports, sessionId: string, packs: UploadPack[]) => - packs.map(({ pack }) => - ports.db - .update(schema.pushPacks) - .set({ records: pack.records, bytes: pack.bytes, raw: pack.raw }) - .where(and(eq(schema.pushPacks.sessionId, sessionId), eq(schema.pushPacks.packId, pack.id))), - ) +/** A record of a key-ordered upload that comes before the one ahead of it. */ +export class OutOfKeyOrder extends Error { + constructor(readonly line: number) { + super( + `Records out of key order at line ${line}: this session was opened in_key_order, so each batch's records go by type, then id`, + ) + } +} + +/** + * Places a key-ordered batch's records as they arrive: each straight into its set's + * pack, its entry left with a place instead of a body. `finish` writes the last + * packs. A record before the one ahead of it throws OutOfKeyOrder. + */ +export function keyOrderPlacer(ports: Ports, sessionId: string, records: RecordSets) { + const writers = new Map() + let last: RunEntry | null = null + let n = 0 + return { + async place(e: RunEntry, line: number): Promise { + if (last && compareRunKeys(e, last) < 0) throw new OutOfKeyOrder(line) + last = e + const set = setOf(e) + let w = writers.get(set) + if (!w) { + w = records[set].uploadWriter!(hooksFor(ports, sessionId, set)) + writers.set(set, w) + } + e.l = w.append({ type: e.t, id: e.k, hash: e.h!, line: e.b! }) + delete e.b + if (++n % 256 === 0) await w.ready() + }, + async finish(): Promise { + for (const w of writers.values()) await w.finish() + }, + } +} /** * List a session's packs in the collection's manifest or registry. Called once the @@ -96,14 +133,16 @@ export async function adoptSessionPacks(ports: Ports, repo: Repo, sessionId: str /** * At storage cleanup, settle a finished session's packs, then drop its rows: - * - a version names the session: its commit listed them, nothing to do; + * - a pack with no stats was never reported written, so no recorded run places a + * record in it: deleted, whatever became of the session; + * - a version names the session: its commit listed the rest, nothing to do; * - no commit of it ever started: nothing indexes their records, so they're deleted; * - a commit started and never published: its index may name their records (one the * collection already held, say, whose newest place is now such a pack), so they're * listed instead, and reclaim drops what no version uses. - * Returns the objects and bytes deleted (a pack is two objects), counting packs that - * were written. A collection whose repository won't open (deleted) is left to the - * sweep, which drops its files. + * A pack the collection lists is never deleted here (KeyedLayout.dropPacks). Returns + * the objects and bytes deleted (a pack is two objects). A collection whose + * repository won't open (deleted) is left to the sweep, which drops its files. */ export async function settleSessionPacks( ports: Ports, @@ -127,28 +166,25 @@ export async function settleSessionPacks( } catch { repo = null } - if (!published && repo?.records) { - if (session.commitStarted) { - if (!dryRun) await adoptSessionPacks(ports, repo, session.id) - } else { - for (const set of SETS) { - const mine = rows.filter((r) => r.set === set) - if (!mine.length) continue - // Refused if a commit listed them after all (KeyedLayout.dropPacks). - const dropped = - dryRun || - (await repo.records[set].dropPacks!( + if (repo?.records) { + const keepWritten = !!published || session.commitStarted + if (!published && session.commitStarted && !dryRun) + await adoptSessionPacks(ports, repo, session.id) + for (const set of SETS) { + const drop = rows.filter((r) => r.set === set && (r.records === null || !keepWritten)) + if (!drop.length) continue + const ids = dryRun + ? drop.map((r) => r.packId) + : await repo.records[set].dropPacks!( session.id, - mine.map((r) => r.packId), - )) - if (!dropped) continue - // A row without stats is a pack never written, or one whose upload died writing it. - for (const r of mine) - if (r.records !== null) { - gone.objects += 2 - gone.bytes += r.bytes ?? 0 - } - } + drop.map((r) => r.packId), + ) + const dropped = new Set(ids) + for (const r of drop) + if (dropped.has(r.packId) && r.records !== null) { + gone.objects += 2 + gone.bytes += r.bytes ?? 0 + } } } if (!dryRun) diff --git a/packages/server/src/push/session.ts b/packages/server/src/push/session.ts index f9f286eb..03a206c2 100644 --- a/packages/server/src/push/session.ts +++ b/packages/server/src/push/session.ts @@ -6,7 +6,7 @@ * list. */ import { checkSchemaFull, hashSchema } from '@underlay/protocol' -import { and, asc, count, eq, gt, or, sql } from 'drizzle-orm' +import { and, asc, count, eq, gt, gte, lte, ne, or, sql } from 'drizzle-orm' import * as schema from '../db/schema.js' import type { Ports } from '../ports.js' @@ -44,8 +44,6 @@ export interface SessionInputs { export type SessionRow = typeof schema.pushSessions.$inferSelect -type BatchStatement = Parameters[0][number] - const inputsKey = (id: string) => `sessions/${id}/inputs.json` export async function createSession( @@ -115,8 +113,6 @@ export async function recordRun( kind: 'records' | 'deletes', index: RunIndex, counters: { records?: number }, - /** More statements for the same batch (packs.ts: the upload's packs' stats). */ - extra: BatchStatement[] = [], ): Promise { const count = runCount(index) await ports.db.batch([ @@ -137,11 +133,40 @@ export async function recordRun( recordsReceived: sql`${schema.pushSessions.recordsReceived} + ${counters.records ?? 0}`, }) .where(eq(schema.pushSessions.id, sessionId)), - ...extra, ]) + if (kind === 'records' && index.parts.length) await noteOverlap(ports, sessionId, index) await maybeCompact(ports, sessionId, index.tier ?? 0) } +/** + * Count an upload whose key range overlaps another run of the session's: checked + * once the upload's own run is recorded, so of two that land together the later + * check sees the earlier (a compacted run's range spans its inputs', gaps included). + */ +async function noteOverlap(ports: Ports, sessionId: string, index: RunIndex): Promise { + const r = schema.pushRuns + const first = index.parts[0]!.first + const last = index.parts[index.parts.length - 1]!.last + const [hit] = await ports.db + .select({ seq: r.seq }) + .from(r) + .where( + and( + eq(r.sessionId, sessionId), + eq(r.kind, 'records'), + ne(r.seq, index.seq), + lte(r.firstKey, last), + gte(r.lastKey, first), + ), + ) + .limit(1) + if (!hit) return + await ports.db + .update(schema.pushSessions) + .set({ overlappingUploads: sql`${schema.pushSessions.overlappingUploads} + 1` }) + .where(eq(schema.pushSessions.id, sessionId)) +} + export async function sessionRuns( ports: Ports, sessionId: string, diff --git a/packages/server/test/upload-packs.test.ts b/packages/server/test/upload-packs.test.ts index 83f04eb9..2c3d7ba9 100644 --- a/packages/server/test/upload-packs.test.ts +++ b/packages/server/test/upload-packs.test.ts @@ -41,16 +41,20 @@ async function json(res: Response) { return (await res.json()) as any } -async function open(h: Harness, user: string, slug: string) { +async function open(h: Harness, user: string, slug: string, extra: object = {}) { const res = await h.request(`/api/collections/org/${slug}/push`, { method: 'POST', user, - json: { schemas: { Doc } }, + json: { schemas: { Doc }, ...extra }, }) expect(res.status).toBe(200) return (await json(res)).session_id as string } +async function status(h: Harness, user: string, slug: string, sid: string) { + return json(await h.request(`/api/collections/org/${slug}/push/${sid}`, { user })) +} + async function upload(h: Harness, user: string, slug: string, sid: string, batch: object[]) { const r = await h.request(`/api/collections/org/${slug}/push/${sid}/records`, { method: 'POST', @@ -362,4 +366,88 @@ describe.runIf(process.env.UNDERLAY_TEST_LAYOUT === 'f')('uploads into layout F' expect((await layout.manifests.latest()).adopted).toContain(sid) expect((await headRecords(h, c.id)).out).toEqual(['public same unchanged']) }) + describe('in key order', () => { + // A batch's records by id, from `from`; ids sort as their numbers do. + const sorted = (from: number, n: number, title = 'r') => + Array.from({ length: n }, (_, i) => doc(id(from + i), `${title}${from + i}`)) + + it('stream into whole packs, take larger batches, and count no overlaps', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('k', { recordLayout: 'f' }) + const res = await h.request('/api/collections/org/k/push', { + method: 'POST', + user, + json: { schemas: { Doc }, in_key_order: true }, + }) + const opened = await json(res) + expect(opened.in_key_order).toBe(true) + expect(opened.limits).toMatchObject({ batch_bytes: 64 * 1024 * 1024, batch_lines: 50_000 }) + const sid = opened.session_id as string + // One batch over the default 8 MB: a single pack, its records together. + const big = (from: number) => + Array.from({ length: 1100 }, (_, i) => doc(id(from + i), `${'z'.repeat(8000)}${from + i}`)) + await upload(h, user, 'k', sid, big(0)) + expect(packKeys(h, c.id)).toHaveLength(2) + await upload(h, user, 'k', sid, sorted(5000, 50)) + expect((await status(h, user, 'k', sid)).overlapping_uploads).toBe(0) + expect(await commit(h, user, 'k', sid)).toBe(201) + const { out } = await headRecords(h, c.id) + expect(out).toHaveLength(1150) + expect(out[1100]).toBe(`public ${id(5000)} r5000`) + }) + + it('refuse a batch out of order, and cleanup deletes what it packed', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('o', { recordLayout: 'f' }) + const sid = await open(h, user, 'o', { in_key_order: true }) + const batch = [...sorted(0, 300), doc(id(10), 'behind')] + const r = await h.request(`/api/collections/org/o/push/${sid}/records`, { + method: 'POST', + user, + ndjson: batch, + }) + expect(r.status).toBe(422) + expect((await json(r)).error).toMatch(/out of key order at line 301/) + // What it packed before the refusal is reported, for cleanup. + const rows = await h.ports.db + .select() + .from(schema.pushPacks) + .where(eq(schema.pushPacks.sessionId, sid)) + expect(rows.length).toBeGreaterThan(0) + expect(rows.every((row) => row.records !== null)).toBe(true) + expect((await status(h, user, 'o', sid)).records_received).toBe(0) + await h.request(`/api/collections/org/o/push/${sid}`, { method: 'DELETE', user }) + await h.ports.db + .update(schema.pushSessions) + .set({ expiresAt: new Date(Date.now() - cleanupConfig.sessionGraceMs - DAY) }) + await startRun(h.ports, 'internal', { trigger: 'manual' }) + await h.drain() + expect(packKeys(h, c.id)).toEqual([]) + }) + + it('is counted when uploads overlap, and ignored on today’s layout', async () => { + const h = await harness() + const user = await h.member() + await h.collection('ov', { recordLayout: 'f' }) + const sid = await open(h, user, 'ov') + await upload(h, user, 'ov', sid, sorted(0, 100)) + await upload(h, user, 'ov', sid, sorted(200, 100)) + expect((await status(h, user, 'ov', sid)).overlapping_uploads).toBe(0) + // Ids 50–149: inside the first upload's range. + await upload(h, user, 'ov', sid, sorted(50, 100)) + expect((await status(h, user, 'ov', sid)).overlapping_uploads).toBe(1) + + await h.collection('t', { recordLayout: 'today' }) + const res = await h.request('/api/collections/org/t/push', { + method: 'POST', + user, + json: { schemas: { Doc }, in_key_order: true }, + }) + const opened = await json(res) + expect(opened.in_key_order).toBe(false) + expect(opened.limits.batch_bytes).toBe(8 * 1024 * 1024) + }) + }) }) From f332ec98b96a1b237f4b7e464377ab3dc2860381 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 21 Oct 2026 20:03:46 +0200 Subject: [PATCH 3/6] Layout F: packs never span types, and merges take one type's packs --- packages/protocol/src/repo/layouts/keyed.ts | 34 +- packages/protocol/src/repo/layouts/kit.ts | 41 +- packages/protocol/test/repo/layouts.test.ts | 52 + .../server/drizzle/0027_push_pack_types.sql | 1 + .../server/drizzle/meta/0027_snapshot.json | 3689 +++++++++++++++++ packages/server/drizzle/meta/_journal.json | 7 + packages/server/src/db/schema.ts | 2 + packages/server/src/push/packs.ts | 11 +- packages/server/test/upload-packs.test.ts | 9 +- 9 files changed, 3824 insertions(+), 22 deletions(-) create mode 100644 packages/server/drizzle/0027_push_pack_types.sql create mode 100644 packages/server/drizzle/meta/0027_snapshot.json diff --git a/packages/protocol/src/repo/layouts/keyed.ts b/packages/protocol/src/repo/layouts/keyed.ts index 91a3ed31..49002598 100644 --- a/packages/protocol/src/repo/layouts/keyed.ts +++ b/packages/protocol/src/repo/layouts/keyed.ts @@ -540,28 +540,36 @@ export class KeyedLayout implements RecordStore { }) } - /** Merge eight small packs of a tier into one, re-pointing their records in a new layer. */ + /** + * Merge eight small packs of a tier and type into one, re-pointing their records in + * a new layer. A merge takes one type's packs, so its output holds one type and a + * type's records gather in fewer packs (a leaf read of one type at three years of + * Crossref dailies, simulated: 72 GETs at the median where mixing types took 255). + * Packs from before types were recorded merge among themselves. + */ async #mergePacks(): Promise { const m = await this.manifests.latest() - const tiers = new Map() + const queues = new Map() for (const p of m.packs) { if (KeyedLayout.final(p)) continue - tiers.set(p.tier, [...(tiers.get(p.tier) ?? []), p]) + const q = `${p.tier}\u0000${p.type ?? ''}` + const queue = queues.get(q) ?? { tier: p.tier, packs: [] } + queue.packs.push(p) + queues.set(q, queue) } - // The tier with the most packs: the lowest that has enough never runs out while - // commits land, so higher tiers never merged (round 14 at 20,000 collections: - // 718 tier-1 packs listed and rising). - const [tier, most] = [...tiers].reduce<[number, number]>( - (best, [t, ps]) => (ps.length > best[1] ? [t, ps.length] : best), - [-1, 0], - ) - if (most < FANOUT) return false - // As many of the tier's packs as fit MERGE_PACKS_BYTES stored (at least FANOUT): many + // The queue with the most packs: the lowest tier that has enough never runs out + // while commits land, so higher tiers never merged (round 14 at 20,000 + // collections: 718 tier-1 packs listed and rising). + let queue: { tier: number; packs: PackInfo[] } | undefined + for (const q of queues.values()) if (q.packs.length > (queue?.packs.length ?? 0)) queue = q + if (!queue || queue.packs.length < FANOUT) return false + const tier = queue.tier + // As many of the queue's packs as fit MERGE_PACKS_BYTES stored (at least FANOUT): many // collections' tiny commits each write a tiny pack, and eight at a time fell behind // (6,639 waiting at 15,000 collections, every commit reading them all). const inputs: PackInfo[] = [] let stored = 0 - for (const p of tiers.get(tier)!) { + for (const p of queue.packs) { if ( inputs.length >= FANOUT && (stored + p.bytes > MERGE_PACKS_BYTES || inputs.length >= layoutConfig.mergeInputs) diff --git a/packages/protocol/src/repo/layouts/kit.ts b/packages/protocol/src/repo/layouts/kit.ts index 15919e66..defdbccc 100644 --- a/packages/protocol/src/repo/layouts/kit.ts +++ b/packages/protocol/src/repo/layouts/kit.ts @@ -74,6 +74,17 @@ export interface PackInfo { /** Stored (compressed) bytes, and raw. */ bytes: number raw: number + /** + * The record type the pack holds: a pack closes where the type changes, so no pack + * spans types and merges take one type's packs. Absent on packs written before. + */ + type?: string +} + +/** The record type a key leads with (recordKey); '' for a key without one. */ +const typeOf = (key: string) => { + const i = key.indexOf('\u0000') + return i < 0 ? '' : key.slice(0, i) } /** @@ -93,6 +104,7 @@ export class PackWriter { #packRaw = 0 #records = 0 #lastKey: string | null = null + #type = '' readonly done: PackInfo[] = [] readonly #writing = new Set>() #error: unknown = null @@ -102,9 +114,15 @@ export class PackWriter { readonly opts: { tier?: number; hooks?: PackHooks } = {}, ) {} - /** Add a record line under `key`; returns its place. A key below the last starts a new pack. */ + /** + * Add a record line under `key`; returns its place. A key below the last, or of + * another type, starts a new pack. + */ append(key: string, line: string): string { - if (this.#lastKey !== null && compareUtf8(key, this.#lastKey) < 0) this.#close() + const type = typeOf(key) + if (this.#lastKey !== null && (type !== this.#type || compareUtf8(key, this.#lastKey) < 0)) + this.#close() + this.#type = type this.#lastKey = key if (!this.#id) this.#id = newId() this.#lines.push(`${key}\t${line}`) @@ -143,7 +161,13 @@ export class PackWriter { this.#seal() this.#lastKey = null if (!this.#blocks.length) return - const job = { id: this.#id, blocks: this.#blocks, records: this.#records, raw: this.#packRaw } + const job = { + id: this.#id, + blocks: this.#blocks, + records: this.#records, + raw: this.#packRaw, + type: this.#type, + } this.#id = '' this.#blocks = [] this.#records = 0 @@ -156,19 +180,26 @@ export class PackWriter { this.#writing.add(p) } - async #write(job: { id: string; blocks: Promise[]; records: number; raw: number }) { + async #write(job: { + id: string + blocks: Promise[] + records: number + raw: number + type: string + }) { const parts = await Promise.all(job.blocks) const offsets = [0] for (const p of parts) offsets.push(offsets.at(-1)! + p.byteLength) await this.opts.hooks?.before?.(job.id) await putJoined(this.store, packKey(job.id), parts, { ifAbsent: true }) await this.store.put(packDirKey(job.id), JSON.stringify(offsets), { ifAbsent: true }) - const pack = { + const pack: PackInfo = { id: job.id, tier: this.opts.tier ?? 0, records: job.records, bytes: offsets.at(-1)!, raw: job.raw, + ...(job.type ? { type: job.type } : {}), } this.done.push(pack) await this.opts.hooks?.after?.(pack) diff --git a/packages/protocol/test/repo/layouts.test.ts b/packages/protocol/test/repo/layouts.test.ts index 6fa35320..ac361307 100644 --- a/packages/protocol/test/repo/layouts.test.ts +++ b/packages/protocol/test/repo/layouts.test.ts @@ -392,6 +392,58 @@ describe('pack merges', () => { }, 30_000) }) +describe('packs never span types', () => { + it('close where the type changes, and say which type they hold', async () => { + const layout = fLayout() + const w = layout.uploadWriter() + for (const type of ['A', 'B', 'C']) + for (let i = 0; i < 5; i++) + w.append({ type, id: ids[i]!, hash: `h${i}`, line: `${type}${i}` }) + // Packs are reported as their writes finish, in any order. + const packs = await w.finish() + expect(packs.map((p) => `${p.type}: ${p.records}`).sort()).toEqual(['A: 5', 'B: 5', 'C: 5']) + }) + + it("merge one type's packs at a time, keeping every record readable", async () => { + const saved = { ...layoutConfig } + layoutConfig.graceMs = 0 + // Eight packs a merge: eight of one type make a pack of 32 records, where eight of + // both types would make two of 16 and tier up half-full. + layoutConfig.mergeInputs = 8 + try { + const layout = fLayout() + // Tiny commits, each of two types: every commit writes a pack of each. + const types = ['A', 'B'] + const rec = (i: number) => ({ key: ids[i]!, hash: `h${i}`, size: 10 }) + for (let c = 0; c < 16; c++) { + const w = layout.writer() + for (const type of types) + for (let i = c * 4; i < c * 4 + 4; i++) w.add(type, rec(i), `${type} ${i}`) + await w.finish() + } + const before = (await layout.manifests.latest()).packs + expect(before).toHaveLength(32) + expect(new Set(before.map((p) => p.type))).toEqual(new Set(types)) + while (await layout.maintain()); + const after = (await layout.manifests.latest()).packs + expect(after.map((p) => `${p.type} tier ${p.tier}: ${p.records}`).sort()).toEqual([ + 'A tier 1: 32', + 'A tier 1: 32', + 'B tier 1: 32', + 'B tier 1: 32', + ]) + for (const type of types) { + const entries = Array.from({ length: 64 }, (_, i) => rec(i)) + expect(await layout.leafLines({ hash: 'leaf', type, entries })).toEqual( + entries.map((_, i) => `${type} ${i}`), + ) + } + } finally { + Object.assign(layoutConfig, saved) + } + }) +}) + describe('layout F', () => { const rec = (i: number, v = 1) => ({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 }) const line = (i: number, v = 1) => `line ${v}.${i}` diff --git a/packages/server/drizzle/0027_push_pack_types.sql b/packages/server/drizzle/0027_push_pack_types.sql new file mode 100644 index 00000000..33b1b2a5 --- /dev/null +++ b/packages/server/drizzle/0027_push_pack_types.sql @@ -0,0 +1 @@ +ALTER TABLE `push_packs` ADD `record_type` text; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0027_snapshot.json b/packages/server/drizzle/meta/0027_snapshot.json new file mode 100644 index 00000000..967ecd8c --- /dev/null +++ b/packages/server/drizzle/meta/0027_snapshot.json @@ -0,0 +1,3689 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "6fed31a6-5812-4a14-a6b3-b5bc8df442b5", + "prevId": "b7f7c1c5-2c58-4d38-a78e-dec398756445", + "tables": { + "abuse_reports": { + "name": "abuse_reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "contact": { + "name": "contact", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reporter_id": { + "name": "reporter_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "resolved_by": { + "name": "resolved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "abuse_reports_status_idx": { + "name": "abuse_reports_status_idx", + "columns": ["status", "created_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 86400000 + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 10 + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_request": { + "name": "last_request", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": ["key"], + "isUnique": false + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": ["reference_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_collections": { + "name": "ark_collections", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "ark_id": { + "name": "ark_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "custom_url": { + "name": "custom_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_collections_ark_id_unique": { + "name": "ark_collections_ark_id_unique", + "columns": ["ark_id"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_collections_collection_id_collections_id_fk": { + "name": "ark_collections_collection_id_collections_id_fk", + "tableFrom": "ark_collections", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_record_types": { + "name": "ark_record_types", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirect_url_field": { + "name": "redirect_url_field", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "ark_record_types_collection_id_collections_id_fk": { + "name": "ark_record_types_collection_id_collections_id_fk", + "tableFrom": "ark_record_types", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "ark_record_types_collection_id_record_type_pk": { + "columns": ["collection_id", "record_type"], + "name": "ark_record_types_collection_id_record_type_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_shoulders": { + "name": "ark_shoulders", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "shoulder": { + "name": "shoulder", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_shoulders_shoulder_unique": { + "name": "ark_shoulders_shoulder_unique", + "columns": ["shoulder"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_shoulders_organization_id_organization_id_fk": { + "name": "ark_shoulders_organization_id_organization_id_fk", + "tableFrom": "ark_shoulders", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cleanup_runs": { + "name": "cleanup_runs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "step": { + "name": "step", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dry_run": { + "name": "dry_run", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "requested_by": { + "name": "requested_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mark_run_id": { + "name": "mark_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lease": { + "name": "lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stats": { + "name": "stats", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "cleanup_runs_created_idx": { + "name": "cleanup_runs_created_idx", + "columns": ["created_at"], + "isUnique": false + }, + "cleanup_runs_step_idx": { + "name": "cleanup_runs_step_idx", + "columns": ["step", "status"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_tombstones": { + "name": "collection_tombstones", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "versions": { + "name": "versions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_by": { + "name": "deleted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_layout": { + "name": "record_layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'today'" + }, + "records_dropped_at": { + "name": "records_dropped_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_webhooks": { + "name": "collection_webhooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bump_filter": { + "name": "bump_filter", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_delivery_at": { + "name": "last_delivery_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "collection_webhooks_collection_idx": { + "name": "collection_webhooks_collection_idx", + "columns": ["collection_id"], + "isUnique": false + } + }, + "foreignKeys": { + "collection_webhooks_collection_id_collections_id_fk": { + "name": "collection_webhooks_collection_id_collections_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collections": { + "name": "collections", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "head_version_id": { + "name": "head_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_salt": { + "name": "private_salt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_files_root": { + "name": "public_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "version_count": { + "name": "version_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "history_bytes": { + "name": "history_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_push_at": { + "name": "last_push_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_started_at": { + "name": "reconcile_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_state": { + "name": "reconcile_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_seq": { + "name": "reconciled_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "reconciled_files_root": { + "name": "reconciled_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_layout": { + "name": "record_layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'today'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "collections_org_slug_uq": { + "name": "collections_org_slug_uq", + "columns": ["organization_id", "slug"], + "isUnique": true + }, + "collections_public_updated_idx": { + "name": "collections_public_updated_idx", + "columns": ["public", "updated_at"], + "isUnique": false + } + }, + "foreignKeys": { + "collections_organization_id_organization_id_fk": { + "name": "collections_organization_id_organization_id_fk", + "tableFrom": "collections", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "commit_units": { + "name": "commit_units", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "plan_id": { + "name": "plan_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ord": { + "name": "ord", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "after": { + "name": "after", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "through": { + "name": "through", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "gap": { + "name": "gap", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "slices_key": { + "name": "slices_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_key": { + "name": "output_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "queued_at": { + "name": "queued_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "commit_units_plan_idx": { + "name": "commit_units_plan_idx", + "columns": ["plan_id", "set", "type", "ord"], + "isUnique": false + }, + "commit_units_status_idx": { + "name": "commit_units_status_idx", + "columns": ["session_id", "status", "queued_at"], + "isUnique": false + } + }, + "foreignKeys": { + "commit_units_session_id_push_sessions_id_fk": { + "name": "commit_units_session_id_push_sessions_id_fk", + "tableFrom": "commit_units", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "denylist": { + "name": "denylist", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "file_uploads": { + "name": "file_uploads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "multipart_upload_id": { + "name": "multipart_upload_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "file_uploads_hash_idx": { + "name": "file_uploads_hash_idx", + "columns": ["hash"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "files": { + "name": "files", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "forks": { + "name": "forks", + "columns": { + "child_collection_id": { + "name": "child_collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "parent_collection_id": { + "name": "parent_collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_seq": { + "name": "parent_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'public'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "forks_child_collection_id_collections_id_fk": { + "name": "forks_child_collection_id_collections_id_fk", + "tableFrom": "forks", + "tableTo": "collections", + "columnsFrom": ["child_collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instance_settings": { + "name": "instance_settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invitation": { + "name": "invitation", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "jobs": { + "name": "jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "run_at": { + "name": "run_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locked_until": { + "name": "locked_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "jobs_ready_idx": { + "name": "jobs_ready_idx", + "columns": ["status", "run_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "member": { + "name": "member", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + }, + "member_user_id_idx": { + "name": "member_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organization": { + "name": "organization", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ark_naan": { + "name": "ark_naan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kf_org_id": { + "name": "kf_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "columns": ["slug"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "placements": { + "name": "placements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location_id": { + "name": "location_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "synced_seq": { + "name": "synced_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until": { + "name": "lease_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "placements_one_primary_uq": { + "name": "placements_one_primary_uq", + "columns": ["collection_id"], + "isUnique": true, + "where": "\"placements\".\"role\" = 'primary' AND \"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_collection_location_uq": { + "name": "placements_collection_location_uq", + "columns": ["collection_id", "location_id"], + "isUnique": true, + "where": "\"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_org_location_uq": { + "name": "placements_org_location_uq", + "columns": ["organization_id", "location_id"], + "isUnique": true, + "where": "\"placements\".\"organization_id\" IS NOT NULL" + }, + "placements_location_idx": { + "name": "placements_location_idx", + "columns": ["location_id"], + "isUnique": false + } + }, + "foreignKeys": { + "placements_collection_id_collections_id_fk": { + "name": "placements_collection_id_collections_id_fk", + "tableFrom": "placements", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_organization_id_organization_id_fk": { + "name": "placements_organization_id_organization_id_fk", + "tableFrom": "placements", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_location_id_storage_locations_id_fk": { + "name": "placements_location_id_storage_locations_id_fk", + "tableFrom": "placements", + "tableTo": "storage_locations", + "columnsFrom": ["location_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_packs": { + "name": "push_packs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pack_id": { + "name": "pack_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_set": { + "name": "record_set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "records": { + "name": "records", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw": { + "name": "raw", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "push_packs_session_id_push_sessions_id_fk": { + "name": "push_packs_session_id_push_sessions_id_fk", + "tableFrom": "push_packs", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_packs_session_id_pack_id_pk": { + "columns": ["session_id", "pack_id"], + "name": "push_packs_session_id_pack_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_runs": { + "name": "push_runs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "first_key": { + "name": "first_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_key": { + "name": "last_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "merging_into": { + "name": "merging_into", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "push_runs_tier_idx": { + "name": "push_runs_tier_idx", + "columns": ["session_id", "tier", "merging_into"], + "isUnique": false + } + }, + "foreignKeys": { + "push_runs_session_id_push_sessions_id_fk": { + "name": "push_runs_session_id_push_sessions_id_fk", + "tableFrom": "push_runs", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_runs_session_id_seq_pk": { + "columns": ["session_id", "seq"], + "name": "push_runs_session_id_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_sessions": { + "name": "push_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_version_id": { + "name": "base_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "strip_unknown_fields": { + "name": "strip_unknown_fields", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "records_received": { + "name": "records_received", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "runs": { + "name": "runs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "result": { + "name": "result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finalize_started_at": { + "name": "finalize_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_started": { + "name": "commit_started", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "in_key_order": { + "name": "in_key_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "overlapping_uploads": { + "name": "overlapping_uploads", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cleaned_at": { + "name": "cleaned_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_phase": { + "name": "commit_phase", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "progress_at": { + "name": "progress_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stalls": { + "name": "stalls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "commit_plan": { + "name": "commit_plan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "assembly_lease": { + "name": "assembly_lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "push_sessions_collection_idx": { + "name": "push_sessions_collection_idx", + "columns": ["collection_id"], + "isUnique": false + }, + "push_sessions_expires_idx": { + "name": "push_sessions_expires_idx", + "columns": ["status", "expires_at"], + "isUnique": false + }, + "push_sessions_user_idx": { + "name": "push_sessions_user_idx", + "columns": ["user_id", "status"], + "isUnique": false + } + }, + "foreignKeys": { + "push_sessions_collection_id_collections_id_fk": { + "name": "push_sessions_collection_id_collections_id_fk", + "tableFrom": "push_sessions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compaction_parts": { + "name": "ref_compaction_parts", + "columns": { + "compaction_id": { + "name": "compaction_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "part": { + "name": "part", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_compaction_parts_compaction_id_part_pk": { + "columns": ["compaction_id", "part"], + "name": "ref_compaction_parts_compaction_id_part_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compactions": { + "name": "ref_compactions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_runs": { + "name": "input_runs", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output_run": { + "name": "output_run", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parts": { + "name": "parts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix_length": { + "name": "prefix_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'running'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_index_units": { + "name": "ref_index_units", + "columns": { + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unit": { + "name": "unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "events": { + "name": "events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_index_units_version_id_unit_pk": { + "columns": ["version_id", "unit"], + "name": "ref_index_units_version_id_unit_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_segments": { + "name": "ref_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_hash": { + "name": "first_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_hash": { + "name": "last_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'live'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ref_segments_range_idx": { + "name": "ref_segments_range_idx", + "columns": ["state", "first_hash", "last_hash"], + "isUnique": false + }, + "ref_segments_run_idx": { + "name": "ref_segments_run_idx", + "columns": ["run_id"], + "isUnique": false + }, + "ref_segments_tier_idx": { + "name": "ref_segments_tier_idx", + "columns": ["tier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_labels": { + "name": "schema_labels", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "schema_labels_label_idx": { + "name": "schema_labels_label_idx", + "columns": ["label"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "schema_labels_schema_hash_label_pk": { + "columns": ["schema_hash", "label"], + "name": "schema_labels_schema_hash_label_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_usage": { + "name": "schema_usage", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type_slug": { + "name": "type_slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_seq": { + "name": "from_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "to_seq": { + "name": "to_seq", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "schema_usage_hash_idx": { + "name": "schema_usage_hash_idx", + "columns": ["schema_hash"], + "isUnique": false + } + }, + "foreignKeys": { + "schema_usage_collection_id_collections_id_fk": { + "name": "schema_usage_collection_id_collections_id_fk", + "tableFrom": "schema_usage", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "schema_usage_collection_id_type_slug_set_from_seq_pk": { + "columns": ["collection_id", "type_slug", "set", "from_seq"], + "name": "schema_usage_collection_id_type_slug_set_from_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schemas": { + "name": "schemas", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": ["token"], + "isUnique": true + }, + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_fence": { + "name": "storage_fence", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "epoch": { + "name": "epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "window_until": { + "name": "window_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "window_run_id": { + "name": "window_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_locations": { + "name": "storage_locations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucket": { + "name": "bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "credentials": { + "name": "credentials", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unverified'" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checked_at": { + "name": "checked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "storage_locations_organization_id_organization_id_fk": { + "name": "storage_locations_organization_id_organization_id_fk", + "tableFrom": "storage_locations", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "usage_rollups": { + "name": "usage_rollups", + "columns": { + "day": { + "name": "day", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metric": { + "name": "metric", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "amount": { + "name": "amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "usage_rollups_account_idx": { + "name": "usage_rollups_account_idx", + "columns": ["account_id", "day"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "usage_rollups_day_account_id_collection_id_metric_pk": { + "columns": ["day", "account_id", "collection_id", "metric"], + "name": "usage_rollups_day_account_id_collection_id_metric_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": ["email"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": ["identifier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "versions": { + "name": "versions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "major": { + "name": "major", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "minor": { + "name": "minor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "patch": { + "name": "patch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pushed_by": { + "name": "pushed_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_record_count": { + "name": "public_record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_file_count": { + "name": "public_file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "public_total_bytes": { + "name": "public_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "type_counts": { + "name": "type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_type_counts": { + "name": "public_type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "has_private": { + "name": "has_private", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "public_refs_root": { + "name": "public_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_refs_root": { + "name": "private_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refs_indexed": { + "name": "refs_indexed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "changes": { + "name": "changes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "push_session_id": { + "name": "push_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "versions_collection_seq_uq": { + "name": "versions_collection_seq_uq", + "columns": ["collection_id", "seq"], + "isUnique": true + }, + "versions_collection_semver_uq": { + "name": "versions_collection_semver_uq", + "columns": ["collection_id", "semver"], + "isUnique": true + }, + "versions_hash_idx": { + "name": "versions_hash_idx", + "columns": ["hash"], + "isUnique": false + }, + "versions_published_idx": { + "name": "versions_published_idx", + "columns": ["published_at"], + "isUnique": false + }, + "versions_push_session_idx": { + "name": "versions_push_session_idx", + "columns": ["push_session_id"], + "isUnique": false + } + }, + "foreignKeys": { + "versions_collection_id_collections_id_fk": { + "name": "versions_collection_id_collections_id_fk", + "tableFrom": "versions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhook_deliveries": { + "name": "webhook_deliveries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "webhook_id": { + "name": "webhook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bump_type": { + "name": "bump_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'version.created'" + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "delivered_at": { + "name": "delivered_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhook_deliveries_webhook_idx": { + "name": "webhook_deliveries_webhook_idx", + "columns": ["webhook_id", "created_at"], + "isUnique": false + }, + "webhook_deliveries_pending_idx": { + "name": "webhook_deliveries_pending_idx", + "columns": ["status", "next_attempt_at"], + "isUnique": false + } + }, + "foreignKeys": { + "webhook_deliveries_webhook_id_collection_webhooks_id_fk": { + "name": "webhook_deliveries_webhook_id_collection_webhooks_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collection_webhooks", + "columnsFrom": ["webhook_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index e60d3a5b..ca816c24 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -190,6 +190,13 @@ "when": 1792524725308, "tag": "0026_push_packs", "breakpoints": true + }, + { + "idx": 27, + "version": "6", + "when": 1792605564163, + "tag": "0027_push_pack_types", + "breakpoints": true } ] } diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index f59a0be9..bb4a59b5 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -699,6 +699,8 @@ export const pushPacks = sqliteTable( records: integer('records'), bytes: integer('bytes'), raw: integer('raw'), + /** The record type the pack holds (PackInfo.type): merges take one type's packs. */ + recordType: text('record_type'), createdAt: createdAt(), }, (t) => [primaryKey({ columns: [t.sessionId, t.packId] })], diff --git a/packages/server/src/push/packs.ts b/packages/server/src/push/packs.ts index dd5d8580..8fcaafa5 100644 --- a/packages/server/src/push/packs.ts +++ b/packages/server/src/push/packs.ts @@ -44,7 +44,7 @@ const hooksFor = (ports: Ports, sessionId: string, set: RecordSet): PackHooks => after: async (pack) => { await ports.db .update(schema.pushPacks) - .set({ records: pack.records, bytes: pack.bytes, raw: pack.raw }) + .set({ records: pack.records, bytes: pack.bytes, raw: pack.raw, recordType: pack.type }) .where(and(eq(schema.pushPacks.sessionId, sessionId), eq(schema.pushPacks.packId, pack.id))) }, }) @@ -126,7 +126,14 @@ export async function adoptSessionPacks(ports: Ports, repo: Repo, sessionId: str for (const set of SETS) { const packs: PackInfo[] = rows .filter((r) => r.set === set) - .map((r) => ({ id: r.packId, tier: 0, records: r.records!, bytes: r.bytes!, raw: r.raw! })) + .map((r) => ({ + id: r.packId, + tier: 0, + records: r.records!, + bytes: r.bytes!, + raw: r.raw!, + ...(r.recordType ? { type: r.recordType } : {}), + })) if (packs.length) await repo.records[set].adoptPacks!(sessionId, packs) } } diff --git a/packages/server/test/upload-packs.test.ts b/packages/server/test/upload-packs.test.ts index 2c3d7ba9..58b0fc99 100644 --- a/packages/server/test/upload-packs.test.ts +++ b/packages/server/test/upload-packs.test.ts @@ -143,6 +143,8 @@ describe.runIf(process.env.UNDERLAY_TEST_LAYOUT === 'f')('uploads into layout F' const written = rows.filter((r) => r.records !== null) expect(written).toHaveLength(1) expect(written[0]!.records).toBe(300) + // Its type, so a merge once it's listed takes it with its own type's packs. + expect(written[0]!.recordType).toBe('Doc') // The commit indexes them where they are: no record is written again. expect(await commit(h, user, 'a', sid)).toBe(201) @@ -284,11 +286,14 @@ describe.runIf(process.env.UNDERLAY_TEST_LAYOUT === 'f')('uploads into layout F' expect(out.filter((r) => /changed|small/.test(r))).toHaveLength(20) const repo = await h.ports.stores.forCollection(c.id) const layout = repo.records!.public as unknown as { - manifests: { latest(): Promise<{ packs: { id: string; tier: number }[] }> } + manifests: { + latest(): Promise<{ packs: { id: string; tier: number; type?: string }[] }> + } } const m = await layout.manifests.latest() - // Every small pack listed, or merged into one above them. + // Every small pack listed, or merged into one above them, and each says its type. expect(m.packs.some((p) => p.tier > 0) || m.packs.length >= 10).toBe(true) + expect(m.packs.every((p) => p.type === 'Doc')).toBe(true) expect(new Set(m.packs.map((p) => p.id)).size).toBe(m.packs.length) }) From a083f5fc60ed37fc33b1107b282bc7bd411803c9 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 21 Oct 2026 20:24:50 +0200 Subject: [PATCH 4/6] Layout F: leveled packs, merges by level with replaced versions in history packs --- packages/protocol/src/repo/layouts/keyed.ts | 387 +- packages/protocol/src/repo/layouts/kit.ts | 42 +- packages/protocol/src/repo/layouts/levels.ts | 206 + packages/protocol/test/repo/leveled.test.ts | 242 ++ .../server/drizzle/0028_push_pack_ranges.sql | 2 + .../server/drizzle/meta/0028_snapshot.json | 3703 +++++++++++++++++ packages/server/drizzle/meta/_journal.json | 7 + packages/server/src/db/schema.ts | 6 + packages/server/src/push/packs.ts | 10 +- packages/server/src/stores.ts | 7 + packages/server/src/worker.ts | 3 + packages/server/test/cleanup.test.ts | 5 +- packages/server/test/harness.ts | 1 + packages/server/test/manage.test.ts | 7 +- packages/server/test/upload-packs.test.ts | 35 +- 15 files changed, 4631 insertions(+), 32 deletions(-) create mode 100644 packages/protocol/src/repo/layouts/levels.ts create mode 100644 packages/protocol/test/repo/leveled.test.ts create mode 100644 packages/server/drizzle/0028_push_pack_ranges.sql create mode 100644 packages/server/drizzle/meta/0028_snapshot.json diff --git a/packages/protocol/src/repo/layouts/keyed.ts b/packages/protocol/src/repo/layouts/keyed.ts index 49002598..d540290a 100644 --- a/packages/protocol/src/repo/layouts/keyed.ts +++ b/packages/protocol/src/repo/layouts/keyed.ts @@ -49,6 +49,16 @@ import { layerKey, filterKey, } from './kit.js' +import { + LEVEL_RATIO, + levelConfig, + type LevelShard, + LevelShards, + levelRaw, + levelTargets, + MAX_LEVEL, + shardKey, +} from './levels.js' import { type KeyRange, LiveHashes } from './reclaim.js' import { type LiveSet, @@ -113,6 +123,10 @@ export interface MergePlan { const mergeKey = (plan: string, name: string) => `merges/${plan}/${name}` +/** Whether a level lists the same shards in both (nothing changed it between). */ +const sameShards = (a: readonly LevelShard[] | undefined, b: readonly LevelShard[] | undefined) => + (a ?? []).map((s) => s.id).join() === (b ?? []).map((s) => s.id).join() + export interface MaintainOptions { /** * Run a merge too large for one job as range jobs (the server queues @@ -142,6 +156,8 @@ const MAX_PACK_TIER = 3 export const UPLOAD_PACK_FINAL_RAW = 1024 * 1024 /** Commits whose uploads' packs a manifest remembers listing (`Manifest.adopted`). */ const ADOPTED_KEPT = 1000 +/** Present in a leveled collection's prefix (`KeyedLayout.markLeveled`). */ +const LEVELED_KEY = 'manifest/f/leveled.json' /** How long a replaced object stays readable (readers holding an older manifest). */ export const layoutConfig = { @@ -164,6 +180,11 @@ export const layoutConfig = { * calls stay well inside what `maintainCalls` leaves (tests set it low). */ mergeInputs: 128, + /** + * Level a collection's packs when its first write finds it empty (levels.ts); one + * holding packs already stays tiered. The server switches it on. + */ + levelNew: false, } /** @@ -198,10 +219,13 @@ export class KeyedLayout implements RecordStore { readonly packs: PackReader readonly index: LayeredIndex readonly layers: LayerReader + /** A leveled collection's level listings (levels.ts). */ + readonly levels: LevelShards /** Every storage call this layout makes goes through `store`, counted here. */ readonly #io = { calls: 0 } readonly store: Store + #leveled: { v: Promise; retry: number } | null = null readonly name = 'f' @@ -216,6 +240,7 @@ export class KeyedLayout implements RecordStore { this.packs = new PackReader(store, lru, scope) this.index = new LayeredIndex(store, lru, scope, this.manifests) this.layers = this.index.reader + this.levels = new LevelShards(store) } /** Whether no pack merge will take `p` again: it then lives in the registry, not the manifest. */ @@ -223,10 +248,68 @@ export class KeyedLayout implements RecordStore { return p.tier >= MAX_PACK_TIER || p.bytes >= SMALL_PACK * 8 ** p.tier } - /** Every pack: the manifest's and the registry's (reclaim, forks). */ + /** + * Whether this collection's packs are leveled (levels.ts): marked once (`markLeveled`), + * by its first write while `layoutConfig.levelNew` is on, or by a fork of one that is. + * A collection holding packs when leveling was switched on stays tiered. Remembered + * once known. + */ + leveled(): Promise { + if (!this.#leveled || Date.now() > this.#leveled.retry) { + const known = { v: Promise.resolve(false), retry: Infinity } + known.v = this.#isLeveled(known) + this.#leveled = known + } + return this.#leveled.v + } + + async #isLeveled(known: { retry: number }): Promise { + if (await this.store.head(LEVELED_KEY)) return true + if (!layoutConfig.levelNew) { + // Read again later: leveling may be switched on meanwhile (the next write decides). + known.retry = Date.now() + 5000 + return false + } + const m = await this.manifests.latest() + const empty = !m.layers.length && !m.packs.length && !(await this.registry.read()).keys.length + if (empty) { + await this.store.put(LEVELED_KEY, '{}', { ifAbsent: true }) + return true + } + // Another writer may have marked it just as this one looked. + return (await this.store.head(LEVELED_KEY)) !== null + } + + /** Make this collection leveled, before anything is written to it. */ + async markLeveled(): Promise { + await this.store.put(LEVELED_KEY, '{}', { ifAbsent: true }) + this.#leveled = { v: Promise.resolve(true), retry: Infinity } + } + + /** + * Whether `p` is listed in the registry rather than the manifest or a level: in a + * leveled collection only history packs are (every other is merged again). + */ + #final(p: PackInfo, leveled: boolean): boolean { + return leveled ? !!p.history : KeyedLayout.final(p) + } + + /** Every pack of levels 1… , by level. */ + async #levelPacks(m: Manifest): Promise> { + const out = new Map() + for (const [l, shards] of Object.entries(m.levels ?? {})) + if (shards.length) out.set(Number(l), await this.levels.packs(shards)) + return out + } + + /** Every pack: the manifest's, the levels' and the registry's (reclaim, forks). */ async allPacks(): Promise { const m = await this.manifests.latest() - return [...m.packs, ...(await this.#registered(m)).packs] + return [ + ...m.packs, + ...[...(await this.#levelPacks(m)).values()].flat(), + ...(await this.#registered(m)).packs, + ] } /** @@ -333,6 +416,7 @@ export class KeyedLayout implements RecordStore { async adoptPacks(name: string, packs: readonly PackInfo[]): Promise { if (!packs.length) return if ((await this.manifests.latest()).adopted?.includes(name)) return + if (await this.leveled()) return this.#adoptLeveled(name, packs) const large = (p: PackInfo) => p.raw >= UPLOAD_PACK_FINAL_RAW || KeyedLayout.final(p) // A fixed key: written once even if this stops before the manifest says so. await this.registry.add(packs.filter(large), `upload-${name}`) @@ -346,6 +430,42 @@ export class KeyedLayout implements RecordStore { }) } + /** + * Leveled: an upload's packs go to level 0, to be merged down. But a load sent in + * key order (packs that never overlap) into an empty collection is already what + * the bottom level is: listed there as written, never rewritten to get there (a + * Crossref-sized load, 1 TB, would otherwise go down every level). + */ + async #adoptLeveled(name: string, packs: readonly PackInfo[]): Promise { + const sorted = [...packs].sort((a, b) => compareUtf8(a.lo ?? '', b.lo ?? '')) + const disjoint = sorted.every( + (p, i) => p.lo && p.hi && (i === 0 || compareUtf8(p.lo, sorted[i - 1]!.hi!) > 0), + ) + const empty = (m: Manifest) => + !m.packs.length && !Object.values(m.levels ?? {}).some((s) => s.length) + if (disjoint && empty(await this.manifests.latest())) { + const shards = await this.levels.write(sorted.map((p) => ({ ...p, level: MAX_LEVEL }))) + let placed = false + await this.manifests.update((m) => { + if (m.adopted?.includes(name) || !empty(m)) return null + m.levels = { ...m.levels, [MAX_LEVEL]: shards } + m.adopted = [...(m.adopted ?? []), name].slice(-ADOPTED_KEPT) + placed = true + return m + }) + if (placed) return + // Something landed first: level 0 after all (or a retry already listed them). + for (const s of shards) await this.store.delete(shardKey(s.id)) + } + await this.manifests.update((m) => { + if (m.adopted?.includes(name)) return null + const have = new Set(m.packs.map((p) => p.id)) + m.packs.push(...packs.filter((p) => !have.has(p.id))) + m.adopted = [...(m.adopted ?? []), name].slice(-ADOPTED_KEPT) + return m + }) + } + /** * Delete packs an upload wrote for commit `name`, but none the collection lists: * a pack `adoptPacks` named (in the manifest, its retired files, or the registry @@ -357,6 +477,7 @@ export class KeyedLayout implements RecordStore { const registered = await this.store.get(this.registry.keyFor(`upload-${name}`)) if (registered) for (const p of JSON.parse(await registered.text()) as PackInfo[]) listed.add(p.id) + for (const ps of (await this.#levelPacks(m)).values()) for (const p of ps) listed.add(p.id) const gone = ids.filter((id) => !listed.has(id) && !listed.has(packKey(id))) for (let i = 0; i < gone.length; i += 16) await Promise.all( @@ -395,9 +516,10 @@ export class KeyedLayout implements RecordStore { // landing throughout, layers alone could keep a job busy and packs pile up. A // job stops well inside its lease and a Worker's subrequests; the next carries on. const until = Date.now() + layoutConfig.maintainBudgetMs + const leveled = await this.leveled() for (;;) { const layers = await this.#mergeLayers(opts.split) - const packs = await this.#mergePacks() + const packs = leveled ? await this.#mergeLevels() : await this.#mergePacks() if (!layers && !packs) break if (Date.now() >= until || spent() >= layoutConfig.maintainCalls) { more = true @@ -598,12 +720,207 @@ export class KeyedLayout implements RecordStore { return this.#swap({ oldPacks: inputs.map((p) => p.id), newLayers: files, newPacks: packs }) } + /** + * Leveled: one merge down, from the level most over its target (levels.ts). Level 0's + * oldest packs merge into the base level; a pack of a level below it merges into + * the next, picking the one whose overlap there is smallest for its size (RocksDB's + * kMinOverlappingRatio: each merge rewrites as little as it can). Both sides go + * through as streams into the next level's packs, cut at 16 MiB and at types, and a + * new layer points the index at where each record now is. + * + * A record id found with several hashes keeps the newest (level 0's newest pack, + * then the upper level) in the levels; the older hashes go to this month's history + * packs. That's a guess at which is current: a revert brings back an old hash, then + * read from history. Only locality depends on it, since the index says where every + * copy is. + */ + async #mergeLevels(): Promise { + const m = await this.manifests.latest() + const { base, targets } = levelTargets(m.levels) + const zero = m.packs.filter((p) => !p.history) + const zeroRaw = zero.reduce((n, p) => n + p.raw, 0) + // Level 0 merges once it's a tenth of the level it merges into: while a collection + // is small its base is the bottom, and a few small commits shouldn't rewrite it all. + const zeroTrigger = Math.max(levelConfig.level0Raw, levelRaw(m.levels, base) / LEVEL_RATIO) + let from = -1 + let score = 1 + if (zero.length >= levelConfig.level0Packs || zeroRaw >= zeroTrigger) { + from = 0 + score = Math.max(zero.length / levelConfig.level0Packs, zeroRaw / zeroTrigger) + } + for (let l = 1; l < MAX_LEVEL; l++) { + const raw = levelRaw(m.levels, l) + if (!raw) continue + // A level above the base has no room: what it holds goes down first. + const s = targets[l]! > 0 ? raw / targets[l]! : Infinity + if (s > score) { + from = l + score = s + } + } + if (from < 0) return false + let upper: PackInfo[] + let target: number + if (from === 0) { + // Oldest first, so what stays in level 0 is newer than what went down. + upper = [] + let raw = 0 + for (const p of zero) { + if (upper.length && raw + p.raw > levelConfig.level0MergeRaw) break + upper.push(p) + raw += p.raw + } + target = base + } else { + target = from + 1 + const here = await this.levels.packs(m.levels?.[from] ?? []) + const below = await this.levels.packs(m.levels?.[target] ?? []) + // Each pack's overlap below, by a walk of both in key order. + let best: PackInfo | null = null + let bestRatio = Infinity + let j = 0 + for (const p of here) { + while (j < below.length && compareUtf8(below[j]!.hi!, p.lo!) < 0) j++ + let over = 0 + for (let k = j; k < below.length && compareUtf8(below[k]!.lo!, p.hi!) <= 0; k++) + over += below[k]!.raw + const ratio = (over + 1) / Math.max(1, p.raw) + if (ratio < bestRatio) { + best = p + bestRatio = ratio + } + } + upper = [best!] + } + const lo = upper.reduce((a, p) => (compareUtf8(p.lo!, a) < 0 ? p.lo! : a), upper[0]!.lo!) + const hi = upper.reduce((a, p) => (compareUtf8(p.hi!, a) > 0 ? p.hi! : a), upper[0]!.hi!) + const lower = await this.levels.overlapping(m.levels?.[target] ?? [], lo, hi) + // Newest first: level 0's newest pack, …, its oldest, then the level below. + const ranked = [...(from === 0 ? [...upper].reverse() : upper), ...lower] + const reader = this.packs + const read = mergeRead(ranked.length) + const streams = ranked.map((p, rank) => + (async function* () { + for await (const { key, line } of reader.all(p.id, read)) yield { key, line, rank } + })(), + ) + const month = new Date().toISOString().slice(0, 7) + const pw = new PackWriter(this.store, { level: target, packRaw: levelConfig.packRaw }) + const hw = new PackWriter(this.store, { history: month, packRaw: levelConfig.packRaw }) + const lw = new RunWriter(this.store, 0) + let group: { key: string; line: string; rank: number }[] = [] + let n = 0 + const flush = async () => { + if (!group.length) return + let cur = group[0]! + for (const x of group) if (x.rank < cur.rank) cur = x + let last = '' + for (const x of group) { + // A copy of a key already written (the same record twice) is one record. + if (x.key === last) continue + if (x.key === cur.key) { + if (x !== cur) continue + lw.add(x.key, pw.append(x.key, x.line)) + } else lw.add(x.key, hw.append(x.key, x.line)) + last = x.key + } + group = [] + if (++n % 64 === 0) await Promise.all([pw.ready(), hw.ready(), lw.ready()]) + } + for await (const x of mergeAll(streams)) { + const id = x.key.slice(0, x.key.lastIndexOf('\u0000')) + if (group.length) { + const g = group[0]!.key + if (g.slice(0, g.lastIndexOf('\u0000')) !== id) await flush() + } + group.push(x) + } + await flush() + const packs = await pw.finish() + const history = await hw.finish() + const files = await lw.finish() + return this.#swapLevels({ + zero: from === 0 ? upper.map((p) => p.id) : [], + from: from > 0 ? { level: from, ids: upper.map((p) => p.id) } : null, + target, + lower: lower.map((p) => p.id), + packs, + history, + layers: files, + }) + } + + /** + * A leveled merge's result in one manifest change: its inputs out of level 0 or + * their level, its output in the target level, its history packs in the registry, + * and its layer the newest. Only if the levels it changed are as it read them and + * level 0's inputs are still listed; otherwise everything it wrote goes. + */ + async #swapLevels(o: { + zero: string[] + from: { level: number; ids: string[] } | null + target: number + lower: string[] + packs: PackInfo[] + history: PackInfo[] + layers: LayerInfo[] + }): Promise { + const m0 = await this.manifests.latest() + const changed = new Map>>() + if (o.from) + changed.set( + o.from.level, + await this.levels.replace(m0.levels?.[o.from.level] ?? [], new Set(o.from.ids), []), + ) + changed.set( + o.target, + await this.levels.replace(m0.levels?.[o.target] ?? [], new Set(o.lower), o.packs), + ) + const registered = await this.registry.add(o.history) + const zero = new Set(o.zero) + let lost = false + await this.manifests.update((m) => { + const have = new Set(m.packs.map((p) => p.id)) + if ( + ![...zero].every((id) => have.has(id)) || + [...changed.keys()].some((l) => !sameShards(m.levels?.[l], m0.levels?.[l])) + ) { + lost = true + return null + } + const now = Date.now() + m.packs = m.packs.filter((p) => !zero.has(p.id)) + m.levels = { ...m.levels } + for (const [l, c] of changed) m.levels[l] = c.shards + const seq = topSeq(m) + 1 + for (const l of o.layers) m.layers.push({ ...l, seq }) + for (const id of [...o.zero, ...(o.from?.ids ?? []), ...o.lower]) + m.retired.push({ key: packKey(id), at: now }, { key: packDirKey(id), at: now }) + for (const c of changed.values()) + for (const id of c.replaced) m.retired.push({ key: shardKey(id), at: now }) + m.stats.levelMerges = (m.stats.levelMerges ?? 0) + 1 + return m + }) + if (!lost) return true + const gone = [ + ...o.layers.flatMap((l) => [layerKey(l.id), layerDirKey(l.id), layerFilterKey(l.id)]), + ...[...o.packs, ...o.history].flatMap((p) => [packKey(p.id), packDirKey(p.id)]), + ...[...changed.values()].flatMap((c) => c.written.map(shardKey)), + ...(registered ? [registered] : []), + ] + for (let i = 0; i < gone.length; i += 16) + await Promise.all(gone.slice(i, i + 16).map((k) => this.store.delete(k))) + return false + } + /** * Replace layers and packs in one manifest change: only if every input the * manifest lists is still there (another job may have merged them first); * otherwise drop what was written. New packs no merge will take are listed in * the registry, the rest in the manifest; `unlisted` are old packs already in - * neither (reclaim took them out of the registry), retired with the rest. + * neither (reclaim took them out of the registry), retired with the rest. In a + * leveled collection old packs a level lists come out of its shards (only if the + * level is as read), and new packs go to level 0 but for history packs. */ async #swap(o: { oldLayers?: string[] @@ -620,15 +937,29 @@ export class KeyedLayout implements RecordStore { const oldPacks = new Set(o.oldPacks ?? []) const newLayers = o.newLayers ?? [] const newPacks = o.newPacks ?? [] - const listed = newPacks.filter((p) => !KeyedLayout.final(p)) - const registered = await this.registry.add(newPacks.filter(KeyedLayout.final)) + const leveled = await this.leveled() + const listed = newPacks.filter((p) => !this.#final(p, leveled)) + const registered = await this.registry.add(newPacks.filter((p) => this.#final(p, leveled))) + const m0 = leveled ? await this.manifests.latest() : null + const changed = new Map>>() + const inLevels = new Set() + if (m0) { + const gone = new Set([...oldPacks, ...(o.unlisted ?? [])]) + for (const [l, ps] of await this.#levelPacks(m0)) { + const ids = new Set(ps.filter((p) => gone.has(p.id)).map((p) => p.id)) + if (!ids.size) continue + for (const id of ids) inLevels.add(id) + changed.set(l, await this.levels.replace(m0.levels![l]!, ids, [])) + } + } let lost = false await this.manifests.update((m) => { const haveL = new Set(m.layers.map((l) => l.id)) const haveP = new Set(m.packs.map((p) => p.id)) if ( ![...oldLayers].every((id) => haveL.has(id)) || - ![...oldPacks].every((id) => haveP.has(id)) + ![...oldPacks].every((id) => haveP.has(id) || inLevels.has(id)) || + [...changed.keys()].some((l) => !sameShards(m.levels?.[l], m0!.levels?.[l])) ) { lost = true return null @@ -636,6 +967,13 @@ export class KeyedLayout implements RecordStore { const now = Date.now() m.layers = m.layers.filter((l) => !oldLayers.has(l.id)) m.packs = m.packs.filter((p) => !oldPacks.has(p.id)) + if (changed.size) { + m.levels = { ...m.levels } + for (const [l, c] of changed) { + m.levels[l] = c.shards + for (const id of c.replaced) m.retired.push({ key: shardKey(id), at: now }) + } + } // A pack merge's layer is the newest: its places win over the old ones. const seq = topSeq(m) + 1 for (const l of newLayers) m.layers.push(o.keepSeq ? l : { ...l, seq }) @@ -660,6 +998,8 @@ export class KeyedLayout implements RecordStore { ]) for (const p of newPacks) await Promise.all([this.store.delete(packKey(p.id)), this.store.delete(packDirKey(p.id))]) + for (const c of changed.values()) + await Promise.all(c.written.map((id) => this.store.delete(shardKey(id)))) if (registered) await this.store.delete(registered) return false } @@ -869,10 +1209,14 @@ export class KeyedLayout implements RecordStore { throw new TooManyRunsError(`Reclaim: ${n} index runs, more than ${RECLAIM_MAX_RUNS}`) const ranges = await this.#cutRanges(m.layers, rangeEntries) const registry = await this.#registered(m) + const levelPacks = [...(await this.#levelPacks(m)).values()].flat() const plan: ReclaimPlan = { id: newId(), layers: m.layers, - packs: [...m.packs, ...registry.packs].map((p) => ({ id: p.id, records: p.records })), + packs: [...m.packs, ...levelPacks, ...registry.packs].map((p) => ({ + id: p.id, + records: p.records, + })), registry: registry.keys, ranges, level: Math.max(0, ...m.layers.map((l) => l.level)), @@ -1095,10 +1439,11 @@ export class KeyedLayout implements RecordStore { throw new Error('forkInto: F needs both collections in their own prefixes') const m = await this.manifests.latest() const registered = (await this.#registered(m)).packs + const levelPacks = await this.#levelPacks(m) // Each file's home: where the parent read it from (its own parent's), else the parent. const mine = await from.inherited() const inherited: Inherited = { packs: {}, layers: {} } - for (const p of [...m.packs, ...registered]) + for (const p of [...m.packs, ...registered, ...[...levelPacks.values()].flat()]) inherited.packs[p.id] = mine.packs[p.id] ?? from.home for (const l of m.layers) inherited.layers[l.id] = mine.layers[l.id] ?? from.home const homes = new Set([...Object.values(inherited.packs), ...Object.values(inherited.layers)]) @@ -1111,9 +1456,17 @@ export class KeyedLayout implements RecordStore { await to.put(INHERITED_KEY, body) to.forget() await target.registry.add(registered) + // A leveled parent's fork is leveled, its levels listing the parent's packs (in + // shards of its own: they change as the fork's merges rewrite its levels). + const levels: Record = {} + if (await this.leveled()) { + await target.markLeveled() + for (const [l, ps] of levelPacks) levels[l] = await target.levels.write(ps) + } // The parent's layers keep their order but number below anything the fork writes. const top = Math.max(0, ...m.layers.map((l) => l.seq)) await target.manifests.update((n) => { + if (Object.keys(levels).length) n.levels = levels n.packs.push(...m.packs) n.layers.push(...m.layers.map((l) => ({ ...l, seq: l.seq - top - 1 }))) return n @@ -1150,11 +1503,16 @@ export class KeyedLayout implements RecordStore { { key: at(h, layerFilterKey(l.id)), optional: true }, ) } - for (const p of [...m.packs, ...registry.packs]) { + const levelPacks = [...(await this.#levelPacks(m)).values()].flat() + for (const p of [...m.packs, ...levelPacks, ...registry.packs]) { const h = inh.packs[p.id] files.push({ key: at(h, packKey(p.id)), bytes: p.bytes }, { key: at(h, packDirKey(p.id)) }) } for (const k of registry.keys) files.push({ key: at(undefined, k) }) + // A leveled collection's mark and its levels' shards: the copy's own, written once. + for (const shards of Object.values(m.levels ?? {})) + for (const s of shards) files.push({ key: at(undefined, shardKey(s.id)) }) + if (await this.leveled()) files.push({ key: at(undefined, LEVELED_KEY) }) if (Object.keys(inh.packs).length || Object.keys(inh.layers).length) files.push({ key: at(undefined, 'inherited.json') }) const dir = `runs/${home.home}/manifest/${this.name}` @@ -1402,9 +1760,12 @@ class KeyedWriter implements RecordWriter { this.#entries = [] const files = await lw.finish() // Listed before the manifest names their index: a pack the manifest never got - // (a failed update) is one reclaim finds with nothing live, and drops. - await this.layout.registry.add(packs.filter(KeyedLayout.final)) - const run = { layers: files, packs: packs.filter((p) => !KeyedLayout.final(p)) } + // (a failed update) is one reclaim finds with nothing live, and drops. A leveled + // collection's go to level 0, every one: merges take them down. + const leveled = await this.layout.leveled() + const final = (p: PackInfo) => !leveled && KeyedLayout.final(p) + await this.layout.registry.add(packs.filter(final)) + const run = { layers: files, packs: packs.filter((p) => !final(p)) } // Held: the caller names them later, with the rest of its commit (appendRuns). if (this.opts.hold) this.#held.push(run) else await this.layout.manifests.append(run.layers, run.packs) diff --git a/packages/protocol/src/repo/layouts/kit.ts b/packages/protocol/src/repo/layouts/kit.ts index defdbccc..bea9a556 100644 --- a/packages/protocol/src/repo/layouts/kit.ts +++ b/packages/protocol/src/repo/layouts/kit.ts @@ -17,6 +17,7 @@ import { compareUtf8 } from '../../utf8.js' import { gunzipText, gzip } from '../gzip.js' import type { Lru } from '../lru.js' import { putJoined, type Store } from '../types.js' +import type { Levels } from './levels.js' export const BLOCK_RAW = 64 * 1024 export const PACK_RAW = 16 * 1024 * 1024 @@ -79,6 +80,13 @@ export interface PackInfo { * spans types and merges take one type's packs. Absent on packs written before. */ type?: string + /** The pack's first and last keys: a level's packs never overlap (KeyedLayout's levels). */ + lo?: string + hi?: string + /** A leveled collection's level the pack is in: 0 (or absent) for a commit's or upload's. */ + level?: number + /** A pack of replaced versions a leveled merge moved out, by month written (`2026-10`). */ + history?: string } /** The record type a key leads with (recordKey); '' for a key without one. */ @@ -105,13 +113,22 @@ export class PackWriter { #records = 0 #lastKey: string | null = null #type = '' + #lo = '' + #hi = '' readonly done: PackInfo[] = [] readonly #writing = new Set>() #error: unknown = null constructor( readonly store: Store, - readonly opts: { tier?: number; hooks?: PackHooks } = {}, + readonly opts: { + tier?: number + hooks?: PackHooks + level?: number + history?: string + /** Raw bytes a pack closes at (PACK_RAW). */ + packRaw?: number + } = {}, ) {} /** @@ -124,13 +141,19 @@ export class PackWriter { this.#close() this.#type = type this.#lastKey = key - if (!this.#id) this.#id = newId() + if (!this.#id) { + this.#id = newId() + this.#lo = key + } + this.#hi = key this.#lines.push(`${key}\t${line}`) const at = loc(this.#id, this.#blocks.length, this.#lines.length - 1) this.#raw += key.length + line.length + 2 this.#records++ if (this.#raw >= BLOCK_RAW) this.#seal() - if (this.#packRaw >= PACK_RAW) this.#close() + // A size given counts the block filling too (a small one closes within a block). + const limit = this.opts.packRaw + if (limit ? this.#packRaw + this.#raw >= limit : this.#packRaw >= PACK_RAW) this.#close() return at } @@ -167,6 +190,8 @@ export class PackWriter { records: this.#records, raw: this.#packRaw, type: this.#type, + lo: this.#lo, + hi: this.#hi, } this.#id = '' this.#blocks = [] @@ -186,6 +211,8 @@ export class PackWriter { records: number raw: number type: string + lo: string + hi: string }) { const parts = await Promise.all(job.blocks) const offsets = [0] @@ -200,6 +227,10 @@ export class PackWriter { bytes: offsets.at(-1)!, raw: job.raw, ...(job.type ? { type: job.type } : {}), + lo: job.lo, + hi: job.hi, + ...(this.opts.level ? { level: this.opts.level } : {}), + ...(this.opts.history ? { history: this.opts.history } : {}), } this.done.push(pack) await this.opts.hooks?.after?.(pack) @@ -844,6 +875,11 @@ export interface Manifest { * retry lists none again, not even one a merge has since taken and swept. */ adopted?: string[] + /** + * A leveled collection's levels 1… (levels.ts), by level: the shards listing each + * one's packs. `packs` above is level 0. + */ + levels?: Levels } const empty = (): Manifest => ({ seq: 0, layers: [], packs: [], retired: [], stats: {} }) diff --git a/packages/protocol/src/repo/layouts/levels.ts b/packages/protocol/src/repo/layouts/levels.ts new file mode 100644 index 00000000..7474a06c --- /dev/null +++ b/packages/protocol/src/repo/layouts/levels.ts @@ -0,0 +1,206 @@ +/** + * A leveled collection's levels (leveled-packs-design.md): below level 0 (the packs + * commits and uploads write, listed in the manifest as ever), levels 1…MAX_LEVEL + * each hold packs that never overlap, in key order, within one type. The bottom + * level holds most of the collection; each level above is 1/T the size of the one + * below, so a record is rewritten about T times a level on its way down, and a leaf + * of the current version reads about one pack a level, however many days of + * changes went in (Crossref's dailies over three years, simulated: 28 GETs a leaf + * at the median where tiered packs took 255). + * + * A level's packs are listed in shards: write-once objects of up to `shardPacks` + * packs each, in key order, which the manifest names with their key ranges. The + * bottom level of a Crossref-sized collection is tens of thousands of packs, too + * many for the manifest every reader reads; a merge rewrites only the shards its + * key range touches. Readers never read them: records are found through the index. + */ +import { compareUtf8 } from '../../utf8.js' +import type { Store } from '../types.js' +import { newId, PACK_RAW, type PackInfo } from './kit.js' + +/** Levels below level 0. The bottom one holds most of a collection (RocksDB's seven, 0–6). */ +export const MAX_LEVEL = 6 +/** Each level's size against the one below it. */ +export const LEVEL_RATIO = 10 + +/** Sizes (tests set them low). */ +export const levelConfig = { + /** + * The smallest a level's target gets: level 0 merges into the highest level whose + * target is at least this, skipping smaller ones (RocksDB's dynamic level sizes). + * About half a day of Crossref's changes: three levels above a 1 TB bottom. + */ + minRaw: 256 * 1024 * 1024, + /** + * Level 0 merges down once it holds this many packs, or `level0Raw` raw bytes (or a + * tenth of the level it merges into, if more). A leaf's recent changes are in level + * 0's packs, one read each, so their number is a read cost. + */ + level0Packs: 16, + level0Raw: 64 * 1024 * 1024, + /** Raw bytes of level-0 packs one merge takes, oldest first. */ + level0MergeRaw: 256 * 1024 * 1024, + /** Raw bytes a merge's packs close at. */ + packRaw: PACK_RAW, + /** Packs a shard lists at most. */ + shardPacks: 512, +} + +/** A shard of a level's listing, as the manifest names it. */ +export interface LevelShard { + id: string + lo: string + hi: string + packs: number + /** Raw and stored bytes of its packs. */ + raw: number + bytes: number +} + +export type Levels = Record + +export const shardKey = (id: string) => `manifest/f/levels/${id}.json` + +/** Whether [aLo, aHi] and [bLo, bHi] share a key. */ +export const overlaps = (aLo: string, aHi: string, bLo: string, bHi: string) => + compareUtf8(aLo, bHi) <= 0 && compareUtf8(bLo, aHi) <= 0 + +const byLo = (a: PackInfo, b: PackInfo) => compareUtf8(a.lo!, b.lo!) + +/** Raw bytes a level holds. */ +export const levelRaw = (levels: Levels | undefined, level: number) => + (levels?.[level] ?? []).reduce((n, s) => n + s.raw, 0) + +/** + * Each level's target size from the bottom up, and the base level, where level 0 + * merges: the highest whose target is at least `levelConfig.minRaw`, and never the bottom, + * so level 0 never rewrites the whole collection (the level above it takes level + * 0's merges and goes down a pack at a time). Levels above the base have no room + * (target 0). + */ +export function levelTargets(levels: Levels | undefined): { base: number; targets: number[] } { + const targets = Array.from({ length: MAX_LEVEL + 1 }).fill(0) + let t = levelRaw(levels, MAX_LEVEL) + targets[MAX_LEVEL] = Infinity + let base = MAX_LEVEL - 1 + targets[base] = t / LEVEL_RATIO + t /= LEVEL_RATIO + for (let l = MAX_LEVEL - 2; l >= 1; l--) { + t /= LEVEL_RATIO + if (t < levelConfig.minRaw) break + targets[l] = t + base = l + } + return { base, targets } +} + +/** The shards a leveled collection's levels are listed in (each read once: they never change). */ +export class LevelShards { + readonly #read = new Map>() + + constructor(readonly store: Store) {} + + read(id: string): Promise { + let p = this.#read.get(id) + if (!p) { + p = this.store.get(shardKey(id)).then(async (o) => { + if (!o) throw new Error(`Missing level shard ${id}`) + return JSON.parse(await o.text()) as PackInfo[] + }) + p.catch(() => this.#read.delete(id)) + this.#read.set(id, p) + } + return p + } + + /** Every pack of `shards`, in key order. */ + async packs(shards: readonly LevelShard[]): Promise { + const out: PackInfo[] = [] + for (let i = 0; i < shards.length; i += 16) + for (const ps of await Promise.all(shards.slice(i, i + 16).map((s) => this.read(s.id)))) + out.push(...ps) + return out + } + + /** The packs of `shards` that hold a key in [lo, hi], in key order. */ + async overlapping(shards: readonly LevelShard[], lo: string, hi: string): Promise { + const touched = shards.filter((s) => overlaps(s.lo, s.hi, lo, hi)) + return (await this.packs(touched)).filter((p) => overlaps(p.lo!, p.hi!, lo, hi)) + } + + /** `packs` (in key order, none overlapping) as new shards, written. */ + async write(packs: readonly PackInfo[]): Promise { + const out: LevelShard[] = [] + for (let i = 0; i < packs.length; i += levelConfig.shardPacks) { + const ps = packs.slice(i, i + levelConfig.shardPacks) + const id = newId() + this.#read.set(id, Promise.resolve(ps)) + await this.store.put(shardKey(id), JSON.stringify(ps), { ifAbsent: true }) + out.push({ + id, + lo: ps[0]!.lo!, + hi: ps.at(-1)!.hi!, + packs: ps.length, + raw: ps.reduce((n, p) => n + p.raw, 0), + bytes: ps.reduce((n, p) => n + p.bytes, 0), + }) + } + return out + } + + /** + * A level's shards with the packs `remove` gone and `add` in. Only the shards + * holding a removed pack, or whose range the added ones fall in or between, are + * written again: the result, the shards written, and the ones they replace. + * Throws if the level would overlap. + */ + async replace( + shards: readonly LevelShard[], + remove: ReadonlySet, + add: readonly PackInfo[], + ): Promise<{ shards: LevelShard[]; written: string[]; replaced: string[] }> { + const added = [...add].sort(byLo) + // Shards touched: those holding a removed pack (found by reading the ones whose + // range could), and those the added packs' range reaches. + const touched = new Set() + for (let i = 0; i < shards.length; i++) { + const s = shards[i]! + if (added.length && overlaps(s.lo, s.hi, added[0]!.lo!, added.at(-1)!.hi!)) touched.add(i) + } + if (remove.size) + for (let i = 0; i < shards.length; i++) { + if (touched.has(i)) continue + const ps = await this.read(shards[i]!.id) + if (ps.some((p) => remove.has(p.id))) touched.add(i) + } + // Added packs that fall between shards: the shards on either side join, so the + // rewritten stretch stays contiguous. + if (added.length && !touched.size) { + const at = shards.findIndex((s) => compareUtf8(s.lo, added[0]!.lo!) > 0) + if (at > 0) touched.add(at - 1) + else if (at === 0) touched.add(0) + else if (shards.length) touched.add(shards.length - 1) + } + if (!touched.size && !added.length) return { shards: [...shards], written: [], replaced: [] } + const idx = [...touched].sort((a, b) => a - b) + const from = idx[0] ?? shards.length + const to = idx.length ? idx.at(-1)! : shards.length - 1 + const kept = (await this.packs(shards.slice(from, to + 1))).filter((p) => !remove.has(p.id)) + const packs = [...kept, ...added].sort(byLo) + for (let i = 1; i < packs.length; i++) + if (compareUtf8(packs[i]!.lo!, packs[i - 1]!.hi!) <= 0) + throw new Error(`Level overlap: pack ${packs[i]!.id} starts inside ${packs[i - 1]!.id}`) + const fresh = await this.write(packs) + const before = shards.slice(0, from) + const after = shards.slice(to + 1) + if (before.length && fresh.length && compareUtf8(fresh[0]!.lo, before.at(-1)!.hi) <= 0) + throw new Error('Level overlap at a shard boundary') + if (after.length && fresh.length && compareUtf8(fresh.at(-1)!.hi, after[0]!.lo) >= 0) + throw new Error('Level overlap at a shard boundary') + return { + shards: [...before, ...fresh, ...after], + written: fresh.map((s) => s.id), + replaced: shards.slice(from, to + 1).map((s) => s.id), + } + } +} diff --git a/packages/protocol/test/repo/leveled.test.ts b/packages/protocol/test/repo/leveled.test.ts new file mode 100644 index 00000000..b598500c --- /dev/null +++ b/packages/protocol/test/repo/leveled.test.ts @@ -0,0 +1,242 @@ +/** + * Layout F's leveled packs (layouts/levels.ts): merges take level 0 down to the base + * and each level into the next, every level's packs apart in key order, replaced + * versions in history packs, and every record readable through the index throughout. + */ +import { afterEach, beforeEach, describe, expect, it } from 'vitest' + +import { + headConfig, + HomeStore, + KeyedLayout, + layoutConfig, + memoryStore, + type PackInfo, + type Store, +} from '../../src/index.js' +import { levelConfig, MAX_LEVEL } from '../../src/repo/layouts/levels.js' +import { repoLru } from '../../src/repo/repo.js' +import { compareUtf8 } from '../../src/utf8.js' + +const saved = { level: { ...levelConfig }, layout: { ...layoutConfig } } +beforeEach(() => { + headConfig.intervalMs = 1 + layoutConfig.graceMs = 0 + // Small sizes, so a few hundred records make several levels of several packs. + Object.assign(levelConfig, { + minRaw: 1_000, + level0Packs: 4, + level0Raw: 4_000, + level0MergeRaw: 8_000, + packRaw: 3_000, + shardPacks: 3, + }) +}) +afterEach(() => { + headConfig.intervalMs = 1000 + Object.assign(levelConfig, saved.level) + Object.assign(layoutConfig, saved.layout) +}) + +const layout = (store: Store = memoryStore(), home = 'c1') => + new KeyedLayout(new HomeStore(store, home), repoLru(8 << 20), home) + +const leveledLayout = async (store?: Store, home?: string) => { + const l = layout(store, home) + await l.markLeveled() + return l +} + +const ids = Array.from( + { length: 400 }, + (_, i) => `10.${1000 + (i % 7)}/w${String(i).padStart(4, '0')}`, +).sort() +const line = (id: string, v: number) => `${id} v${v} ${'x'.repeat(80)}` + +/** Every level's packs, checked apart and in key order; and the level each says it's in. */ +async function levelPacks(l: KeyedLayout) { + const m = await l.manifests.latest() + const out = new Map() + for (const [lv, shards] of Object.entries(m.levels ?? {})) { + const ps = await l.levels.packs(shards) + for (let i = 1; i < ps.length; i++) + expect(compareUtf8(ps[i]!.lo!, ps[i - 1]!.hi!), `level ${lv} overlaps`).toBeGreaterThan(0) + for (const p of ps) expect(p.level).toBe(Number(lv)) + for (const s of shards) { + const mine = await l.levels.read(s.id) + expect(s.lo).toBe(mine[0]!.lo) + expect(s.hi).toBe(mine.at(-1)!.hi) + } + if (ps.length) out.set(Number(lv), ps) + } + return { m, levels: out } +} + +/** Commits of a few records each, some ids new, some changed; their versions. */ +async function commits(l: KeyedLayout, n: number, seed = 1) { + let s = seed + const rnd = () => (s = (s * 1103515245 + 12345) % 2 ** 31) / 2 ** 31 + const current = new Map() + const all: [string, number][] = [] + for (let c = 0; c < n; c++) { + const w = l.writer() + const picked = new Set() + while (picked.size < 20) picked.add(ids[Math.floor(rnd() * ids.length)]!) + for (const id of [...picked].sort()) { + const v = (current.get(id) ?? 0) + 1 + current.set(id, v) + all.push([id, v]) + w.add('T', { key: id, hash: `h${v}`, size: 10 }, line(id, v)) + } + await w.finish() + while (await l.maintain()); + await l.maintain() + } + return { current, all } +} + +async function readAll(l: KeyedLayout, versions: Iterable<[string, number]>) { + const vs = [...versions].sort((a, b) => compareUtf8(a[0], b[0]) || a[1] - b[1]) + const got = await l.leafLines({ + hash: 'leaf', + type: 'T', + entries: vs.map(([id, v]) => ({ key: id, hash: `h${v}`, size: 10 })), + }) + expect(got).toEqual(vs.map(([id, v]) => line(id, v))) +} + +describe('leveled packs', () => { + it('merge down level by level, each level apart, every record readable', async () => { + const l = await leveledLayout() + const { current, all } = await commits(l, 40) + const { m, levels } = await levelPacks(l) + // Data went below level 0, into more than one level, and a level holds several packs. + expect(levels.size).toBeGreaterThan(1) + expect([...levels.values()].some((ps) => ps.length > 1)).toBe(true) + expect(m.stats.levelMerges).toBeGreaterThan(0) + expect(m.packs.length).toBeLessThan(levelConfig.level0Packs) + // The current versions, and every older one, through the index. + await readAll(l, current) + await readAll(l, all) + }) + + it('move replaced versions to history packs, out of the levels', async () => { + const l = await leveledLayout() + const { current } = await commits(l, 40) + const { levels } = await levelPacks(l) + const history = (await l.allPacks()).filter((p) => p.history) + expect(history.length).toBeGreaterThan(0) + expect(history.every((p) => p.history === new Date().toISOString().slice(0, 7))).toBe(true) + // The levels hold about one version an id: the newest, or one a merge hasn't met yet. + const inLevels = [...levels.values()].flat().reduce((n, p) => n + p.records, 0) + expect(inLevels).toBeLessThan(current.size * 1.5) + }) + + it('take a load sent in key order as the bottom level, as written', async () => { + const l = await leveledLayout() + const w = l.uploadWriter() + for (const id of ids) w.append({ type: 'T', id, hash: 'h1', line: line(id, 1) }) + // Sent in two parts, so the load is two packs that don't overlap. + const packs = [...(await w.finish())] + const w2 = l.uploadWriter() + for (const id of ids) w2.append({ type: 'U', id, hash: 'h1', line: line(id, 1) }) + packs.push(...(await w2.finish())) + expect(packs).toHaveLength(2) + await l.adoptPacks('load', packs) + const { m, levels } = await levelPacks(l) + expect(m.packs).toEqual([]) + expect( + levels + .get(MAX_LEVEL)! + .map((p) => p.id) + .sort(), + ).toEqual(packs.map((p) => p.id).sort()) + // A retry lists nothing again. + await l.adoptPacks('load', packs) + expect((await levelPacks(l)).levels.get(MAX_LEVEL)!.length).toBe(packs.length) + // An upload out of order into a collection holding data goes to level 0. + const u = l.uploadWriter() + for (const id of [ids[5]!, ids[1]!]) u.append({ type: 'T', id, hash: 'h2', line: line(id, 2) }) + const more = await u.finish() + await l.adoptPacks('more', more) + expect((await l.manifests.latest()).packs.map((p) => p.id).sort()).toEqual( + more.map((p) => p.id).sort(), + ) + }) + + it('keep every record readable while commits land during merges', async () => { + const l = await leveledLayout() + const current = new Map() + for (let c = 0; c < 30; c++) { + const w = l.writer() + const batch = ids.slice((c * 13) % 380, ((c * 13) % 380) + 12) + for (const id of batch) { + const v = (current.get(id) ?? 0) + 1 + current.set(id, v) + w.add('T', { key: id, hash: `h${v}`, size: 10 }, line(id, v)) + } + await Promise.all([w.finish(), l.maintain(), l.maintain()]) + } + while (await l.maintain()); + await levelPacks(l) + await readAll(l, current) + }) + + it('reclaim drops what nothing keeps, from the levels and history alike', async () => { + const l = await leveledLayout() + const { current, all } = await commits(l, 40) + const before = (await l.allPacks()).length + // Keep the current versions only: older ones (mostly in history) are dropped. + const stats = await l.reclaim({ + records: [...current].map(([id, v]) => `T\u0000${id}\u0000h${v}`), + } as never) + expect(stats.recordsDropped).toBe(all.length - current.size) + while (await l.maintain()); + await levelPacks(l) + await readAll(l, current) + expect((await l.allPacks()).length).toBeLessThan(before) + }) + + it('fork as leveled, the fork reading its parent and its merges leaving the parent alone', async () => { + const root = memoryStore() + const parent = await leveledLayout(root, 'p') + const { current } = await commits(parent, 20) + const fork = layout(root, 'f') + await parent.forkInto(fork) + expect(await fork.leveled()).toBe(true) + await readAll(fork, current) + // The fork changes and merges; the parent reads as before. + const more = await commits(fork, 20, 7) + await levelPacks(fork) + await readAll(parent, current) + const merged = new Map(current) + for (const [id, v] of more.current) merged.set(id, Math.max(v, merged.get(id) ?? 0)) + expect((await levelPacks(parent)).levels.size).toBeGreaterThan(0) + }) + + it('level an empty collection at its first write once switched on, but not one holding packs', async () => { + const root = memoryStore() + const older = layout(root, 'old') + await commits(older, 3) + layoutConfig.levelNew = true + try { + // Opened again (a new isolate): it held packs before leveling, so it stays tiered. + expect(await layout(root, 'old').leveled()).toBe(false) + const fresh = layout(root, 'new') + const { current } = await commits(fresh, 30) + expect(await layout(root, 'new').leveled()).toBe(true) + expect((await levelPacks(fresh)).levels.size).toBeGreaterThan(0) + await readAll(fresh, current) + } finally { + layoutConfig.levelNew = false + } + }) + + it('leave a collection that is not leveled tiered, as before', async () => { + const l = layout() + await commits(l, 10) + const m = await l.manifests.latest() + expect(m.levels).toBeUndefined() + expect(await l.leveled()).toBe(false) + }) +}) diff --git a/packages/server/drizzle/0028_push_pack_ranges.sql b/packages/server/drizzle/0028_push_pack_ranges.sql new file mode 100644 index 00000000..ed0512ee --- /dev/null +++ b/packages/server/drizzle/0028_push_pack_ranges.sql @@ -0,0 +1,2 @@ +ALTER TABLE `push_packs` ADD `lo_key` text;--> statement-breakpoint +ALTER TABLE `push_packs` ADD `hi_key` text; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0028_snapshot.json b/packages/server/drizzle/meta/0028_snapshot.json new file mode 100644 index 00000000..d6689377 --- /dev/null +++ b/packages/server/drizzle/meta/0028_snapshot.json @@ -0,0 +1,3703 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "715b9c5b-f4b6-4fa6-8d5a-b7a936a3e6d9", + "prevId": "6fed31a6-5812-4a14-a6b3-b5bc8df442b5", + "tables": { + "abuse_reports": { + "name": "abuse_reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "contact": { + "name": "contact", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reporter_id": { + "name": "reporter_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "resolved_by": { + "name": "resolved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "abuse_reports_status_idx": { + "name": "abuse_reports_status_idx", + "columns": ["status", "created_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 86400000 + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 10 + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_request": { + "name": "last_request", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": ["key"], + "isUnique": false + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": ["reference_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_collections": { + "name": "ark_collections", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "ark_id": { + "name": "ark_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "custom_url": { + "name": "custom_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_collections_ark_id_unique": { + "name": "ark_collections_ark_id_unique", + "columns": ["ark_id"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_collections_collection_id_collections_id_fk": { + "name": "ark_collections_collection_id_collections_id_fk", + "tableFrom": "ark_collections", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_record_types": { + "name": "ark_record_types", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirect_url_field": { + "name": "redirect_url_field", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "ark_record_types_collection_id_collections_id_fk": { + "name": "ark_record_types_collection_id_collections_id_fk", + "tableFrom": "ark_record_types", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "ark_record_types_collection_id_record_type_pk": { + "columns": ["collection_id", "record_type"], + "name": "ark_record_types_collection_id_record_type_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_shoulders": { + "name": "ark_shoulders", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "shoulder": { + "name": "shoulder", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_shoulders_shoulder_unique": { + "name": "ark_shoulders_shoulder_unique", + "columns": ["shoulder"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_shoulders_organization_id_organization_id_fk": { + "name": "ark_shoulders_organization_id_organization_id_fk", + "tableFrom": "ark_shoulders", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cleanup_runs": { + "name": "cleanup_runs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "step": { + "name": "step", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dry_run": { + "name": "dry_run", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "requested_by": { + "name": "requested_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mark_run_id": { + "name": "mark_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lease": { + "name": "lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stats": { + "name": "stats", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "cleanup_runs_created_idx": { + "name": "cleanup_runs_created_idx", + "columns": ["created_at"], + "isUnique": false + }, + "cleanup_runs_step_idx": { + "name": "cleanup_runs_step_idx", + "columns": ["step", "status"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_tombstones": { + "name": "collection_tombstones", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "versions": { + "name": "versions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_by": { + "name": "deleted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_layout": { + "name": "record_layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'today'" + }, + "records_dropped_at": { + "name": "records_dropped_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_webhooks": { + "name": "collection_webhooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bump_filter": { + "name": "bump_filter", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_delivery_at": { + "name": "last_delivery_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "collection_webhooks_collection_idx": { + "name": "collection_webhooks_collection_idx", + "columns": ["collection_id"], + "isUnique": false + } + }, + "foreignKeys": { + "collection_webhooks_collection_id_collections_id_fk": { + "name": "collection_webhooks_collection_id_collections_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collections": { + "name": "collections", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "head_version_id": { + "name": "head_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_salt": { + "name": "private_salt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_files_root": { + "name": "public_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "version_count": { + "name": "version_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "history_bytes": { + "name": "history_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_push_at": { + "name": "last_push_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_started_at": { + "name": "reconcile_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_state": { + "name": "reconcile_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_seq": { + "name": "reconciled_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "reconciled_files_root": { + "name": "reconciled_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_layout": { + "name": "record_layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'today'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "collections_org_slug_uq": { + "name": "collections_org_slug_uq", + "columns": ["organization_id", "slug"], + "isUnique": true + }, + "collections_public_updated_idx": { + "name": "collections_public_updated_idx", + "columns": ["public", "updated_at"], + "isUnique": false + } + }, + "foreignKeys": { + "collections_organization_id_organization_id_fk": { + "name": "collections_organization_id_organization_id_fk", + "tableFrom": "collections", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "commit_units": { + "name": "commit_units", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "plan_id": { + "name": "plan_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ord": { + "name": "ord", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "after": { + "name": "after", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "through": { + "name": "through", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "gap": { + "name": "gap", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "slices_key": { + "name": "slices_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_key": { + "name": "output_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "queued_at": { + "name": "queued_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "commit_units_plan_idx": { + "name": "commit_units_plan_idx", + "columns": ["plan_id", "set", "type", "ord"], + "isUnique": false + }, + "commit_units_status_idx": { + "name": "commit_units_status_idx", + "columns": ["session_id", "status", "queued_at"], + "isUnique": false + } + }, + "foreignKeys": { + "commit_units_session_id_push_sessions_id_fk": { + "name": "commit_units_session_id_push_sessions_id_fk", + "tableFrom": "commit_units", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "denylist": { + "name": "denylist", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "file_uploads": { + "name": "file_uploads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "multipart_upload_id": { + "name": "multipart_upload_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "file_uploads_hash_idx": { + "name": "file_uploads_hash_idx", + "columns": ["hash"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "files": { + "name": "files", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "forks": { + "name": "forks", + "columns": { + "child_collection_id": { + "name": "child_collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "parent_collection_id": { + "name": "parent_collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_seq": { + "name": "parent_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'public'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "forks_child_collection_id_collections_id_fk": { + "name": "forks_child_collection_id_collections_id_fk", + "tableFrom": "forks", + "tableTo": "collections", + "columnsFrom": ["child_collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instance_settings": { + "name": "instance_settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invitation": { + "name": "invitation", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "jobs": { + "name": "jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "run_at": { + "name": "run_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locked_until": { + "name": "locked_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "jobs_ready_idx": { + "name": "jobs_ready_idx", + "columns": ["status", "run_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "member": { + "name": "member", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + }, + "member_user_id_idx": { + "name": "member_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organization": { + "name": "organization", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ark_naan": { + "name": "ark_naan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kf_org_id": { + "name": "kf_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "columns": ["slug"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "placements": { + "name": "placements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location_id": { + "name": "location_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "synced_seq": { + "name": "synced_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until": { + "name": "lease_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "placements_one_primary_uq": { + "name": "placements_one_primary_uq", + "columns": ["collection_id"], + "isUnique": true, + "where": "\"placements\".\"role\" = 'primary' AND \"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_collection_location_uq": { + "name": "placements_collection_location_uq", + "columns": ["collection_id", "location_id"], + "isUnique": true, + "where": "\"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_org_location_uq": { + "name": "placements_org_location_uq", + "columns": ["organization_id", "location_id"], + "isUnique": true, + "where": "\"placements\".\"organization_id\" IS NOT NULL" + }, + "placements_location_idx": { + "name": "placements_location_idx", + "columns": ["location_id"], + "isUnique": false + } + }, + "foreignKeys": { + "placements_collection_id_collections_id_fk": { + "name": "placements_collection_id_collections_id_fk", + "tableFrom": "placements", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_organization_id_organization_id_fk": { + "name": "placements_organization_id_organization_id_fk", + "tableFrom": "placements", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_location_id_storage_locations_id_fk": { + "name": "placements_location_id_storage_locations_id_fk", + "tableFrom": "placements", + "tableTo": "storage_locations", + "columnsFrom": ["location_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_packs": { + "name": "push_packs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pack_id": { + "name": "pack_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_set": { + "name": "record_set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "records": { + "name": "records", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw": { + "name": "raw", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lo_key": { + "name": "lo_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hi_key": { + "name": "hi_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "push_packs_session_id_push_sessions_id_fk": { + "name": "push_packs_session_id_push_sessions_id_fk", + "tableFrom": "push_packs", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_packs_session_id_pack_id_pk": { + "columns": ["session_id", "pack_id"], + "name": "push_packs_session_id_pack_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_runs": { + "name": "push_runs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "first_key": { + "name": "first_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_key": { + "name": "last_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "merging_into": { + "name": "merging_into", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "push_runs_tier_idx": { + "name": "push_runs_tier_idx", + "columns": ["session_id", "tier", "merging_into"], + "isUnique": false + } + }, + "foreignKeys": { + "push_runs_session_id_push_sessions_id_fk": { + "name": "push_runs_session_id_push_sessions_id_fk", + "tableFrom": "push_runs", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_runs_session_id_seq_pk": { + "columns": ["session_id", "seq"], + "name": "push_runs_session_id_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_sessions": { + "name": "push_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_version_id": { + "name": "base_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "strip_unknown_fields": { + "name": "strip_unknown_fields", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "records_received": { + "name": "records_received", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "runs": { + "name": "runs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "result": { + "name": "result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finalize_started_at": { + "name": "finalize_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_started": { + "name": "commit_started", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "in_key_order": { + "name": "in_key_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "overlapping_uploads": { + "name": "overlapping_uploads", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cleaned_at": { + "name": "cleaned_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_phase": { + "name": "commit_phase", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "progress_at": { + "name": "progress_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stalls": { + "name": "stalls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "commit_plan": { + "name": "commit_plan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "assembly_lease": { + "name": "assembly_lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "push_sessions_collection_idx": { + "name": "push_sessions_collection_idx", + "columns": ["collection_id"], + "isUnique": false + }, + "push_sessions_expires_idx": { + "name": "push_sessions_expires_idx", + "columns": ["status", "expires_at"], + "isUnique": false + }, + "push_sessions_user_idx": { + "name": "push_sessions_user_idx", + "columns": ["user_id", "status"], + "isUnique": false + } + }, + "foreignKeys": { + "push_sessions_collection_id_collections_id_fk": { + "name": "push_sessions_collection_id_collections_id_fk", + "tableFrom": "push_sessions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compaction_parts": { + "name": "ref_compaction_parts", + "columns": { + "compaction_id": { + "name": "compaction_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "part": { + "name": "part", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_compaction_parts_compaction_id_part_pk": { + "columns": ["compaction_id", "part"], + "name": "ref_compaction_parts_compaction_id_part_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compactions": { + "name": "ref_compactions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_runs": { + "name": "input_runs", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output_run": { + "name": "output_run", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parts": { + "name": "parts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix_length": { + "name": "prefix_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'running'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_index_units": { + "name": "ref_index_units", + "columns": { + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unit": { + "name": "unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "events": { + "name": "events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_index_units_version_id_unit_pk": { + "columns": ["version_id", "unit"], + "name": "ref_index_units_version_id_unit_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_segments": { + "name": "ref_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_hash": { + "name": "first_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_hash": { + "name": "last_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'live'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ref_segments_range_idx": { + "name": "ref_segments_range_idx", + "columns": ["state", "first_hash", "last_hash"], + "isUnique": false + }, + "ref_segments_run_idx": { + "name": "ref_segments_run_idx", + "columns": ["run_id"], + "isUnique": false + }, + "ref_segments_tier_idx": { + "name": "ref_segments_tier_idx", + "columns": ["tier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_labels": { + "name": "schema_labels", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "schema_labels_label_idx": { + "name": "schema_labels_label_idx", + "columns": ["label"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "schema_labels_schema_hash_label_pk": { + "columns": ["schema_hash", "label"], + "name": "schema_labels_schema_hash_label_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_usage": { + "name": "schema_usage", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type_slug": { + "name": "type_slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_seq": { + "name": "from_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "to_seq": { + "name": "to_seq", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "schema_usage_hash_idx": { + "name": "schema_usage_hash_idx", + "columns": ["schema_hash"], + "isUnique": false + } + }, + "foreignKeys": { + "schema_usage_collection_id_collections_id_fk": { + "name": "schema_usage_collection_id_collections_id_fk", + "tableFrom": "schema_usage", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "schema_usage_collection_id_type_slug_set_from_seq_pk": { + "columns": ["collection_id", "type_slug", "set", "from_seq"], + "name": "schema_usage_collection_id_type_slug_set_from_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schemas": { + "name": "schemas", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": ["token"], + "isUnique": true + }, + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_fence": { + "name": "storage_fence", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "epoch": { + "name": "epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "window_until": { + "name": "window_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "window_run_id": { + "name": "window_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_locations": { + "name": "storage_locations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucket": { + "name": "bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "credentials": { + "name": "credentials", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unverified'" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checked_at": { + "name": "checked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "storage_locations_organization_id_organization_id_fk": { + "name": "storage_locations_organization_id_organization_id_fk", + "tableFrom": "storage_locations", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "usage_rollups": { + "name": "usage_rollups", + "columns": { + "day": { + "name": "day", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metric": { + "name": "metric", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "amount": { + "name": "amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "usage_rollups_account_idx": { + "name": "usage_rollups_account_idx", + "columns": ["account_id", "day"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "usage_rollups_day_account_id_collection_id_metric_pk": { + "columns": ["day", "account_id", "collection_id", "metric"], + "name": "usage_rollups_day_account_id_collection_id_metric_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": ["email"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": ["identifier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "versions": { + "name": "versions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "major": { + "name": "major", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "minor": { + "name": "minor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "patch": { + "name": "patch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pushed_by": { + "name": "pushed_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_record_count": { + "name": "public_record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_file_count": { + "name": "public_file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "public_total_bytes": { + "name": "public_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "type_counts": { + "name": "type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_type_counts": { + "name": "public_type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "has_private": { + "name": "has_private", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "public_refs_root": { + "name": "public_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_refs_root": { + "name": "private_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refs_indexed": { + "name": "refs_indexed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "changes": { + "name": "changes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "push_session_id": { + "name": "push_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "versions_collection_seq_uq": { + "name": "versions_collection_seq_uq", + "columns": ["collection_id", "seq"], + "isUnique": true + }, + "versions_collection_semver_uq": { + "name": "versions_collection_semver_uq", + "columns": ["collection_id", "semver"], + "isUnique": true + }, + "versions_hash_idx": { + "name": "versions_hash_idx", + "columns": ["hash"], + "isUnique": false + }, + "versions_published_idx": { + "name": "versions_published_idx", + "columns": ["published_at"], + "isUnique": false + }, + "versions_push_session_idx": { + "name": "versions_push_session_idx", + "columns": ["push_session_id"], + "isUnique": false + } + }, + "foreignKeys": { + "versions_collection_id_collections_id_fk": { + "name": "versions_collection_id_collections_id_fk", + "tableFrom": "versions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhook_deliveries": { + "name": "webhook_deliveries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "webhook_id": { + "name": "webhook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bump_type": { + "name": "bump_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'version.created'" + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "delivered_at": { + "name": "delivered_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhook_deliveries_webhook_idx": { + "name": "webhook_deliveries_webhook_idx", + "columns": ["webhook_id", "created_at"], + "isUnique": false + }, + "webhook_deliveries_pending_idx": { + "name": "webhook_deliveries_pending_idx", + "columns": ["status", "next_attempt_at"], + "isUnique": false + } + }, + "foreignKeys": { + "webhook_deliveries_webhook_id_collection_webhooks_id_fk": { + "name": "webhook_deliveries_webhook_id_collection_webhooks_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collection_webhooks", + "columnsFrom": ["webhook_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index ca816c24..11a6e3da 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -197,6 +197,13 @@ "when": 1792605564163, "tag": "0027_push_pack_types", "breakpoints": true + }, + { + "idx": 28, + "version": "6", + "when": 1792606671953, + "tag": "0028_push_pack_ranges", + "breakpoints": true } ] } diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index bb4a59b5..b9da904f 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -701,6 +701,12 @@ export const pushPacks = sqliteTable( raw: integer('raw'), /** The record type the pack holds (PackInfo.type): merges take one type's packs. */ recordType: text('record_type'), + /** + * Its first and last keys (PackInfo.lo, hi): a leveled collection takes a sorted + * load's packs, none overlapping, as its bottom level. + */ + loKey: text('lo_key'), + hiKey: text('hi_key'), createdAt: createdAt(), }, (t) => [primaryKey({ columns: [t.sessionId, t.packId] })], diff --git a/packages/server/src/push/packs.ts b/packages/server/src/push/packs.ts index 8fcaafa5..ad62d03e 100644 --- a/packages/server/src/push/packs.ts +++ b/packages/server/src/push/packs.ts @@ -44,7 +44,14 @@ const hooksFor = (ports: Ports, sessionId: string, set: RecordSet): PackHooks => after: async (pack) => { await ports.db .update(schema.pushPacks) - .set({ records: pack.records, bytes: pack.bytes, raw: pack.raw, recordType: pack.type }) + .set({ + records: pack.records, + bytes: pack.bytes, + raw: pack.raw, + recordType: pack.type, + loKey: pack.lo, + hiKey: pack.hi, + }) .where(and(eq(schema.pushPacks.sessionId, sessionId), eq(schema.pushPacks.packId, pack.id))) }, }) @@ -133,6 +140,7 @@ export async function adoptSessionPacks(ports: Ports, repo: Repo, sessionId: str bytes: r.bytes!, raw: r.raw!, ...(r.recordType ? { type: r.recordType } : {}), + ...(r.loKey && r.hiKey ? { lo: r.loKey, hi: r.hiKey } : {}), })) if (packs.length) await repo.records[set].adoptPacks!(sessionId, packs) } diff --git a/packages/server/src/stores.ts b/packages/server/src/stores.ts index daf13401..fd920eac 100644 --- a/packages/server/src/stores.ts +++ b/packages/server/src/stores.ts @@ -9,6 +9,7 @@ import { type LayoutName, type Cache, + layoutConfig, PrefixedStore, type PresigningStore, Repo, @@ -42,6 +43,11 @@ export interface PlatformStorage { * parent's. */ newCollections?: LayoutName + /** + * Layout F: level a collection's packs (leveled-packs-design.md) when its first write + * finds it empty. A collection holding packs already stays tiered. + */ + leveledPacks?: boolean } /** Primary placements change rarely, a collection's layout only by a conversion; remember them briefly per isolate. */ @@ -60,6 +66,7 @@ export function createStores(db: Db, cache: Cache, platform: PlatformStorage): S /** F's, by location and collection; a Map's order is its recency (a hit moves to the end). */ const runRepos = new Map() const files = platform.files ?? presigning(platform.bucket) + layoutConfig.levelNew = platform.leveledPacks === true const forLocation = async (locationId: string): Promise => { const known = repos.get(locationId) diff --git a/packages/server/src/worker.ts b/packages/server/src/worker.ts index bcfff27a..f83f1295 100644 --- a/packages/server/src/worker.ts +++ b/packages/server/src/worker.ts @@ -66,6 +66,8 @@ export interface Env { AUTH_INTERNAL_API_KEY?: string REPO_PREFIX?: string INTERNAL_PREFIX?: string + /** '1': layout F levels new collections' packs (stores.ts `leveledPacks`). */ + LEVELED_PACKS?: string /** * Public assets (org logos): a world-readable bucket served at ASSETS_BASE_URL. * Reached through the PUBLIC_ASSETS binding when there is one, otherwise @@ -136,6 +138,7 @@ function makePorts(env: Env, ctx: ExecutionContext, req?: Request): Ports { files: s3, repoPrefix: env.REPO_PREFIX ?? 'repo', internalPrefix: env.INTERNAL_PREFIX ?? 'internal', + leveledPacks: env.LEVELED_PACKS === '1', // The bulk Worker's isolates run jobs, whose reads rarely repeat. ...(env.ROLE === 'bulk' ? { memoryBytes: 8 * 1024 * 1024 } : {}), }), diff --git a/packages/server/test/cleanup.test.ts b/packages/server/test/cleanup.test.ts index 594b2a3c..d70ea84c 100644 --- a/packages/server/test/cleanup.test.ts +++ b/packages/server/test/cleanup.test.ts @@ -1,6 +1,6 @@ import { createHash } from 'node:crypto' -import { fsck, newSalt, openRepo } from '@underlay/protocol' +import { fsck, layoutConfig, newSalt, openRepo } from '@underlay/protocol' import { eq, sql } from 'drizzle-orm' import { afterAll, beforeEach, describe, expect, it } from 'vitest' @@ -308,7 +308,8 @@ describe('storage cleanup, steps 2 and 3: mark and sweep', () => { const fresh = await h.collection('new') const puts = h.bucket.puts await push(h, user, 'new', records, 'the pdf') - expect(h.bucket.puts - puts).toBeLessThan(15) // mostly reused + // Mostly reused (leveled, the new collection's mark is one more). + expect(h.bucket.puts - puts).toBeLessThan(layoutConfig.levelNew ? 17 : 15) ageAll(h) const sweep = await run(h, 'sweep') diff --git a/packages/server/test/harness.ts b/packages/server/test/harness.ts index e9dad084..ef16ba49 100644 --- a/packages/server/test/harness.ts +++ b/packages/server/test/harness.ts @@ -89,6 +89,7 @@ export async function harness(opts: { newCollections?: LayoutName } = {}): Promi repoPrefix: 'repo', internalPrefix: 'internal', newCollections: opts.newCollections ?? testLayout, + leveledPacks: process.env.UNDERLAY_TEST_LEVELED === '1', }), cache, signer: async () => signer, diff --git a/packages/server/test/manage.test.ts b/packages/server/test/manage.test.ts index 06f36ca3..63ef4f30 100644 --- a/packages/server/test/manage.test.ts +++ b/packages/server/test/manage.test.ts @@ -1,4 +1,4 @@ -import { readHead, verifyLog } from '@underlay/protocol' +import { layoutConfig, readHead, verifyLog } from '@underlay/protocol' import { eq } from 'drizzle-orm' import { afterAll, describe, expect, it } from 'vitest' @@ -136,9 +136,10 @@ describe('collection management', () => { version: { recordCount: 1 }, }) // Root, log entry, head.json; layout F adds the fork's list of its parent's files, a - // marker in the parent, and the fork's manifest and its head (the public set's alone). + // marker in the parent, and the fork's manifest and its head (the public set's alone); + // leveled, the fork's leveled mark and its level's shard. expect(h.bucket.puts - putsBefore).toBeLessThanOrEqual( - h.ports.stores.newCollections === 'f' ? 8 : 4, + h.ports.stores.newCollections === 'f' ? (layoutConfig.levelNew ? 10 : 8) : 4, ) const [child] = await h.ports.db .select() diff --git a/packages/server/test/upload-packs.test.ts b/packages/server/test/upload-packs.test.ts index 58b0fc99..fcab8719 100644 --- a/packages/server/test/upload-packs.test.ts +++ b/packages/server/test/upload-packs.test.ts @@ -6,7 +6,14 @@ */ import { createHash } from 'node:crypto' -import { countTree, iterate, type RecordSet, RepoSource, recordTree } from '@underlay/protocol' +import { + countTree, + iterate, + layoutConfig, + type RecordSet, + RepoSource, + recordTree, +} from '@underlay/protocol' import { eq } from 'drizzle-orm' import { afterAll, afterEach, describe, expect, it } from 'vitest' @@ -274,7 +281,19 @@ describe.runIf(process.env.UNDERLAY_TEST_LAYOUT === 'f')('uploads into layout F' ) expect(await commit(h, user, 'l', sid)).toBe(201) const registry = `repo/runs/${c.id}/public/manifest/f/packs/upload-${sid}.json` - expect(h.bucket.objects.has(registry)).toBe(true) + // Leveled, the first load (into an empty collection, none of its packs overlapping) + // is the bottom level instead, as written. + expect(h.bucket.objects.has(registry)).toBe(!layoutConfig.levelNew) + const repo = await h.ports.stores.forCollection(c.id) + const layout = repo.records!.public as unknown as { + manifests: { + latest(): Promise<{ + packs: { id: string; tier: number; type?: string }[] + levels?: Record + }> + } + } + if (layoutConfig.levelNew) expect((await layout.manifests.latest()).levels?.[6]).toHaveLength(1) // Small pushes: their packs go in the manifest, and merges re-point their records. for (let j = 0; j < 10; j++) { const s = await open(h, user, 'l') @@ -284,15 +303,11 @@ describe.runIf(process.env.UNDERLAY_TEST_LAYOUT === 'f')('uploads into layout F' const { out } = await headRecords(h, c.id) expect(out).toHaveLength(610) expect(out.filter((r) => /changed|small/.test(r))).toHaveLength(20) - const repo = await h.ports.stores.forCollection(c.id) - const layout = repo.records!.public as unknown as { - manifests: { - latest(): Promise<{ packs: { id: string; tier: number; type?: string }[] }> - } - } const m = await layout.manifests.latest() - // Every small pack listed, or merged into one above them, and each says its type. - expect(m.packs.some((p) => p.tier > 0) || m.packs.length >= 10).toBe(true) + // Every small pack listed, or merged into one above them (leveled: all ten in level + // 0, under its 16-pack trigger), and each says its type. + if (layoutConfig.levelNew) expect(m.packs).toHaveLength(10) + else expect(m.packs.some((p) => p.tier > 0) || m.packs.length >= 10).toBe(true) expect(m.packs.every((p) => p.type === 'Doc')).toBe(true) expect(new Set(m.packs.map((p) => p.id)).size).toBe(m.packs.length) }) From 2c91484b593afc87a7c918b8d155df84051288ad Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 21 Oct 2026 21:17:03 +0200 Subject: [PATCH 5/6] push_packs keys as JSON: SQLite text ends at a key's first NUL --- packages/server/src/db/schema.ts | 5 +++-- packages/server/src/push/packs.ts | 11 ++++++++--- packages/server/test/upload-packs.test.ts | 12 +++++++++++- 3 files changed, 22 insertions(+), 6 deletions(-) diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index b9da904f..6ec7161a 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -702,8 +702,9 @@ export const pushPacks = sqliteTable( /** The record type the pack holds (PackInfo.type): merges take one type's packs. */ recordType: text('record_type'), /** - * Its first and last keys (PackInfo.lo, hi): a leveled collection takes a sorted - * load's packs, none overlapping, as its bottom level. + * Its first and last keys (PackInfo.lo, hi), as JSON strings (a key's NULs would + * end SQLite text): a leveled collection takes a sorted load's packs, none + * overlapping, as its bottom level. */ loKey: text('lo_key'), hiKey: text('hi_key'), diff --git a/packages/server/src/push/packs.ts b/packages/server/src/push/packs.ts index ad62d03e..c3473d55 100644 --- a/packages/server/src/push/packs.ts +++ b/packages/server/src/push/packs.ts @@ -36,6 +36,9 @@ const SETS = ['public', 'private'] as const /** An upload marks an entry private exactly when it goes to the private set (deltaChanges). */ const setOf = (e: RunEntry): RecordSet => (e.p ? 'private' : 'public') +/** A record key as SQLite text: JSON, since keys hold NULs and text stops at one. */ +const keyText = (key: string | undefined) => (key === undefined ? undefined : JSON.stringify(key)) + /** Each pack recorded before it's written, and its stats once it is. */ const hooksFor = (ports: Ports, sessionId: string, set: RecordSet): PackHooks => ({ before: async (packId) => { @@ -49,8 +52,8 @@ const hooksFor = (ports: Ports, sessionId: string, set: RecordSet): PackHooks => bytes: pack.bytes, raw: pack.raw, recordType: pack.type, - loKey: pack.lo, - hiKey: pack.hi, + loKey: keyText(pack.lo), + hiKey: keyText(pack.hi), }) .where(and(eq(schema.pushPacks.sessionId, sessionId), eq(schema.pushPacks.packId, pack.id))) }, @@ -140,7 +143,9 @@ export async function adoptSessionPacks(ports: Ports, repo: Repo, sessionId: str bytes: r.bytes!, raw: r.raw!, ...(r.recordType ? { type: r.recordType } : {}), - ...(r.loKey && r.hiKey ? { lo: r.loKey, hi: r.hiKey } : {}), + ...(r.loKey && r.hiKey + ? { lo: JSON.parse(r.loKey) as string, hi: JSON.parse(r.hiKey) as string } + : {}), })) if (packs.length) await repo.records[set].adoptPacks!(sessionId, packs) } diff --git a/packages/server/test/upload-packs.test.ts b/packages/server/test/upload-packs.test.ts index fcab8719..3fe0fdc6 100644 --- a/packages/server/test/upload-packs.test.ts +++ b/packages/server/test/upload-packs.test.ts @@ -292,8 +292,18 @@ describe.runIf(process.env.UNDERLAY_TEST_LAYOUT === 'f')('uploads into layout F' levels?: Record }> } + levels: { packs(s: { id: string }[]): Promise<{ lo?: string; hi?: string }[]> } + } + if (layoutConfig.levelNew) { + const bottom = (await layout.manifests.latest()).levels?.[6] + expect(bottom).toHaveLength(1) + // Its key range whole, NULs and all, through the database. + const [p] = await layout.levels.packs(bottom!) + expect([p!.lo!.split('\u0000').slice(0, 2), p!.hi!.split('\u0000').slice(0, 2)]).toEqual([ + ['Doc', id(0)], + ['Doc', id(599)], + ]) } - if (layoutConfig.levelNew) expect((await layout.manifests.latest()).levels?.[6]).toHaveLength(1) // Small pushes: their packs go in the manifest, and merges re-point their records. for (let j = 0; j < 10; j++) { const s = await open(h, user, 'l') From a3c6766fcae288c872c3052fbd83ff32fe6912a2 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Wed, 21 Oct 2026 20:03:46 +0200 Subject: [PATCH 6/6] Layout F: packs never span types, and merges take one type's packs (#68, onto main) --- packages/protocol/src/repo/layouts/keyed.ts | 34 +- packages/protocol/src/repo/layouts/kit.ts | 41 +- packages/protocol/test/repo/layouts.test.ts | 52 + .../server/drizzle/0027_push_pack_types.sql | 1 + .../server/drizzle/meta/0027_snapshot.json | 3689 +++++++++++++++++ packages/server/drizzle/meta/_journal.json | 7 + packages/server/src/db/schema.ts | 2 + packages/server/src/push/packs.ts | 11 +- packages/server/test/upload-packs.test.ts | 9 +- 9 files changed, 3824 insertions(+), 22 deletions(-) create mode 100644 packages/server/drizzle/0027_push_pack_types.sql create mode 100644 packages/server/drizzle/meta/0027_snapshot.json diff --git a/packages/protocol/src/repo/layouts/keyed.ts b/packages/protocol/src/repo/layouts/keyed.ts index 91a3ed31..49002598 100644 --- a/packages/protocol/src/repo/layouts/keyed.ts +++ b/packages/protocol/src/repo/layouts/keyed.ts @@ -540,28 +540,36 @@ export class KeyedLayout implements RecordStore { }) } - /** Merge eight small packs of a tier into one, re-pointing their records in a new layer. */ + /** + * Merge eight small packs of a tier and type into one, re-pointing their records in + * a new layer. A merge takes one type's packs, so its output holds one type and a + * type's records gather in fewer packs (a leaf read of one type at three years of + * Crossref dailies, simulated: 72 GETs at the median where mixing types took 255). + * Packs from before types were recorded merge among themselves. + */ async #mergePacks(): Promise { const m = await this.manifests.latest() - const tiers = new Map() + const queues = new Map() for (const p of m.packs) { if (KeyedLayout.final(p)) continue - tiers.set(p.tier, [...(tiers.get(p.tier) ?? []), p]) + const q = `${p.tier}\u0000${p.type ?? ''}` + const queue = queues.get(q) ?? { tier: p.tier, packs: [] } + queue.packs.push(p) + queues.set(q, queue) } - // The tier with the most packs: the lowest that has enough never runs out while - // commits land, so higher tiers never merged (round 14 at 20,000 collections: - // 718 tier-1 packs listed and rising). - const [tier, most] = [...tiers].reduce<[number, number]>( - (best, [t, ps]) => (ps.length > best[1] ? [t, ps.length] : best), - [-1, 0], - ) - if (most < FANOUT) return false - // As many of the tier's packs as fit MERGE_PACKS_BYTES stored (at least FANOUT): many + // The queue with the most packs: the lowest tier that has enough never runs out + // while commits land, so higher tiers never merged (round 14 at 20,000 + // collections: 718 tier-1 packs listed and rising). + let queue: { tier: number; packs: PackInfo[] } | undefined + for (const q of queues.values()) if (q.packs.length > (queue?.packs.length ?? 0)) queue = q + if (!queue || queue.packs.length < FANOUT) return false + const tier = queue.tier + // As many of the queue's packs as fit MERGE_PACKS_BYTES stored (at least FANOUT): many // collections' tiny commits each write a tiny pack, and eight at a time fell behind // (6,639 waiting at 15,000 collections, every commit reading them all). const inputs: PackInfo[] = [] let stored = 0 - for (const p of tiers.get(tier)!) { + for (const p of queue.packs) { if ( inputs.length >= FANOUT && (stored + p.bytes > MERGE_PACKS_BYTES || inputs.length >= layoutConfig.mergeInputs) diff --git a/packages/protocol/src/repo/layouts/kit.ts b/packages/protocol/src/repo/layouts/kit.ts index 79750875..b87da132 100644 --- a/packages/protocol/src/repo/layouts/kit.ts +++ b/packages/protocol/src/repo/layouts/kit.ts @@ -82,6 +82,17 @@ export interface PackInfo { /** Stored (compressed) bytes, and raw. */ bytes: number raw: number + /** + * The record type the pack holds: a pack closes where the type changes, so no pack + * spans types and merges take one type's packs. Absent on packs written before. + */ + type?: string +} + +/** The record type a key leads with (recordKey); '' for a key without one. */ +const typeOf = (key: string) => { + const i = key.indexOf('\u0000') + return i < 0 ? '' : key.slice(0, i) } /** @@ -101,6 +112,7 @@ export class PackWriter { #packRaw = 0 #records = 0 #lastKey: string | null = null + #type = '' readonly done: PackInfo[] = [] readonly #writing = new Set>() #error: unknown = null @@ -110,9 +122,15 @@ export class PackWriter { readonly opts: { tier?: number; hooks?: PackHooks } = {}, ) {} - /** Add a record line under `key`; returns its place. A key below the last starts a new pack. */ + /** + * Add a record line under `key`; returns its place. A key below the last, or of + * another type, starts a new pack. + */ append(key: string, line: string): string { - if (this.#lastKey !== null && compareUtf8(key, this.#lastKey) < 0) this.#close() + const type = typeOf(key) + if (this.#lastKey !== null && (type !== this.#type || compareUtf8(key, this.#lastKey) < 0)) + this.#close() + this.#type = type this.#lastKey = key if (!this.#id) this.#id = newId() this.#lines.push(`${key}\t${line}`) @@ -151,7 +169,13 @@ export class PackWriter { this.#seal() this.#lastKey = null if (!this.#blocks.length) return - const job = { id: this.#id, blocks: this.#blocks, records: this.#records, raw: this.#packRaw } + const job = { + id: this.#id, + blocks: this.#blocks, + records: this.#records, + raw: this.#packRaw, + type: this.#type, + } this.#id = '' this.#blocks = [] this.#records = 0 @@ -164,19 +188,26 @@ export class PackWriter { this.#writing.add(p) } - async #write(job: { id: string; blocks: Promise[]; records: number; raw: number }) { + async #write(job: { + id: string + blocks: Promise[] + records: number + raw: number + type: string + }) { const parts = await Promise.all(job.blocks) const offsets = [0] for (const p of parts) offsets.push(offsets.at(-1)! + p.byteLength) await this.opts.hooks?.before?.(job.id) await putJoined(this.store, packKey(job.id), parts, { ifAbsent: true }) await this.store.put(packDirKey(job.id), JSON.stringify(offsets), { ifAbsent: true }) - const pack = { + const pack: PackInfo = { id: job.id, tier: this.opts.tier ?? 0, records: job.records, bytes: offsets.at(-1)!, raw: job.raw, + ...(job.type ? { type: job.type } : {}), } this.done.push(pack) await this.opts.hooks?.after?.(pack) diff --git a/packages/protocol/test/repo/layouts.test.ts b/packages/protocol/test/repo/layouts.test.ts index a8a4e742..11a82a6f 100644 --- a/packages/protocol/test/repo/layouts.test.ts +++ b/packages/protocol/test/repo/layouts.test.ts @@ -392,6 +392,58 @@ describe('pack merges', () => { }, 30_000) }) +describe('packs never span types', () => { + it('close where the type changes, and say which type they hold', async () => { + const layout = fLayout() + const w = layout.uploadWriter() + for (const type of ['A', 'B', 'C']) + for (let i = 0; i < 5; i++) + w.append({ type, id: ids[i]!, hash: `h${i}`, line: `${type}${i}` }) + // Packs are reported as their writes finish, in any order. + const packs = await w.finish() + expect(packs.map((p) => `${p.type}: ${p.records}`).sort()).toEqual(['A: 5', 'B: 5', 'C: 5']) + }) + + it("merge one type's packs at a time, keeping every record readable", async () => { + const saved = { ...layoutConfig } + layoutConfig.graceMs = 0 + // Eight packs a merge: eight of one type make a pack of 32 records, where eight of + // both types would make two of 16 and tier up half-full. + layoutConfig.mergeInputs = 8 + try { + const layout = fLayout() + // Tiny commits, each of two types: every commit writes a pack of each. + const types = ['A', 'B'] + const rec = (i: number) => ({ key: ids[i]!, hash: `h${i}`, size: 10 }) + for (let c = 0; c < 16; c++) { + const w = layout.writer() + for (const type of types) + for (let i = c * 4; i < c * 4 + 4; i++) w.add(type, rec(i), `${type} ${i}`) + await w.finish() + } + const before = (await layout.manifests.latest()).packs + expect(before).toHaveLength(32) + expect(new Set(before.map((p) => p.type))).toEqual(new Set(types)) + while (await layout.maintain()); + const after = (await layout.manifests.latest()).packs + expect(after.map((p) => `${p.type} tier ${p.tier}: ${p.records}`).sort()).toEqual([ + 'A tier 1: 32', + 'A tier 1: 32', + 'B tier 1: 32', + 'B tier 1: 32', + ]) + for (const type of types) { + const entries = Array.from({ length: 64 }, (_, i) => rec(i)) + expect(await layout.leafLines({ hash: 'leaf', type, entries })).toEqual( + entries.map((_, i) => `${type} ${i}`), + ) + } + } finally { + Object.assign(layoutConfig, saved) + } + }) +}) + describe('layout F', () => { const rec = (i: number, v = 1) => ({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 }) const line = (i: number, v = 1) => `line ${v}.${i}` diff --git a/packages/server/drizzle/0027_push_pack_types.sql b/packages/server/drizzle/0027_push_pack_types.sql new file mode 100644 index 00000000..33b1b2a5 --- /dev/null +++ b/packages/server/drizzle/0027_push_pack_types.sql @@ -0,0 +1 @@ +ALTER TABLE `push_packs` ADD `record_type` text; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0027_snapshot.json b/packages/server/drizzle/meta/0027_snapshot.json new file mode 100644 index 00000000..967ecd8c --- /dev/null +++ b/packages/server/drizzle/meta/0027_snapshot.json @@ -0,0 +1,3689 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "6fed31a6-5812-4a14-a6b3-b5bc8df442b5", + "prevId": "b7f7c1c5-2c58-4d38-a78e-dec398756445", + "tables": { + "abuse_reports": { + "name": "abuse_reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "contact": { + "name": "contact", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reporter_id": { + "name": "reporter_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "resolved_by": { + "name": "resolved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "abuse_reports_status_idx": { + "name": "abuse_reports_status_idx", + "columns": ["status", "created_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 86400000 + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 10 + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_request": { + "name": "last_request", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": ["key"], + "isUnique": false + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": ["reference_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_collections": { + "name": "ark_collections", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "ark_id": { + "name": "ark_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "custom_url": { + "name": "custom_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_collections_ark_id_unique": { + "name": "ark_collections_ark_id_unique", + "columns": ["ark_id"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_collections_collection_id_collections_id_fk": { + "name": "ark_collections_collection_id_collections_id_fk", + "tableFrom": "ark_collections", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_record_types": { + "name": "ark_record_types", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirect_url_field": { + "name": "redirect_url_field", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "ark_record_types_collection_id_collections_id_fk": { + "name": "ark_record_types_collection_id_collections_id_fk", + "tableFrom": "ark_record_types", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "ark_record_types_collection_id_record_type_pk": { + "columns": ["collection_id", "record_type"], + "name": "ark_record_types_collection_id_record_type_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_shoulders": { + "name": "ark_shoulders", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "shoulder": { + "name": "shoulder", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_shoulders_shoulder_unique": { + "name": "ark_shoulders_shoulder_unique", + "columns": ["shoulder"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_shoulders_organization_id_organization_id_fk": { + "name": "ark_shoulders_organization_id_organization_id_fk", + "tableFrom": "ark_shoulders", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cleanup_runs": { + "name": "cleanup_runs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "step": { + "name": "step", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dry_run": { + "name": "dry_run", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "requested_by": { + "name": "requested_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mark_run_id": { + "name": "mark_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lease": { + "name": "lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stats": { + "name": "stats", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "cleanup_runs_created_idx": { + "name": "cleanup_runs_created_idx", + "columns": ["created_at"], + "isUnique": false + }, + "cleanup_runs_step_idx": { + "name": "cleanup_runs_step_idx", + "columns": ["step", "status"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_tombstones": { + "name": "collection_tombstones", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "versions": { + "name": "versions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_by": { + "name": "deleted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_layout": { + "name": "record_layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'today'" + }, + "records_dropped_at": { + "name": "records_dropped_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_webhooks": { + "name": "collection_webhooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bump_filter": { + "name": "bump_filter", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_delivery_at": { + "name": "last_delivery_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "collection_webhooks_collection_idx": { + "name": "collection_webhooks_collection_idx", + "columns": ["collection_id"], + "isUnique": false + } + }, + "foreignKeys": { + "collection_webhooks_collection_id_collections_id_fk": { + "name": "collection_webhooks_collection_id_collections_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collections": { + "name": "collections", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "head_version_id": { + "name": "head_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_salt": { + "name": "private_salt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_files_root": { + "name": "public_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "version_count": { + "name": "version_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "history_bytes": { + "name": "history_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_push_at": { + "name": "last_push_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_started_at": { + "name": "reconcile_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_state": { + "name": "reconcile_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_seq": { + "name": "reconciled_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "reconciled_files_root": { + "name": "reconciled_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_layout": { + "name": "record_layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'today'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "collections_org_slug_uq": { + "name": "collections_org_slug_uq", + "columns": ["organization_id", "slug"], + "isUnique": true + }, + "collections_public_updated_idx": { + "name": "collections_public_updated_idx", + "columns": ["public", "updated_at"], + "isUnique": false + } + }, + "foreignKeys": { + "collections_organization_id_organization_id_fk": { + "name": "collections_organization_id_organization_id_fk", + "tableFrom": "collections", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "commit_units": { + "name": "commit_units", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "plan_id": { + "name": "plan_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ord": { + "name": "ord", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "after": { + "name": "after", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "through": { + "name": "through", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "gap": { + "name": "gap", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "slices_key": { + "name": "slices_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_key": { + "name": "output_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "queued_at": { + "name": "queued_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "commit_units_plan_idx": { + "name": "commit_units_plan_idx", + "columns": ["plan_id", "set", "type", "ord"], + "isUnique": false + }, + "commit_units_status_idx": { + "name": "commit_units_status_idx", + "columns": ["session_id", "status", "queued_at"], + "isUnique": false + } + }, + "foreignKeys": { + "commit_units_session_id_push_sessions_id_fk": { + "name": "commit_units_session_id_push_sessions_id_fk", + "tableFrom": "commit_units", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "denylist": { + "name": "denylist", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "file_uploads": { + "name": "file_uploads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "multipart_upload_id": { + "name": "multipart_upload_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "file_uploads_hash_idx": { + "name": "file_uploads_hash_idx", + "columns": ["hash"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "files": { + "name": "files", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "forks": { + "name": "forks", + "columns": { + "child_collection_id": { + "name": "child_collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "parent_collection_id": { + "name": "parent_collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_seq": { + "name": "parent_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'public'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "forks_child_collection_id_collections_id_fk": { + "name": "forks_child_collection_id_collections_id_fk", + "tableFrom": "forks", + "tableTo": "collections", + "columnsFrom": ["child_collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instance_settings": { + "name": "instance_settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invitation": { + "name": "invitation", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "jobs": { + "name": "jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "run_at": { + "name": "run_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locked_until": { + "name": "locked_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "jobs_ready_idx": { + "name": "jobs_ready_idx", + "columns": ["status", "run_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "member": { + "name": "member", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + }, + "member_user_id_idx": { + "name": "member_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organization": { + "name": "organization", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ark_naan": { + "name": "ark_naan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kf_org_id": { + "name": "kf_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "columns": ["slug"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "placements": { + "name": "placements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location_id": { + "name": "location_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "synced_seq": { + "name": "synced_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until": { + "name": "lease_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "placements_one_primary_uq": { + "name": "placements_one_primary_uq", + "columns": ["collection_id"], + "isUnique": true, + "where": "\"placements\".\"role\" = 'primary' AND \"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_collection_location_uq": { + "name": "placements_collection_location_uq", + "columns": ["collection_id", "location_id"], + "isUnique": true, + "where": "\"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_org_location_uq": { + "name": "placements_org_location_uq", + "columns": ["organization_id", "location_id"], + "isUnique": true, + "where": "\"placements\".\"organization_id\" IS NOT NULL" + }, + "placements_location_idx": { + "name": "placements_location_idx", + "columns": ["location_id"], + "isUnique": false + } + }, + "foreignKeys": { + "placements_collection_id_collections_id_fk": { + "name": "placements_collection_id_collections_id_fk", + "tableFrom": "placements", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_organization_id_organization_id_fk": { + "name": "placements_organization_id_organization_id_fk", + "tableFrom": "placements", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_location_id_storage_locations_id_fk": { + "name": "placements_location_id_storage_locations_id_fk", + "tableFrom": "placements", + "tableTo": "storage_locations", + "columnsFrom": ["location_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_packs": { + "name": "push_packs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pack_id": { + "name": "pack_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_set": { + "name": "record_set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "records": { + "name": "records", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw": { + "name": "raw", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "push_packs_session_id_push_sessions_id_fk": { + "name": "push_packs_session_id_push_sessions_id_fk", + "tableFrom": "push_packs", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_packs_session_id_pack_id_pk": { + "columns": ["session_id", "pack_id"], + "name": "push_packs_session_id_pack_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_runs": { + "name": "push_runs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "first_key": { + "name": "first_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_key": { + "name": "last_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "merging_into": { + "name": "merging_into", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "push_runs_tier_idx": { + "name": "push_runs_tier_idx", + "columns": ["session_id", "tier", "merging_into"], + "isUnique": false + } + }, + "foreignKeys": { + "push_runs_session_id_push_sessions_id_fk": { + "name": "push_runs_session_id_push_sessions_id_fk", + "tableFrom": "push_runs", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_runs_session_id_seq_pk": { + "columns": ["session_id", "seq"], + "name": "push_runs_session_id_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_sessions": { + "name": "push_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_version_id": { + "name": "base_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "strip_unknown_fields": { + "name": "strip_unknown_fields", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "records_received": { + "name": "records_received", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "runs": { + "name": "runs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "result": { + "name": "result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finalize_started_at": { + "name": "finalize_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_started": { + "name": "commit_started", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "in_key_order": { + "name": "in_key_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "overlapping_uploads": { + "name": "overlapping_uploads", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cleaned_at": { + "name": "cleaned_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_phase": { + "name": "commit_phase", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "progress_at": { + "name": "progress_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stalls": { + "name": "stalls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "commit_plan": { + "name": "commit_plan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "assembly_lease": { + "name": "assembly_lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "push_sessions_collection_idx": { + "name": "push_sessions_collection_idx", + "columns": ["collection_id"], + "isUnique": false + }, + "push_sessions_expires_idx": { + "name": "push_sessions_expires_idx", + "columns": ["status", "expires_at"], + "isUnique": false + }, + "push_sessions_user_idx": { + "name": "push_sessions_user_idx", + "columns": ["user_id", "status"], + "isUnique": false + } + }, + "foreignKeys": { + "push_sessions_collection_id_collections_id_fk": { + "name": "push_sessions_collection_id_collections_id_fk", + "tableFrom": "push_sessions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compaction_parts": { + "name": "ref_compaction_parts", + "columns": { + "compaction_id": { + "name": "compaction_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "part": { + "name": "part", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_compaction_parts_compaction_id_part_pk": { + "columns": ["compaction_id", "part"], + "name": "ref_compaction_parts_compaction_id_part_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compactions": { + "name": "ref_compactions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_runs": { + "name": "input_runs", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output_run": { + "name": "output_run", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parts": { + "name": "parts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix_length": { + "name": "prefix_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'running'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_index_units": { + "name": "ref_index_units", + "columns": { + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unit": { + "name": "unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "events": { + "name": "events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_index_units_version_id_unit_pk": { + "columns": ["version_id", "unit"], + "name": "ref_index_units_version_id_unit_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_segments": { + "name": "ref_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_hash": { + "name": "first_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_hash": { + "name": "last_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'live'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ref_segments_range_idx": { + "name": "ref_segments_range_idx", + "columns": ["state", "first_hash", "last_hash"], + "isUnique": false + }, + "ref_segments_run_idx": { + "name": "ref_segments_run_idx", + "columns": ["run_id"], + "isUnique": false + }, + "ref_segments_tier_idx": { + "name": "ref_segments_tier_idx", + "columns": ["tier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_labels": { + "name": "schema_labels", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "schema_labels_label_idx": { + "name": "schema_labels_label_idx", + "columns": ["label"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "schema_labels_schema_hash_label_pk": { + "columns": ["schema_hash", "label"], + "name": "schema_labels_schema_hash_label_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_usage": { + "name": "schema_usage", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type_slug": { + "name": "type_slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_seq": { + "name": "from_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "to_seq": { + "name": "to_seq", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "schema_usage_hash_idx": { + "name": "schema_usage_hash_idx", + "columns": ["schema_hash"], + "isUnique": false + } + }, + "foreignKeys": { + "schema_usage_collection_id_collections_id_fk": { + "name": "schema_usage_collection_id_collections_id_fk", + "tableFrom": "schema_usage", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "schema_usage_collection_id_type_slug_set_from_seq_pk": { + "columns": ["collection_id", "type_slug", "set", "from_seq"], + "name": "schema_usage_collection_id_type_slug_set_from_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schemas": { + "name": "schemas", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": ["token"], + "isUnique": true + }, + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_fence": { + "name": "storage_fence", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "epoch": { + "name": "epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "window_until": { + "name": "window_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "window_run_id": { + "name": "window_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_locations": { + "name": "storage_locations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucket": { + "name": "bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "credentials": { + "name": "credentials", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unverified'" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checked_at": { + "name": "checked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "storage_locations_organization_id_organization_id_fk": { + "name": "storage_locations_organization_id_organization_id_fk", + "tableFrom": "storage_locations", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "usage_rollups": { + "name": "usage_rollups", + "columns": { + "day": { + "name": "day", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metric": { + "name": "metric", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "amount": { + "name": "amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "usage_rollups_account_idx": { + "name": "usage_rollups_account_idx", + "columns": ["account_id", "day"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "usage_rollups_day_account_id_collection_id_metric_pk": { + "columns": ["day", "account_id", "collection_id", "metric"], + "name": "usage_rollups_day_account_id_collection_id_metric_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": ["email"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": ["identifier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "versions": { + "name": "versions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "major": { + "name": "major", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "minor": { + "name": "minor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "patch": { + "name": "patch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pushed_by": { + "name": "pushed_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_record_count": { + "name": "public_record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_file_count": { + "name": "public_file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "public_total_bytes": { + "name": "public_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "type_counts": { + "name": "type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_type_counts": { + "name": "public_type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "has_private": { + "name": "has_private", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "public_refs_root": { + "name": "public_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_refs_root": { + "name": "private_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refs_indexed": { + "name": "refs_indexed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "changes": { + "name": "changes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "push_session_id": { + "name": "push_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "versions_collection_seq_uq": { + "name": "versions_collection_seq_uq", + "columns": ["collection_id", "seq"], + "isUnique": true + }, + "versions_collection_semver_uq": { + "name": "versions_collection_semver_uq", + "columns": ["collection_id", "semver"], + "isUnique": true + }, + "versions_hash_idx": { + "name": "versions_hash_idx", + "columns": ["hash"], + "isUnique": false + }, + "versions_published_idx": { + "name": "versions_published_idx", + "columns": ["published_at"], + "isUnique": false + }, + "versions_push_session_idx": { + "name": "versions_push_session_idx", + "columns": ["push_session_id"], + "isUnique": false + } + }, + "foreignKeys": { + "versions_collection_id_collections_id_fk": { + "name": "versions_collection_id_collections_id_fk", + "tableFrom": "versions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhook_deliveries": { + "name": "webhook_deliveries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "webhook_id": { + "name": "webhook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bump_type": { + "name": "bump_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'version.created'" + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "delivered_at": { + "name": "delivered_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhook_deliveries_webhook_idx": { + "name": "webhook_deliveries_webhook_idx", + "columns": ["webhook_id", "created_at"], + "isUnique": false + }, + "webhook_deliveries_pending_idx": { + "name": "webhook_deliveries_pending_idx", + "columns": ["status", "next_attempt_at"], + "isUnique": false + } + }, + "foreignKeys": { + "webhook_deliveries_webhook_id_collection_webhooks_id_fk": { + "name": "webhook_deliveries_webhook_id_collection_webhooks_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collection_webhooks", + "columnsFrom": ["webhook_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index e60d3a5b..ca816c24 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -190,6 +190,13 @@ "when": 1792524725308, "tag": "0026_push_packs", "breakpoints": true + }, + { + "idx": 27, + "version": "6", + "when": 1792605564163, + "tag": "0027_push_pack_types", + "breakpoints": true } ] } diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index f59a0be9..bb4a59b5 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -699,6 +699,8 @@ export const pushPacks = sqliteTable( records: integer('records'), bytes: integer('bytes'), raw: integer('raw'), + /** The record type the pack holds (PackInfo.type): merges take one type's packs. */ + recordType: text('record_type'), createdAt: createdAt(), }, (t) => [primaryKey({ columns: [t.sessionId, t.packId] })], diff --git a/packages/server/src/push/packs.ts b/packages/server/src/push/packs.ts index dd5d8580..8fcaafa5 100644 --- a/packages/server/src/push/packs.ts +++ b/packages/server/src/push/packs.ts @@ -44,7 +44,7 @@ const hooksFor = (ports: Ports, sessionId: string, set: RecordSet): PackHooks => after: async (pack) => { await ports.db .update(schema.pushPacks) - .set({ records: pack.records, bytes: pack.bytes, raw: pack.raw }) + .set({ records: pack.records, bytes: pack.bytes, raw: pack.raw, recordType: pack.type }) .where(and(eq(schema.pushPacks.sessionId, sessionId), eq(schema.pushPacks.packId, pack.id))) }, }) @@ -126,7 +126,14 @@ export async function adoptSessionPacks(ports: Ports, repo: Repo, sessionId: str for (const set of SETS) { const packs: PackInfo[] = rows .filter((r) => r.set === set) - .map((r) => ({ id: r.packId, tier: 0, records: r.records!, bytes: r.bytes!, raw: r.raw! })) + .map((r) => ({ + id: r.packId, + tier: 0, + records: r.records!, + bytes: r.bytes!, + raw: r.raw!, + ...(r.recordType ? { type: r.recordType } : {}), + })) if (packs.length) await repo.records[set].adoptPacks!(sessionId, packs) } } diff --git a/packages/server/test/upload-packs.test.ts b/packages/server/test/upload-packs.test.ts index 2c3d7ba9..58b0fc99 100644 --- a/packages/server/test/upload-packs.test.ts +++ b/packages/server/test/upload-packs.test.ts @@ -143,6 +143,8 @@ describe.runIf(process.env.UNDERLAY_TEST_LAYOUT === 'f')('uploads into layout F' const written = rows.filter((r) => r.records !== null) expect(written).toHaveLength(1) expect(written[0]!.records).toBe(300) + // Its type, so a merge once it's listed takes it with its own type's packs. + expect(written[0]!.recordType).toBe('Doc') // The commit indexes them where they are: no record is written again. expect(await commit(h, user, 'a', sid)).toBe(201) @@ -284,11 +286,14 @@ describe.runIf(process.env.UNDERLAY_TEST_LAYOUT === 'f')('uploads into layout F' expect(out.filter((r) => /changed|small/.test(r))).toHaveLength(20) const repo = await h.ports.stores.forCollection(c.id) const layout = repo.records!.public as unknown as { - manifests: { latest(): Promise<{ packs: { id: string; tier: number }[] }> } + manifests: { + latest(): Promise<{ packs: { id: string; tier: number; type?: string }[] }> + } } const m = await layout.manifests.latest() - // Every small pack listed, or merged into one above them. + // Every small pack listed, or merged into one above them, and each says its type. expect(m.packs.some((p) => p.tier > 0) || m.packs.length >= 10).toBe(true) + expect(m.packs.every((p) => p.type === 'Doc')).toBe(true) expect(new Set(m.packs.map((p) => p.id)).size).toBe(m.packs.length) })