From 1d3cfa4e92373bc5f89ace253587fb196df42409 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 20 Oct 2026 19:51:10 +0200 Subject: [PATCH 1/2] Layout F: uploads write records straight into packs; runs carry their places --- packages/protocol/src/index.ts | 4 + packages/protocol/src/repo/file-sets.ts | 13 +- packages/protocol/src/repo/layouts/index.ts | 2 + packages/protocol/src/repo/layouts/keyed.ts | 107 +- packages/protocol/src/repo/layouts/kit.ts | 32 +- packages/protocol/src/repo/layouts/types.ts | 36 +- packages/protocol/src/repo/repo.ts | 11 +- packages/protocol/test/repo/layouts.test.ts | 92 + packages/server/drizzle/0026_push_packs.sql | 13 + .../server/drizzle/meta/0026_snapshot.json | 3666 +++++++++++++++++ packages/server/drizzle/meta/_journal.json | 7 + packages/server/src/api/push.ts | 2 + packages/server/src/cleanup/internal.ts | 14 +- packages/server/src/db/schema.ts | 30 + packages/server/src/push/changes.ts | 14 +- packages/server/src/push/delta.ts | 24 +- packages/server/src/push/packs.ts | 157 + packages/server/src/push/runs.ts | 6 +- packages/server/src/push/session.ts | 5 + packages/server/src/versions/commit.ts | 4 + packages/server/test/cleanup.test.ts | 76 +- packages/server/test/stream-push.test.ts | 5 +- packages/server/test/upload-packs.test.ts | 365 ++ 23 files changed, 4620 insertions(+), 65 deletions(-) create mode 100644 packages/server/drizzle/0026_push_packs.sql create mode 100644 packages/server/drizzle/meta/0026_snapshot.json create mode 100644 packages/server/src/push/packs.ts create mode 100644 packages/server/test/upload-packs.test.ts diff --git a/packages/protocol/src/index.ts b/packages/protocol/src/index.ts index 9b78817f..704c036b 100644 --- a/packages/protocol/src/index.ts +++ b/packages/protocol/src/index.ts @@ -20,9 +20,13 @@ export { type LiveSet, type MergePlan, type PackInfo, + packIdsFor, + type PackRecord, + type Placed, MissingRecordError, type ReclaimPlan, recordKey, + UPLOAD_PACK_FINAL_RAW, type RecordSet, type RecordSets, setHome, diff --git a/packages/protocol/src/repo/file-sets.ts b/packages/protocol/src/repo/file-sets.ts index 854c290a..9816e708 100644 --- a/packages/protocol/src/repo/file-sets.ts +++ b/packages/protocol/src/repo/file-sets.ts @@ -38,6 +38,7 @@ import { type SetObject, type TreeSummary, } from '../format.js' +import type { Placed } from './layouts/types.js' import { outOfLineHash, type Repo, RepoSink, RepoSource } from './repo.js' export type SetName = 'public' | 'private' @@ -103,8 +104,16 @@ export class FileRefDelta { /** Feed one record change of a type in a set (mergeTree's onChange). */ record(set: SetName, before: RecordEntry | null, after: RecordEntry | null, type: string): void { - if (before && !this.noOldRefs.has(set)) this.#body(set, type, before.body, -1) - if (after) this.#body(set, type, after.body, +1) + if (before && !this.noOldRefs.has(set)) this.#entry(set, type, before, -1) + if (after) this.#entry(set, type, after, +1) + } + + #entry(set: SetName, type: string, e: RecordEntry, by: number) { + // Written into a pack at upload: its references were read from it then. + const p = e as Placed + if (p.body === undefined && p.packed !== undefined) { + for (const h of p.fileRefs ?? []) this.add(set, type, h, by) + } else this.#body(set, type, e.body, by) } /** diff --git a/packages/protocol/src/repo/layouts/index.ts b/packages/protocol/src/repo/layouts/index.ts index d6e153db..b7c1e21b 100644 --- a/packages/protocol/src/repo/layouts/index.ts +++ b/packages/protocol/src/repo/layouts/index.ts @@ -11,7 +11,9 @@ export { layoutConfig, type MergePlan, MissingRecordError, + packIdsFor, type ReclaimPlan, + UPLOAD_PACK_FINAL_RAW, } from './keyed.js' export { headConfig, type LayerInfo, type PackInfo, pendingConfig } from './kit.js' export * from './types.js' diff --git a/packages/protocol/src/repo/layouts/keyed.ts b/packages/protocol/src/repo/layouts/keyed.ts index ff47313b..7d32f4cf 100644 --- a/packages/protocol/src/repo/layouts/keyed.ts +++ b/packages/protocol/src/repo/layouts/keyed.ts @@ -36,6 +36,7 @@ import { newId, type PackInfo, PackReader, + PACK_RAW, PackWriter, packDirKey, packKey, @@ -47,7 +48,13 @@ import { filterKey, } from './kit.js' import { type KeyRange, LiveHashes } from './reclaim.js' -import { type LiveSet, recordKey, type RecordStore, type RecordWriter } from './types.js' +import { + type LiveSet, + type PackRecord, + recordKey, + type RecordStore, + type RecordWriter, +} from './types.js' /** Index entries a reclaim range job works on: its live set is 8 bytes an entry. */ export const RECLAIM_RANGE_ENTRIES = 250_000 @@ -113,6 +120,27 @@ const MAX_RUNS = 10 /** Packs under this many stored bytes, times 8 per tier, are merged eight at a time. */ const SMALL_PACK = 4 * 1024 * 1024 const MAX_PACK_TIER = 3 +/** + * An upload's pack this large (raw) is listed with the packs no merge takes: a bulk + * load's batches (8 MB each) are written once, where they stay. Merging them was + * the external sort again, one pack at a time on the set's one maintain lane. + * Smaller ones (a small push's) are merged like a commit's. + */ +export const UPLOAD_PACK_FINAL_RAW = 1024 * 1024 +/** Commits whose uploads' packs a manifest remembers listing (`Manifest.adopted`). */ +const ADOPTED_KEPT = 1000 + +/** + * Ids for the packs `writePacks` makes of these records, recorded before they're + * written (so an upload that dies leaves nothing unknown). A pack closes only once + * it holds PACK_RAW (PackWriter, the same count of a line), so this many at most. + */ +export function packIdsFor(records: readonly PackRecord[]): string[] { + let raw = 0 + for (const r of records) raw += r.id.length + r.hash.length + r.line.length + 3 + return Array.from({ length: Math.floor(raw / PACK_RAW) + 1 }, newId) +} + /** How long a replaced object stays readable (readers holding an older manifest). */ export const layoutConfig = { graceMs: 60_000, @@ -272,6 +300,78 @@ export class KeyedLayout implements RecordStore { return new KeyedWriter(this) } + /** + * An upload's records into packs of their own, in key order, outside the index and + * the manifest until a commit names them (adoptPacks). Their places, in the order + * given, go in the push's runs, so a commit indexes the records without moving them. + */ + async writePacks( + records: readonly PackRecord[], + ids: readonly string[], + ): Promise<{ places: string[]; packs: PackInfo[] }> { + let next = 0 + const pw = new PackWriter(this.store, { + ids: () => { + const id = ids[next++] + if (id === undefined) throw new Error('writePacks: the records need more packs than ids') + return id + }, + }) + const keys = records.map((r) => recordKey(r.id, r.hash)) + const order = keys.map((_, i) => i).sort((a, b) => compareUtf8(keys[a]!, keys[b]!)) + const places: string[] = Array.from({ length: records.length }) + let n = 0 + for (const i of order) { + places[i] = pw.append(keys[i]!, records[i]!.line) + if (++n % 1024 === 0) await pw.ready() + } + return { places, packs: await pw.finish() } + } + + /** + * List packs uploads wrote, once the commit's index names their records (after its + * runs are appended: a merge re-points a pack's records in a newer layer, which a + * commit's layer folded in later would otherwise override with the old places). + * Large ones go in the registry, small ones in the manifest for merges. Once only + * a `name`: a retry finds it in `adopted` and lists nothing. + */ + async adoptPacks(name: string, packs: readonly PackInfo[]): Promise { + if (!packs.length) return + if ((await this.manifests.latest()).adopted?.includes(name)) return + const large = (p: PackInfo) => p.raw >= UPLOAD_PACK_FINAL_RAW || KeyedLayout.final(p) + // A fixed key: written once even if this stops before the manifest says so. + await this.registry.add(packs.filter(large), `upload-${name}`) + const small = packs.filter((p) => !large(p)) + await this.manifests.update((m) => { + if (m.adopted?.includes(name)) return null + const have = new Set(m.packs.map((p) => p.id)) + m.packs.push(...small.filter((p) => !have.has(p.id))) + m.adopted = [...(m.adopted ?? []), name].slice(-ADOPTED_KEPT) + return m + }) + } + + /** + * Delete packs an upload wrote for a commit `name` that never named them (an + * abandoned push). A commit that named them and then didn't publish (a conflict, + * then the session failed) leaves them: the manifest or registry lists them, so + * they go only by reclaim. False when kept. + */ + async dropPacks(name: string, ids: readonly string[]): Promise { + const m = await this.manifests.latest() + if (m.adopted?.includes(name)) return false + if (await this.store.head(this.registry.keyFor(`upload-${name}`))) return false + const listed = new Set([...m.packs.map((p) => p.id), ...m.retired.map((r) => r.key)]) + const gone = ids.filter((id) => !listed.has(id) && !listed.has(packKey(id))) + for (let i = 0; i < gone.length; i += 16) + await Promise.all( + gone + .slice(i, i + 16) + .flatMap((id) => [this.store.delete(packKey(id)), this.store.delete(packDirKey(id))]), + ) + return true + } + /** A commit's runs its units held back (RecordWriter.finish), as one pending object. */ appendRuns(runs: PendingRun[]): Promise { return this.manifests.appendRuns(runs) @@ -1229,6 +1329,11 @@ class KeyedWriter implements RecordWriter { return at } + addPlaced(entry: RecordEntry, place: string): string { + this.#entries.push([recordKey(entry.key, entry.hash), place]) + return place + } + async finish(opts: { hold?: boolean } = {}): Promise { const packs = await this.#packs.finish() if (!this.#entries.length) return null diff --git a/packages/protocol/src/repo/layouts/kit.ts b/packages/protocol/src/repo/layouts/kit.ts index bb9ae831..9933b000 100644 --- a/packages/protocol/src/repo/layouts/kit.ts +++ b/packages/protocol/src/repo/layouts/kit.ts @@ -77,14 +77,15 @@ export class PackWriter { constructor( readonly store: Store, - readonly opts: { tier?: number } = {}, + /** `ids` names each new pack (an upload's, recorded before they're written); else random. */ + readonly opts: { tier?: number; ids?: () => string } = {}, ) {} /** Add a record line under `key`; returns its place. A key below the last starts a new pack. */ append(key: string, line: string): string { if (this.#lastKey !== null && compareUtf8(key, this.#lastKey) < 0) this.#close() this.#lastKey = key - if (!this.#id) this.#id = newId() + if (!this.#id) this.#id = this.opts.ids ? this.opts.ids() : newId() this.#lines.push(`${key}\t${line}`) const at = loc(this.#id, this.#blocks.length, this.#lines.length - 1) this.#raw += key.length + line.length + 2 @@ -758,6 +759,11 @@ export interface Manifest { * delete failed must never be folded again once its layers are merged away. */ folded?: string[] + /** + * The latest commits whose uploads' packs were listed (KeyedLayout.adoptPacks): a + * retry lists none again, not even one a merge has since taken and swept. + */ + adopted?: string[] } const empty = (): Manifest => ({ seq: 0, layers: [], packs: [], retired: [], stats: {} }) @@ -812,14 +818,22 @@ export class PackRegistry { return `manifest/${this.name}/packs/` } - /** List `packs` in a new object; its key (null for none). */ - async add(packs: readonly PackInfo[]): Promise { + /** + * List `packs` in a new object; its key (null for none). `name` makes the key + * fixed, so listing the same packs again (a retried commit) writes nothing. + */ + async add(packs: readonly PackInfo[], name: string = newId()): Promise { if (!packs.length) return null - const key = `${this.#prefix}${newId()}.json` + const key = this.keyFor(name) await this.store.put(key, JSON.stringify(packs), { ifAbsent: true }) return key } + /** The key of the registry object `name`. */ + keyFor(name: string): string { + return `${this.#prefix}${name}.json` + } + async read(): Promise<{ keys: string[]; packs: PackInfo[] }> { const keys: string[] = [] let cursor: string | undefined @@ -828,11 +842,13 @@ export class PackRegistry { keys.push(...page.keys) cursor = page.cursor } while (cursor) - const packs: PackInfo[] = [] + // A pack listed twice (a retried commit naming an upload's packs after a reclaim + // replaced their object) is one pack. + const packs = new Map() for (let i = 0; i < keys.length; i += 16) for (const o of await Promise.all(keys.slice(i, i + 16).map((k) => this.store.get(k)))) - if (o) packs.push(...(JSON.parse(await o.text()) as PackInfo[])) - return { keys, packs } + if (o) for (const p of JSON.parse(await o.text()) as PackInfo[]) packs.set(p.id, p) + return { keys, packs: [...packs.values()] } } /** Replace the objects `keys` with one listing `packs`. */ diff --git a/packages/protocol/src/repo/layouts/types.ts b/packages/protocol/src/repo/layouts/types.ts index 8fc847bb..b181d0bc 100644 --- a/packages/protocol/src/repo/layouts/types.ts +++ b/packages/protocol/src/repo/layouts/types.ts @@ -5,13 +5,24 @@ * holds. Neither changes a node, a root or any hash. */ import type { RecordEntry } from '../../tree/node.js' -import type { PendingRun } from './kit.js' +import type { PackInfo, PendingRun } from './kit.js' export const LAYOUTS = ['today', 'f'] as const export type LayoutName = (typeof LAYOUTS)[number] -/** A record entry as layout F carries it through a build: with its place once stored. */ -export type Placed = RecordEntry & { loc?: string } +/** + * A record entry as layout F carries it through a build: with its place once stored. + * A record written into a pack at upload comes without its body: `packed` is its + * place there and `fileRefs` the files it references, read from it at upload. + */ +export type Placed = RecordEntry & { loc?: string; packed?: string; fileRefs?: string[] } + +/** A record an upload writes into a pack: its id, hash and canonical line. */ +export interface PackRecord { + id: string + hash: string + line: string +} /** Records still in use, from a mark over every retained version (storage reclaim). */ export interface LiveSet { @@ -57,11 +68,30 @@ export interface RecordStore { appendRuns?(runs: HeldRun[]): Promise /** Give a fork its records: it lists the parent's files and reads them where they are. */ forkInto?(target: RecordStore): Promise> + /** + * Write an upload's records into packs of their own, named by `ids` (more than it + * needs; see `packIdsFor`), outside the index and the manifest: their places, in + * the order given, and the packs written. + */ + writePacks?( + records: readonly PackRecord[], + ids: readonly string[], + ): Promise<{ places: string[]; packs: PackInfo[] }> + /** + * List packs uploads wrote, once the commit's index names their records: large + * ones in the registry, small ones in the manifest for merges. `name` is the + * commit's, so a retry lists nothing twice. + */ + adoptPacks?(name: string, packs: readonly PackInfo[]): Promise + /** Delete packs uploads wrote for commit `name`, unless it named them (false then). */ + dropPacks?(name: string, ids: readonly string[]): Promise } export interface RecordWriter { /** Store one record line (a spilled or new entry): its place. */ add(entry: RecordEntry, line: string): string + /** Index a record an upload already wrote into a pack, at `place`; returns it. */ + addPlaced(entry: RecordEntry, place: string): string /** * Write what's held: packs, the index layer, then the manifest. With `hold`, the * manifest isn't told: the run is returned, for the commit to name with its others. diff --git a/packages/protocol/src/repo/repo.ts b/packages/protocol/src/repo/repo.ts index c7028659..43a65dc8 100644 --- a/packages/protocol/src/repo/repo.ts +++ b/packages/protocol/src/repo/repo.ts @@ -875,11 +875,16 @@ export class RepoSink implements TreeSink { this.#members.push(this.#member(entries)) } - /** Store a new record in the packed layout, and note its place on the entry. */ + /** + * Store a new record in the packed layout, and note its place on the entry. One an + * upload already wrote into a pack (`packed`) is only indexed there: its bytes + * aren't read or written again. + */ #place(e: E) { const p = e as unknown as Placed - if (p.loc !== undefined || p.body === undefined) return - p.loc = this.#writer!.add(p, p.body) + if (p.loc !== undefined) return + if (p.packed !== undefined) p.loc = this.#writer!.addPlaced(p, p.packed) + else if (p.body !== undefined) p.loc = this.#writer!.add(p, p.body) } leaf(desc: NodeDesc, json: string, entries: readonly E[], spilled: number): void { diff --git a/packages/protocol/test/repo/layouts.test.ts b/packages/protocol/test/repo/layouts.test.ts index c7cae337..7e765c4f 100644 --- a/packages/protocol/test/repo/layouts.test.ts +++ b/packages/protocol/test/repo/layouts.test.ts @@ -25,12 +25,14 @@ import { mergeTree, type NodeSource, openRepo, + packIdsFor, type RecordEntry, recordCanonical, RepoSink, recordTree, sha256Hex, treeObjects, + UPLOAD_PACK_FINAL_RAW, utf8ByteLength, } from '../../src/index.js' import { @@ -595,6 +597,96 @@ describe("a commit's held runs", () => { }) }) +describe("an upload's packs", () => { + const rec = (i: number, v = 1) => ({ key: ids[i]!, hash: `h${v}.${i}`, size: 10 }) + const line = (i: number, v = 1) => `line ${v}.${i}` + const upload = (from: number, to: number, v = 1) => + Array.from({ length: to - from }, (_, j) => { + // Out of key order, as a batch arrives. + const i = to - 1 - j + return { id: ids[i]!, hash: `h${v}.${i}`, line: line(i, v) } + }) + + it('are indexed where they lie, and merges re-point them once listed', async () => { + const saved = layoutConfig.graceMs + layoutConfig.graceMs = 0 + try { + const root = memoryStore() + const layout = fLayout(root) + const packsNow = async () => (await root.list('runs/c1/packs/')).keys.length + // Ten small uploads, each into a pack of its own; their commits index them. + for (let u = 0; u < 10; u++) { + const recs = upload(u * 100, u * 100 + 100) + const ids_ = packIdsFor(recs) + expect(ids_).toHaveLength(1) + const { places, packs } = await layout.writePacks(recs, ids_) + expect(packs.map((p) => p.id)).toEqual(ids_) + const before = await packsNow() + const w = layout.writer() + recs.forEach((r, i) => w.addPlaced(rec(u * 100 + (99 - i)), places[i]!)) + await w.finish() + // The commit wrote no pack: only its index. + expect(await packsNow()).toBe(before) + await layout.adoptPacks(`s${u}`, packs) + } + const all = Array.from({ length: 1000 }, (_, i) => i) + const read = () => + layout.leafLines( + 'leaf', + all.map((i) => rec(i)), + ) + expect(await read()).toEqual(all.map((i) => line(i))) + // Small packs: listed in the manifest, merged, their records re-pointed. + expect((await layout.manifests.latest()).packs).toHaveLength(10) + await layout.maintain() + await new Promise((r) => setTimeout(r, 5)) + await layout.maintain() + const m = await layout.manifests.latest() + expect(m.packs.length).toBeLessThan(10) + expect(await read()).toEqual(all.map((i) => line(i))) + } finally { + layoutConfig.graceMs = saved + } + }) + + it('are listed once however often a commit retries, even after a merge took them', async () => { + const root = memoryStore() + const layout = fLayout(root) + const recs = upload(0, 50) + const { packs } = await layout.writePacks(recs, packIdsFor(recs)) + await layout.adoptPacks('s1', packs) + await layout.adoptPacks('s1', packs) + expect((await layout.manifests.latest()).packs.map((p) => p.id)).toEqual([packs[0]!.id]) + // As if a merge took it: a retry must not list it again. + await layout.manifests.update((m) => ({ ...m, packs: [] })) + await layout.adoptPacks('s1', packs) + expect((await layout.manifests.latest()).packs).toEqual([]) + + // A large one goes in the registry, once. + const large = { ...packs[0]!, raw: UPLOAD_PACK_FINAL_RAW } + await layout.adoptPacks('s2', [large]) + await layout.adoptPacks('s2', [large]) + expect((await layout.registry.read()).packs).toEqual([large]) + }) + + it('are dropped for a push that never listed them, and kept once it did', async () => { + const root = memoryStore() + const layout = fLayout(root) + const named = upload(0, 20) + const a = await layout.writePacks(named, packIdsFor(named)) + await layout.adoptPacks('kept', a.packs) + const left = upload(20, 40) + const b = await layout.writePacks(left, packIdsFor(left)) + const files = async () => (await root.list('runs/c1/packs/')).keys + expect(await files()).toHaveLength(4) + expect(await layout.dropPacks('kept', [a.packs[0]!.id])).toBe(false) + expect(await layout.dropPacks('left', [b.packs[0]!.id, 'neverwritten'])).toBe(true) + expect((await files()).sort()).toEqual( + [`runs/c1/packs/${a.packs[0]!.id}.dir`, `runs/c1/packs/${a.packs[0]!.id}.pk`].sort(), + ) + }) +}) + describe('the maintain lease', () => { it('is left alone when another job took it over', async () => { // A job that ran past its lease, which another job then took: releasing it at diff --git a/packages/server/drizzle/0026_push_packs.sql b/packages/server/drizzle/0026_push_packs.sql new file mode 100644 index 00000000..70d99ec2 --- /dev/null +++ b/packages/server/drizzle/0026_push_packs.sql @@ -0,0 +1,13 @@ +CREATE TABLE `push_packs` ( + `session_id` text NOT NULL, + `pack_id` text NOT NULL, + `record_set` text NOT NULL, + `records` integer, + `bytes` integer, + `raw` integer, + `created_at` integer DEFAULT (unixepoch('subsec') * 1000) NOT NULL, + PRIMARY KEY(`session_id`, `pack_id`), + FOREIGN KEY (`session_id`) REFERENCES `push_sessions`(`id`) ON UPDATE no action ON DELETE cascade +); +--> statement-breakpoint +ALTER TABLE `push_sessions` ADD `commit_started` integer DEFAULT false NOT NULL; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0026_snapshot.json b/packages/server/drizzle/meta/0026_snapshot.json new file mode 100644 index 00000000..5a88d9a7 --- /dev/null +++ b/packages/server/drizzle/meta/0026_snapshot.json @@ -0,0 +1,3666 @@ +{ + "version": "6", + "dialect": "sqlite", + "id": "da1a4cfe-5e4b-4fb0-9240-470d050217f9", + "prevId": "42b4a563-0b06-417e-a83f-2216ffba5eb8", + "tables": { + "abuse_reports": { + "name": "abuse_reports", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "contact": { + "name": "contact", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reporter_id": { + "name": "reporter_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "resolved_by": { + "name": "resolved_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "resolved_at": { + "name": "resolved_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "abuse_reports_status_idx": { + "name": "abuse_reports_status_idx", + "columns": ["status", "created_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "account": { + "name": "account", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "provider_id": { + "name": "provider_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "access_token": { + "name": "access_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token": { + "name": "refresh_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "id_token": { + "name": "id_token", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "access_token_expires_at": { + "name": "access_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refresh_token_expires_at": { + "name": "refresh_token_expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "scope": { + "name": "scope", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "password": { + "name": "password", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "account_user_id_idx": { + "name": "account_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "account_user_id_user_id_fk": { + "name": "account_user_id_user_id_fk", + "tableFrom": "account", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "apikey": { + "name": "apikey", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "config_id": { + "name": "config_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'default'" + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "start": { + "name": "start", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reference_id": { + "name": "reference_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "key": { + "name": "key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "refill_interval": { + "name": "refill_interval", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refill_amount": { + "name": "refill_amount", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_refill_at": { + "name": "last_refill_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_enabled": { + "name": "rate_limit_enabled", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": true + }, + "rate_limit_time_window": { + "name": "rate_limit_time_window", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 86400000 + }, + "rate_limit_max": { + "name": "rate_limit_max", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 10 + }, + "request_count": { + "name": "request_count", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": 0 + }, + "remaining": { + "name": "remaining", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_request": { + "name": "last_request", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "permissions": { + "name": "permissions", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "apikey_key_idx": { + "name": "apikey_key_idx", + "columns": ["key"], + "isUnique": false + }, + "apikey_reference_id_idx": { + "name": "apikey_reference_id_idx", + "columns": ["reference_id"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_collections": { + "name": "ark_collections", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "ark_id": { + "name": "ark_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "custom_url": { + "name": "custom_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_collections_ark_id_unique": { + "name": "ark_collections_ark_id_unique", + "columns": ["ark_id"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_collections_collection_id_collections_id_fk": { + "name": "ark_collections_collection_id_collections_id_fk", + "tableFrom": "ark_collections", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_record_types": { + "name": "ark_record_types", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_type": { + "name": "record_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "redirect_url_field": { + "name": "redirect_url_field", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": { + "ark_record_types_collection_id_collections_id_fk": { + "name": "ark_record_types_collection_id_collections_id_fk", + "tableFrom": "ark_record_types", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "ark_record_types_collection_id_record_type_pk": { + "columns": ["collection_id", "record_type"], + "name": "ark_record_types_collection_id_record_type_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ark_shoulders": { + "name": "ark_shoulders", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "shoulder": { + "name": "shoulder", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ark_shoulders_shoulder_unique": { + "name": "ark_shoulders_shoulder_unique", + "columns": ["shoulder"], + "isUnique": true + } + }, + "foreignKeys": { + "ark_shoulders_organization_id_organization_id_fk": { + "name": "ark_shoulders_organization_id_organization_id_fk", + "tableFrom": "ark_shoulders", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "cleanup_runs": { + "name": "cleanup_runs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "step": { + "name": "step", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "trigger": { + "name": "trigger", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "dry_run": { + "name": "dry_run", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "requested_by": { + "name": "requested_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "mark_run_id": { + "name": "mark_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "lease": { + "name": "lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stats": { + "name": "stats", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "started_at": { + "name": "started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finished_at": { + "name": "finished_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "cleanup_runs_created_idx": { + "name": "cleanup_runs_created_idx", + "columns": ["created_at"], + "isUnique": false + }, + "cleanup_runs_step_idx": { + "name": "cleanup_runs_step_idx", + "columns": ["step", "status"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_tombstones": { + "name": "collection_tombstones", + "columns": { + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "versions": { + "name": "versions", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "deleted_by": { + "name": "deleted_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_layout": { + "name": "record_layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'today'" + }, + "records_dropped_at": { + "name": "records_dropped_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collection_webhooks": { + "name": "collection_webhooks", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "url": { + "name": "url", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bump_filter": { + "name": "bump_filter", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "secret": { + "name": "secret", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "enabled": { + "name": "enabled", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": true + }, + "created_by": { + "name": "created_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_delivery_at": { + "name": "last_delivery_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "collection_webhooks_collection_idx": { + "name": "collection_webhooks_collection_idx", + "columns": ["collection_id"], + "isUnique": false + } + }, + "foreignKeys": { + "collection_webhooks_collection_id_collections_id_fk": { + "name": "collection_webhooks_collection_id_collections_id_fk", + "tableFrom": "collection_webhooks", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "collections": { + "name": "collections", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public": { + "name": "public", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "head_version_id": { + "name": "head_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_salt": { + "name": "private_salt", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_files_root": { + "name": "public_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "summary": { + "name": "summary", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "version_count": { + "name": "version_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "history_bytes": { + "name": "history_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "last_push_at": { + "name": "last_push_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_started_at": { + "name": "reconcile_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_state": { + "name": "reconcile_state", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_seq": { + "name": "reconciled_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "reconciled_files_root": { + "name": "reconciled_files_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_layout": { + "name": "record_layout", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'today'" + }, + "deleted_at": { + "name": "deleted_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "collections_org_slug_uq": { + "name": "collections_org_slug_uq", + "columns": ["organization_id", "slug"], + "isUnique": true + }, + "collections_public_updated_idx": { + "name": "collections_public_updated_idx", + "columns": ["public", "updated_at"], + "isUnique": false + } + }, + "foreignKeys": { + "collections_organization_id_organization_id_fk": { + "name": "collections_organization_id_organization_id_fk", + "tableFrom": "collections", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "commit_units": { + "name": "commit_units", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "plan_id": { + "name": "plan_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ord": { + "name": "ord", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "after": { + "name": "after", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "through": { + "name": "through", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "gap": { + "name": "gap", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "slices_key": { + "name": "slices_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "output_key": { + "name": "output_key", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "queued_at": { + "name": "queued_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "commit_units_plan_idx": { + "name": "commit_units_plan_idx", + "columns": ["plan_id", "set", "type", "ord"], + "isUnique": false + }, + "commit_units_status_idx": { + "name": "commit_units_status_idx", + "columns": ["session_id", "status", "queued_at"], + "isUnique": false + } + }, + "foreignKeys": { + "commit_units_session_id_push_sessions_id_fk": { + "name": "commit_units_session_id_push_sessions_id_fk", + "tableFrom": "commit_units", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "denylist": { + "name": "denylist", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "reason": { + "name": "reason", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "file_uploads": { + "name": "file_uploads", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "multipart_upload_id": { + "name": "multipart_upload_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "file_uploads_hash_idx": { + "name": "file_uploads_hash_idx", + "columns": ["hash"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "files": { + "name": "files", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "size": { + "name": "size", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "mime_type": { + "name": "mime_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "storage_key": { + "name": "storage_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "forks": { + "name": "forks", + "columns": { + "child_collection_id": { + "name": "child_collection_id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "parent_collection_id": { + "name": "parent_collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parent_seq": { + "name": "parent_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'public'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "forks_child_collection_id_collections_id_fk": { + "name": "forks_child_collection_id_collections_id_fk", + "tableFrom": "forks", + "tableTo": "collections", + "columnsFrom": ["child_collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "instance_settings": { + "name": "instance_settings", + "columns": { + "key": { + "name": "key", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "invitation": { + "name": "invitation", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "inviter_id": { + "name": "inviter_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "invitation_organization_id_idx": { + "name": "invitation_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + } + }, + "foreignKeys": { + "invitation_organization_id_organization_id_fk": { + "name": "invitation_organization_id_organization_id_fk", + "tableFrom": "invitation", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "invitation_inviter_id_user_id_fk": { + "name": "invitation_inviter_id_user_id_fk", + "tableFrom": "invitation", + "tableTo": "user", + "columnsFrom": ["inviter_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "jobs": { + "name": "jobs", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "type": { + "name": "type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'queued'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "run_at": { + "name": "run_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "locked_until": { + "name": "locked_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "jobs_ready_idx": { + "name": "jobs_ready_idx", + "columns": ["status", "run_at"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "member": { + "name": "member", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'member'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "member_organization_id_idx": { + "name": "member_organization_id_idx", + "columns": ["organization_id"], + "isUnique": false + }, + "member_user_id_idx": { + "name": "member_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "member_organization_id_organization_id_fk": { + "name": "member_organization_id_organization_id_fk", + "tableFrom": "member", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "member_user_id_user_id_fk": { + "name": "member_user_id_user_id_fk", + "tableFrom": "member", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "organization": { + "name": "organization", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "slug": { + "name": "slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "logo": { + "name": "logo", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "metadata": { + "name": "metadata", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bio": { + "name": "bio", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "website": { + "name": "website", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "avatar_url": { + "name": "avatar_url", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ark_naan": { + "name": "ark_naan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kf_org_id": { + "name": "kf_org_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "is_default": { + "name": "is_default", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false, + "default": false + } + }, + "indexes": { + "organization_slug_unique": { + "name": "organization_slug_unique", + "columns": ["slug"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "placements": { + "name": "placements", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "location_id": { + "name": "location_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "role": { + "name": "role", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "sets": { + "name": "sets", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'active'" + }, + "synced_seq": { + "name": "synced_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "cursor": { + "name": "cursor", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "lease_until": { + "name": "lease_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "placements_one_primary_uq": { + "name": "placements_one_primary_uq", + "columns": ["collection_id"], + "isUnique": true, + "where": "\"placements\".\"role\" = 'primary' AND \"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_collection_location_uq": { + "name": "placements_collection_location_uq", + "columns": ["collection_id", "location_id"], + "isUnique": true, + "where": "\"placements\".\"collection_id\" IS NOT NULL" + }, + "placements_org_location_uq": { + "name": "placements_org_location_uq", + "columns": ["organization_id", "location_id"], + "isUnique": true, + "where": "\"placements\".\"organization_id\" IS NOT NULL" + }, + "placements_location_idx": { + "name": "placements_location_idx", + "columns": ["location_id"], + "isUnique": false + } + }, + "foreignKeys": { + "placements_collection_id_collections_id_fk": { + "name": "placements_collection_id_collections_id_fk", + "tableFrom": "placements", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_organization_id_organization_id_fk": { + "name": "placements_organization_id_organization_id_fk", + "tableFrom": "placements", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + }, + "placements_location_id_storage_locations_id_fk": { + "name": "placements_location_id_storage_locations_id_fk", + "tableFrom": "placements", + "tableTo": "storage_locations", + "columnsFrom": ["location_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_packs": { + "name": "push_packs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "pack_id": { + "name": "pack_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "record_set": { + "name": "record_set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "records": { + "name": "records", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "raw": { + "name": "raw", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "push_packs_session_id_push_sessions_id_fk": { + "name": "push_packs_session_id_push_sessions_id_fk", + "tableFrom": "push_packs", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_packs_session_id_pack_id_pk": { + "columns": ["session_id", "pack_id"], + "name": "push_packs_session_id_pack_id_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_runs": { + "name": "push_runs", + "columns": { + "session_id": { + "name": "session_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "object_key": { + "name": "object_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "first_key": { + "name": "first_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_key": { + "name": "last_key", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "merging_into": { + "name": "merging_into", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "claimed_at": { + "name": "claimed_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "push_runs_tier_idx": { + "name": "push_runs_tier_idx", + "columns": ["session_id", "tier", "merging_into"], + "isUnique": false + } + }, + "foreignKeys": { + "push_runs_session_id_push_sessions_id_fk": { + "name": "push_runs_session_id_push_sessions_id_fk", + "tableFrom": "push_runs", + "tableTo": "push_sessions", + "columnsFrom": ["session_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "push_runs_session_id_seq_pk": { + "columns": ["session_id", "seq"], + "name": "push_runs_session_id_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "push_sessions": { + "name": "push_sessions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_version_id": { + "name": "base_version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "strip_unknown_fields": { + "name": "strip_unknown_fields", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "records_received": { + "name": "records_received", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "runs": { + "name": "runs", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'open'" + }, + "result": { + "name": "result", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "finalize_started_at": { + "name": "finalize_started_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_started": { + "name": "commit_started", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "cleaned_at": { + "name": "cleaned_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "commit_phase": { + "name": "commit_phase", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "progress_at": { + "name": "progress_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "stalls": { + "name": "stalls", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "commit_plan": { + "name": "commit_plan", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "assembly_lease": { + "name": "assembly_lease", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "push_sessions_collection_idx": { + "name": "push_sessions_collection_idx", + "columns": ["collection_id"], + "isUnique": false + }, + "push_sessions_expires_idx": { + "name": "push_sessions_expires_idx", + "columns": ["status", "expires_at"], + "isUnique": false + }, + "push_sessions_user_idx": { + "name": "push_sessions_user_idx", + "columns": ["user_id", "status"], + "isUnique": false + } + }, + "foreignKeys": { + "push_sessions_collection_id_collections_id_fk": { + "name": "push_sessions_collection_id_collections_id_fk", + "tableFrom": "push_sessions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compaction_parts": { + "name": "ref_compaction_parts", + "columns": { + "compaction_id": { + "name": "compaction_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "part": { + "name": "part", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_compaction_parts_compaction_id_part_pk": { + "columns": ["compaction_id", "part"], + "name": "ref_compaction_parts_compaction_id_part_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_compactions": { + "name": "ref_compactions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "input_runs": { + "name": "input_runs", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "output_run": { + "name": "output_run", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "parts": { + "name": "parts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "prefix_length": { + "name": "prefix_length", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'running'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_index_units": { + "name": "ref_index_units", + "columns": { + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "unit": { + "name": "unit", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "events": { + "name": "events", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": { + "ref_index_units_version_id_unit_pk": { + "columns": ["version_id", "unit"], + "name": "ref_index_units_version_id_unit_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "ref_segments": { + "name": "ref_segments", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "run_id": { + "name": "run_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "tier": { + "name": "tier", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "first_hash": { + "name": "first_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "last_hash": { + "name": "last_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "count": { + "name": "count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "bytes": { + "name": "bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "state": { + "name": "state", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'live'" + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "ref_segments_range_idx": { + "name": "ref_segments_range_idx", + "columns": ["state", "first_hash", "last_hash"], + "isUnique": false + }, + "ref_segments_run_idx": { + "name": "ref_segments_run_idx", + "columns": ["run_id"], + "isUnique": false + }, + "ref_segments_tier_idx": { + "name": "ref_segments_tier_idx", + "columns": ["tier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_labels": { + "name": "schema_labels", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": { + "schema_labels_label_idx": { + "name": "schema_labels_label_idx", + "columns": ["label"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "schema_labels_schema_hash_label_pk": { + "columns": ["schema_hash", "label"], + "name": "schema_labels_schema_hash_label_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schema_usage": { + "name": "schema_usage", + "columns": { + "schema_hash": { + "name": "schema_hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "type_slug": { + "name": "type_slug", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "set": { + "name": "set", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "from_seq": { + "name": "from_seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "to_seq": { + "name": "to_seq", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "schema_usage_hash_idx": { + "name": "schema_usage_hash_idx", + "columns": ["schema_hash"], + "isUnique": false + } + }, + "foreignKeys": { + "schema_usage_collection_id_collections_id_fk": { + "name": "schema_usage_collection_id_collections_id_fk", + "tableFrom": "schema_usage", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": { + "schema_usage_collection_id_type_slug_set_from_seq_pk": { + "columns": ["collection_id", "type_slug", "set", "from_seq"], + "name": "schema_usage_collection_id_type_slug_set_from_seq_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "schemas": { + "name": "schemas", + "columns": { + "hash": { + "name": "hash", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "session": { + "name": "session", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "token": { + "name": "token", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "ip_address": { + "name": "ip_address", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_agent": { + "name": "user_agent", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "user_id": { + "name": "user_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "active_organization_id": { + "name": "active_organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "session_token_unique": { + "name": "session_token_unique", + "columns": ["token"], + "isUnique": true + }, + "session_user_id_idx": { + "name": "session_user_id_idx", + "columns": ["user_id"], + "isUnique": false + } + }, + "foreignKeys": { + "session_user_id_user_id_fk": { + "name": "session_user_id_user_id_fk", + "tableFrom": "session", + "tableTo": "user", + "columnsFrom": ["user_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_fence": { + "name": "storage_fence", + "columns": { + "id": { + "name": "id", + "type": "integer", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "epoch": { + "name": "epoch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "window_until": { + "name": "window_until", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "window_run_id": { + "name": "window_run_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "storage_locations": { + "name": "storage_locations", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "organization_id": { + "name": "organization_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "kind": { + "name": "kind", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "endpoint": { + "name": "endpoint", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "region": { + "name": "region", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bucket": { + "name": "bucket", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "prefix": { + "name": "prefix", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "''" + }, + "credentials": { + "name": "credentials", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'unverified'" + }, + "last_error": { + "name": "last_error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "verified_at": { + "name": "verified_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "checked_at": { + "name": "checked_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + } + }, + "indexes": {}, + "foreignKeys": { + "storage_locations_organization_id_organization_id_fk": { + "name": "storage_locations_organization_id_organization_id_fk", + "tableFrom": "storage_locations", + "tableTo": "organization", + "columnsFrom": ["organization_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "usage_rollups": { + "name": "usage_rollups", + "columns": { + "day": { + "name": "day", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "account_id": { + "name": "account_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "metric": { + "name": "metric", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "amount": { + "name": "amount", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "usage_rollups_account_idx": { + "name": "usage_rollups_account_idx", + "columns": ["account_id", "day"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": { + "usage_rollups_day_account_id_collection_id_metric_pk": { + "columns": ["day", "account_id", "collection_id", "metric"], + "name": "usage_rollups_day_account_id_collection_id_metric_pk" + } + }, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "user": { + "name": "user", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email": { + "name": "email", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "email_verified": { + "name": "email_verified", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "image": { + "name": "image", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "user_email_unique": { + "name": "user_email_unique", + "columns": ["email"], + "isUnique": true + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "verification": { + "name": "verification", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "identifier": { + "name": "identifier", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "value": { + "name": "value", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "expires_at": { + "name": "expires_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "updated_at": { + "name": "updated_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + } + }, + "indexes": { + "verification_identifier_idx": { + "name": "verification_identifier_idx", + "columns": ["identifier"], + "isUnique": false + } + }, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "versions": { + "name": "versions", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "seq": { + "name": "seq", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "major": { + "name": "major", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "minor": { + "name": "minor", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "patch": { + "name": "patch", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "hash": { + "name": "hash", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "base_semver": { + "name": "base_semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "message": { + "name": "message", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "pushed_by": { + "name": "pushed_by", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "app_id": { + "name": "app_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "actor_id": { + "name": "actor_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "signature": { + "name": "signature", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "record_count": { + "name": "record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_record_count": { + "name": "public_record_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "file_count": { + "name": "file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "total_bytes": { + "name": "total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_file_count": { + "name": "public_file_count", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "public_total_bytes": { + "name": "public_total_bytes", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "type_counts": { + "name": "type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "public_type_counts": { + "name": "public_type_counts", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "has_private": { + "name": "has_private", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "public_refs_root": { + "name": "public_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "private_refs_root": { + "name": "private_refs_root", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "refs_indexed": { + "name": "refs_indexed", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "ref_events": { + "name": "ref_events", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "ref_bytes": { + "name": "ref_bytes", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconciled_at": { + "name": "reconciled_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "reconcile_report": { + "name": "reconcile_report", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "changes": { + "name": "changes", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "published_at": { + "name": "published_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "push_session_id": { + "name": "push_session_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "versions_collection_seq_uq": { + "name": "versions_collection_seq_uq", + "columns": ["collection_id", "seq"], + "isUnique": true + }, + "versions_collection_semver_uq": { + "name": "versions_collection_semver_uq", + "columns": ["collection_id", "semver"], + "isUnique": true + }, + "versions_hash_idx": { + "name": "versions_hash_idx", + "columns": ["hash"], + "isUnique": false + }, + "versions_published_idx": { + "name": "versions_published_idx", + "columns": ["published_at"], + "isUnique": false + }, + "versions_push_session_idx": { + "name": "versions_push_session_idx", + "columns": ["push_session_id"], + "isUnique": false + } + }, + "foreignKeys": { + "versions_collection_id_collections_id_fk": { + "name": "versions_collection_id_collections_id_fk", + "tableFrom": "versions", + "tableTo": "collections", + "columnsFrom": ["collection_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + }, + "webhook_deliveries": { + "name": "webhook_deliveries", + "columns": { + "id": { + "name": "id", + "type": "text", + "primaryKey": true, + "notNull": true, + "autoincrement": false + }, + "webhook_id": { + "name": "webhook_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "collection_id": { + "name": "collection_id", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "version_id": { + "name": "version_id", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "semver": { + "name": "semver", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "bump_type": { + "name": "bump_type", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "event": { + "name": "event", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'version.created'" + }, + "payload": { + "name": "payload", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false + }, + "status": { + "name": "status", + "type": "text", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "'pending'" + }, + "attempts": { + "name": "attempts", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, + "response_code": { + "name": "response_code", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "error": { + "name": "error", + "type": "text", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "duration_ms": { + "name": "duration_ms", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "next_attempt_at": { + "name": "next_attempt_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + }, + "created_at": { + "name": "created_at", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": "(unixepoch('subsec') * 1000)" + }, + "delivered_at": { + "name": "delivered_at", + "type": "integer", + "primaryKey": false, + "notNull": false, + "autoincrement": false + } + }, + "indexes": { + "webhook_deliveries_webhook_idx": { + "name": "webhook_deliveries_webhook_idx", + "columns": ["webhook_id", "created_at"], + "isUnique": false + }, + "webhook_deliveries_pending_idx": { + "name": "webhook_deliveries_pending_idx", + "columns": ["status", "next_attempt_at"], + "isUnique": false + } + }, + "foreignKeys": { + "webhook_deliveries_webhook_id_collection_webhooks_id_fk": { + "name": "webhook_deliveries_webhook_id_collection_webhooks_id_fk", + "tableFrom": "webhook_deliveries", + "tableTo": "collection_webhooks", + "columnsFrom": ["webhook_id"], + "columnsTo": ["id"], + "onDelete": "cascade", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "checkConstraints": {} + } + }, + "views": {}, + "enums": {}, + "_meta": { + "schemas": {}, + "tables": {}, + "columns": {} + }, + "internal": { + "indexes": {} + } +} diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index f8fd9100..225cdc62 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -183,6 +183,13 @@ "when": 1792506002064, "tag": "0025_record_layout", "breakpoints": true + }, + { + "idx": 26, + "version": "6", + "when": 1792518549719, + "tag": "0026_push_packs", + "breakpoints": true } ] } diff --git a/packages/server/src/api/push.ts b/packages/server/src/api/push.ts index 3e654f85..adceb665 100644 --- a/packages/server/src/api/push.ts +++ b/packages/server/src/api/push.ts @@ -139,6 +139,7 @@ async function commitRoute(c: Context, session: SessionRow) { // uploads (still in storage) and a fresh plan. await transition(ports, session.id, 'failed', 'committing', { finalizeStartedAt: new Date(), + commitStarted: true, error: null, commitPlan: null, commitPhase: null, @@ -151,6 +152,7 @@ async function commitRoute(c: Context, session: SessionRow) { : // The error of a commit refused for missing files is cleared on the next try. await transition(ports, session.id, 'open', 'committing', { finalizeStartedAt: new Date(), + commitStarted: true, error: null, ...rebased, }) diff --git a/packages/server/src/cleanup/internal.ts b/packages/server/src/cleanup/internal.ts index 067749f6..eff5b6da 100644 --- a/packages/server/src/cleanup/internal.ts +++ b/packages/server/src/cleanup/internal.ts @@ -19,6 +19,7 @@ import { and, eq, inArray, isNull, lt, sql } from 'drizzle-orm' import { chunks } from '../db/chunks.js' import * as schema from '../db/schema.js' import type { Ports } from '../ports.js' +import { settleSessionPacks } from '../push/packs.js' import { cleanupConfig, count, emptyStats, problem } from './config.js' const TERMINAL: schema.SessionStatus[] = ['committed', 'failed', 'expired'] @@ -67,7 +68,11 @@ export async function cleanInternal( // Push sessions. const sessions = await db - .select({ id: schema.pushSessions.id }) + .select({ + id: schema.pushSessions.id, + collectionId: schema.pushSessions.collectionId, + commitStarted: schema.pushSessions.commitStarted, + }) .from(schema.pushSessions) .where( and( @@ -89,7 +94,12 @@ export async function cleanInternal( count(stats, 'sessions', gone.bytes, gone.objects) // A session bigger than what's left of the budget finishes next time. if (!gone.complete) more = true - if (dryRun || !gone.complete) continue + if (!gone.complete) continue + // Layout F: packs the uploads wrote into the collection (push/packs.ts). + const packs = await settleSessionPacks(ports, s, dryRun) + budget -= packs.objects + if (packs.objects) count(stats, 'upload packs', packs.bytes, packs.objects) + if (dryRun) continue const [runs, units] = await Promise.all([ db .delete(schema.pushRuns) diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index 095f757a..39d87f99 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -607,6 +607,12 @@ export const pushSessions = sqliteTable( result: json>('result'), error: json<{ statusCode: number; error: string; [k: string]: unknown }>('error'), finalizeStartedAt: ts('finalize_started_at'), + /** + * A commit of this session has started, ever (unlike `finalizeStartedAt`, never + * cleared): its index may name records in the packs its uploads wrote + * (push/packs.ts), so cleanup leaves those to reclaim instead of deleting them. + */ + commitStarted: bool('commit_started').notNull().default(false), /** When storage cleanup deleted the session's objects (sessions//) and run rows. */ cleanedAt: ts('cleaned_at'), /** @@ -662,6 +668,30 @@ export const pushRuns = sqliteTable( ], ) +/** + * A pack an upload wrote straight into a layout F collection (push/delta.ts): its + * records' places go in the session's runs, so the commit indexes them without + * moving them. A row is written before its pack; the stats once the pack is + * written (null: it never was). The commit lists the packs in the collection's + * manifest or registry; storage cleanup deletes those of a push that never did. + */ +export const pushPacks = sqliteTable( + 'push_packs', + { + sessionId: text('session_id') + .notNull() + .references(() => pushSessions.id, { onDelete: 'cascade' }), + packId: text('pack_id').notNull(), + set: text('record_set', { enum: ['public', 'private'] }).notNull(), + /** PackInfo, once written. */ + records: integer('records'), + bytes: integer('bytes'), + raw: integer('raw'), + createdAt: createdAt(), + }, + (t) => [primaryKey({ columns: [t.sessionId, t.packId] })], +) + export type CommitUnitStatus = 'pending' | 'done' | 'failed' | 'superseded' /** diff --git a/packages/server/src/push/changes.ts b/packages/server/src/push/changes.ts index 06c6971a..3b52863b 100644 --- a/packages/server/src/push/changes.ts +++ b/packages/server/src/push/changes.ts @@ -9,6 +9,7 @@ import { keys, outOfLineHash, outOfLinePointer, + type Placed, type RecordEntry, type Repo, type SetName, @@ -64,12 +65,13 @@ export async function takeStaged( export const isPrivateSchema = (s: Record) => s.private === true -export const toRecordEntry = (e: RunEntry): RecordEntry => ({ - key: e.k, - hash: e.h!, - size: e.s!, - body: e.b!, -}) +/** A run entry as the commit takes it: with its body, or its place in an upload's pack. */ +export function toRecordEntry(e: RunEntry): RecordEntry { + if (e.l === undefined) return { key: e.k, hash: e.h!, size: e.s!, body: e.b! } + const placed: Placed = { key: e.k, hash: e.h!, size: e.s!, packed: e.l } + if (e.r) placed.fileRefs = e.r + return placed +} /** Which sets of a type the base has a tree in. */ export interface BaseSets { diff --git a/packages/server/src/push/delta.ts b/packages/server/src/push/delta.ts index ae7107be..153b3b98 100644 --- a/packages/server/src/push/delta.ts +++ b/packages/server/src/push/delta.ts @@ -12,6 +12,7 @@ import { checkTypeSlug, compileSchema, emptySet, + fileRefs, InputRuleError, OUT_OF_LINE_BYTES, parseRecordLine, @@ -30,6 +31,7 @@ import type { Ports } from '../ports.js' import { type BaseVersion, commitVersion, type TypeInput } from '../versions/commit.js' import { deltaChanges, isPrivateSchema, stageOutOfLine } from './changes.js' import type { SessionCommitResult } from './outcome.js' +import { packEntries, packStats } from './packs.js' import { planParallel, SerialTooLargeError } from './parallel.js' import { type RunEntry, type RunIndex, writeRun } from './runs.js' import { @@ -80,7 +82,8 @@ export async function prepareRecords( sessionId: string, inputs: SessionInputs, input: string | AsyncIterable, - opts: { stripUnknownFields: boolean }, + /** `packed`: the records go into packs (layout F), so large ones aren't staged apart. */ + opts: { stripUnknownFields: boolean; packed?: boolean }, ): Promise< { entries: RunEntry[] } | { errors: LineError[]; total: number } | { tooManyLines: true } > { @@ -161,10 +164,13 @@ export async function prepareRecords( const hash = sha256Hex(canonical) const size = utf8ByteLength(canonical) const body = - size > OUT_OF_LINE_BYTES + size > OUT_OF_LINE_BYTES && !opts.packed ? await stageOutOfLine(internal, sessionId, hash, canonical) : canonical const isPrivate = isPrivateSchema(typeSchema) || rec.private === true + // A packed record's body leaves its entry (push/packs.ts): the commit counts its + // file references from these. + const refs = opts.packed && canonical.includes('"$file"') ? fileRefs(data) : [] const entry: RunEntry = { t: rec.type, k: rec.id, @@ -172,6 +178,7 @@ export async function prepareRecords( s: size, b: body, ...(isPrivate ? { p: true } : {}), + ...(refs.length ? { r: refs } : {}), } entries.push(entry) } @@ -184,8 +191,11 @@ export async function ingestRecords( input: string | AsyncIterable, ): Promise { const inputs = await loadInputs(ports, session.id) + // Layout F: the records go straight into the collection's packs. + const records = (await ports.stores.forCollection(session.collectionId)).records const prepared = await prepareRecords(ports, session.id, inputs, input, { stripUnknownFields: session.stripUnknownFields, + packed: records !== null, }) if ('tooManyLines' in prepared) { return { ok: false, status: 413, error: `At most ${MAX_BATCH_LINES} records per batch` } @@ -202,8 +212,16 @@ export async function ingestRecords( if (prepared.entries.length === 0) return { ok: false, status: 400, error: 'Empty batch' } const seq = await nextRunSeq(ports, session.id) if (seq === null) return { ok: false, status: 409, error: 'Session is not open' } + const packs = records ? await packEntries(ports, session.id, records, prepared.entries) : [] const index = await writeRun(ports.stores.internal, session.id, seq, prepared.entries) - await recordRun(ports, session.id, 'records', index, { records: prepared.entries.length }) + await recordRun( + ports, + session.id, + 'records', + index, + { records: prepared.entries.length }, + packStats(ports, session.id, packs), + ) return { ok: true, received: prepared.entries.length } } diff --git a/packages/server/src/push/packs.ts b/packages/server/src/push/packs.ts new file mode 100644 index 00000000..37e507b2 --- /dev/null +++ b/packages/server/src/push/packs.ts @@ -0,0 +1,157 @@ +/** + * Uploads into a layout F collection write their records straight into packs of + * the collection's, where they stay. Each record's run entry then carries its + * place (`l`) instead of its body, so compaction moves entries of about 150 bytes + * instead of whole records (a 30M-record Crossref session spent 10.5 hours + * compacting 206 GB, 2026-10-19), and commit units index the records without + * reading or writing them again. + * + * A pack is outside the collection's index and manifest until a commit names it + * (`adoptSessionPacks`, just before the version publishes). Each is recorded in + * `push_packs` before it's written, so storage cleanup can delete the packs of a + * push that never published (`dropSessionPacks`). + */ +import { + packIdsFor, + type PackInfo, + type RecordSet, + type RecordSets, + type Repo, +} from '@underlay/protocol' +import { and, eq, isNotNull } from 'drizzle-orm' + +import * as schema from '../db/schema.js' +import type { Ports } from '../ports.js' +import { type RunEntry, runKey } from './runs.js' + +const SETS = ['public', 'private'] as const + +/** A pack an upload wrote, for its row's stats. */ +export interface UploadPack { + set: RecordSet + pack: PackInfo +} + +/** + * Write a batch's records into packs, a set each, and give their entries places + * instead of bodies. Within the batch the last entry of a key wins (writeRun), + * so only those are written. + */ +export async function packEntries( + ports: Ports, + sessionId: string, + records: RecordSets, + entries: RunEntry[], +): Promise { + const last = new Map() + for (const e of entries) last.set(runKey(e), e) + const out: UploadPack[] = [] + for (const set of SETS) { + // An upload marks an entry private exactly when it goes to the private set (deltaChanges). + const group = [...last.values()].filter((e) => (e.p === true) === (set === 'private')) + if (!group.length) continue + const recs = group.map((e) => ({ id: e.k, hash: e.h!, line: e.b! })) + const ids = packIdsFor(recs) + // Recorded before they're written: an upload that dies leaves no pack cleanup can't find. + await ports.db + .insert(schema.pushPacks) + .values(ids.map((packId) => ({ sessionId, packId, set }))) + const { places, packs } = await records[set].writePacks!(recs, ids) + group.forEach((e, i) => { + e.l = places[i]! + delete e.b + }) + out.push(...packs.map((pack) => ({ set, pack }))) + } + return out +} + +/** The statements that record written packs' stats (run with the upload's run row). */ +export const packStats = (ports: Ports, sessionId: string, packs: UploadPack[]) => + packs.map(({ pack }) => + ports.db + .update(schema.pushPacks) + .set({ records: pack.records, bytes: pack.bytes, raw: pack.raw }) + .where(and(eq(schema.pushPacks.sessionId, sessionId), eq(schema.pushPacks.packId, pack.id))), + ) + +/** + * List a session's packs in the collection's manifest or registry. Called once the + * commit's index names their records, before its version publishes; a retry lists + * nothing twice (KeyedLayout.adoptPacks). + */ +export async function adoptSessionPacks(ports: Ports, repo: Repo, sessionId: string) { + if (!repo.records) return + const rows = await ports.db + .select() + .from(schema.pushPacks) + .where(and(eq(schema.pushPacks.sessionId, sessionId), isNotNull(schema.pushPacks.records))) + for (const set of SETS) { + const packs: PackInfo[] = rows + .filter((r) => r.set === set) + .map((r) => ({ id: r.packId, tier: 0, records: r.records!, bytes: r.bytes!, raw: r.raw! })) + if (packs.length) await repo.records[set].adoptPacks!(sessionId, packs) + } +} + +/** + * At storage cleanup, settle a finished session's packs, then drop its rows: + * - a version names the session: its commit listed them, nothing to do; + * - no commit of it ever started: nothing indexes their records, so they're deleted; + * - a commit started and never published: its index may name their records (one the + * collection already held, say, whose newest place is now such a pack), so they're + * listed instead, and reclaim drops what no version uses. + * Returns the objects and bytes deleted (a pack is two objects), counting packs that + * were written. A collection whose repository won't open (deleted) is left to the + * sweep, which drops its files. + */ +export async function settleSessionPacks( + ports: Ports, + session: { id: string; collectionId: string; commitStarted: boolean }, + dryRun: boolean, +): Promise<{ objects: number; bytes: number }> { + const gone = { objects: 0, bytes: 0 } + const rows = await ports.db + .select() + .from(schema.pushPacks) + .where(eq(schema.pushPacks.sessionId, session.id)) + if (!rows.length) return gone + const [published] = await ports.db + .select({ id: schema.versions.id }) + .from(schema.versions) + .where(eq(schema.versions.pushSessionId, session.id)) + .limit(1) + let repo: Repo | null = null + try { + repo = await ports.stores.forCollection(session.collectionId) + } catch { + repo = null + } + if (!published && repo?.records) { + if (session.commitStarted) { + if (!dryRun) await adoptSessionPacks(ports, repo, session.id) + } else { + for (const set of SETS) { + const mine = rows.filter((r) => r.set === set) + if (!mine.length) continue + // Refused if a commit listed them after all (KeyedLayout.dropPacks). + const dropped = + dryRun || + (await repo.records[set].dropPacks!( + session.id, + mine.map((r) => r.packId), + )) + if (!dropped) continue + // A row without stats is a pack never written, or one whose upload died writing it. + for (const r of mine) + if (r.records !== null) { + gone.objects += 2 + gone.bytes += r.bytes ?? 0 + } + } + } + } + if (!dryRun) + await ports.db.delete(schema.pushPacks).where(eq(schema.pushPacks.sessionId, session.id)) + return gone +} diff --git a/packages/server/src/push/runs.ts b/packages/server/src/push/runs.ts index 657a6e22..889cc74a 100644 --- a/packages/server/src/push/runs.ts +++ b/packages/server/src/push/runs.ts @@ -62,8 +62,12 @@ export interface RunEntry { s?: number /** Pushed as private. */ p?: boolean - /** The canonical record (records runs). */ + /** The canonical record (records runs), unless it's in a pack (`l`). */ b?: string + /** Layout F: the record's place in a pack the upload wrote (push/packs.ts). */ + l?: string + /** With `l`: the files the record references, read at upload. */ + r?: string[] /** A delete (delta deletes runs). */ x?: boolean /** Precedence: the seq of the upload this entry came from (compacted runs only). */ diff --git a/packages/server/src/push/session.ts b/packages/server/src/push/session.ts index 118ac9f1..f9f286eb 100644 --- a/packages/server/src/push/session.ts +++ b/packages/server/src/push/session.ts @@ -44,6 +44,8 @@ export interface SessionInputs { export type SessionRow = typeof schema.pushSessions.$inferSelect +type BatchStatement = Parameters[0][number] + const inputsKey = (id: string) => `sessions/${id}/inputs.json` export async function createSession( @@ -113,6 +115,8 @@ export async function recordRun( kind: 'records' | 'deletes', index: RunIndex, counters: { records?: number }, + /** More statements for the same batch (packs.ts: the upload's packs' stats). */ + extra: BatchStatement[] = [], ): Promise { const count = runCount(index) await ports.db.batch([ @@ -133,6 +137,7 @@ export async function recordRun( recordsReceived: sql`${schema.pushSessions.recordsReceived} + ${counters.records ?? 0}`, }) .where(eq(schema.pushSessions.id, sessionId)), + ...extra, ]) await maybeCompact(ports, sessionId, index.tier ?? 0) } diff --git a/packages/server/src/versions/commit.ts b/packages/server/src/versions/commit.ts index fa2851ec..bd6b775a 100644 --- a/packages/server/src/versions/commit.ts +++ b/packages/server/src/versions/commit.ts @@ -44,6 +44,7 @@ import { eq } from 'drizzle-orm' import { FenceError, writeFence } from '../cleanup/fence.js' import * as schema from '../db/schema.js' import type { Ports } from '../ports.js' +import { adoptSessionPacks } from '../push/packs.js' import { publishCollectionInfo } from './collection-info.js' import { collectionFileSizes, fileSizes } from './file-refs.js' import { publishVersion, type SchemaUsageChange } from './publish.js' @@ -152,6 +153,9 @@ export async function commitVersion(ports: Ports, input: CommitInput): Promise { expect(h.bucket.objects.has(`repo/files/${sha('the pdf')}`)).toBe(true) }) - it('keeps a large record a push uploaded while the sweep ran', async () => { - const h = await harness() - const user = await h.member() - // Over OUT_OF_LINE_BYTES: stored as its own records/ object. - const big = { id: 'big', type: 'Doc', data: { title: 'x'.repeat(70_000) } } - await h.collection('old') - await push(h, user, 'old', [big]) - const recordKeys = () => keys(h, 'repo/records/') - expect(recordKeys()).toHaveLength(1) - await h.request('/api/collections/org/old', { method: 'DELETE', user }) - await pastGrace(h) - await run(h, 'mark') - - // A new push uploads the same record and stays open while the sweep runs. - const fresh = await h.collection('new') - const base = '/api/collections/org/new' - const open = await h.request(`${base}/push`, { - method: 'POST', - user, - json: { schemas: { Doc } }, - }) - const sid = ((await open.json()) as { session_id: string }).session_id - await h.request(`${base}/push/${sid}/records`, { method: 'POST', user, ndjson: [big] }) - // Staged in the session's area, not written to the repository yet. - expect(keys(h, `internal/sessions/${sid}/records/`)).toHaveLength(1) - ageAll(h) - const sweep = await run(h, 'sweep') - expect(sweep.stats!.deleted.records).toMatchObject({ objects: 1 }) - expect(recordKeys()).toEqual([]) + // Today's layout: on layout F an upload writes a large record into the collection's + // packs instead (push/packs.ts), and cleanup deletes those only for a push that never + // published (upload-packs.test.ts). + it.skipIf(process.env.UNDERLAY_TEST_LAYOUT === 'f')( + 'keeps a large record a push uploaded while the sweep ran', + async () => { + const h = await harness() + const user = await h.member() + // Over OUT_OF_LINE_BYTES: stored as its own records/ object. + const big = { id: 'big', type: 'Doc', data: { title: 'x'.repeat(70_000) } } + await h.collection('old') + await push(h, user, 'old', [big]) + const recordKeys = () => keys(h, 'repo/records/') + expect(recordKeys()).toHaveLength(1) + await h.request('/api/collections/org/old', { method: 'DELETE', user }) + await pastGrace(h) + await run(h, 'mark') - // The commit puts it back, under its fence. - const commit = await h.request(`${base}/push/${sid}/commit`, { method: 'POST', user }) - expect(commit.status).toBe(201) - expect(recordKeys()).toHaveLength(1) - await healthy(h, fresh.id) - }) + // A new push uploads the same record and stays open while the sweep runs. + const fresh = await h.collection('new') + const base = '/api/collections/org/new' + const open = await h.request(`${base}/push`, { + method: 'POST', + user, + json: { schemas: { Doc } }, + }) + const sid = ((await open.json()) as { session_id: string }).session_id + await h.request(`${base}/push/${sid}/records`, { method: 'POST', user, ndjson: [big] }) + // Staged in the session's area, not written to the repository yet. + expect(keys(h, `internal/sessions/${sid}/records/`)).toHaveLength(1) + ageAll(h) + const sweep = await run(h, 'sweep') + expect(sweep.stats!.deleted.records).toMatchObject({ objects: 1 }) + expect(recordKeys()).toEqual([]) + + // The commit puts it back, under its fence. + const commit = await h.request(`${base}/push/${sid}/commit`, { method: 'POST', user }) + expect(commit.status).toBe(201) + expect(recordKeys()).toHaveLength(1) + await healthy(h, fresh.id) + }, + ) it('gives the same result in many small jobs, shards and windows', async () => { const saved = { ...cleanupConfig } diff --git a/packages/server/test/stream-push.test.ts b/packages/server/test/stream-push.test.ts index e1565e63..54bea238 100644 --- a/packages/server/test/stream-push.test.ts +++ b/packages/server/test/stream-push.test.ts @@ -220,7 +220,10 @@ describe('one version from many uploads', () => { }) const status = async () => json(await h.request(`${path}/push/${sid}`, { user })) expect((await status()).progress).toMatchObject({ runs: 6, phase: null, units: null }) - expect((await status()).progress.run_bytes).toBeGreaterThan(6 * 10 * 512) + // Runs hold the records, or on layout F their places in the uploads' packs. + if (process.env.UNDERLAY_TEST_LAYOUT === 'f') + expect((await status()).progress.run_bytes).toBeLessThan(6 * 10 * 512) + else expect((await status()).progress.run_bytes).toBeGreaterThan(6 * 10 * 512) Object.assign(compactConfig, { commitRuns: 2 }) const res = await h.request(`${path}/push/${sid}/commit`, { method: 'POST', user }) // Too many runs for one commit: it compacts first, and answers 202. diff --git a/packages/server/test/upload-packs.test.ts b/packages/server/test/upload-packs.test.ts new file mode 100644 index 00000000..22f3063f --- /dev/null +++ b/packages/server/test/upload-packs.test.ts @@ -0,0 +1,365 @@ +/** + * Layout F uploads write their records straight into the collection's packs + * (push/packs.ts): runs carry places instead of bodies, a commit indexes the + * records without writing them again, its packs are listed once before the version + * publishes, and cleanup deletes the packs of a push that never did. + */ +import { createHash } from 'node:crypto' + +import { countTree, iterate, type RecordSet, RepoSource, recordTree } from '@underlay/protocol' +import { eq } from 'drizzle-orm' +import { afterAll, afterEach, describe, expect, it } from 'vitest' + +import { cleanupConfig, getRun } from '../src/cleanup/config.js' +import { startRun } from '../src/cleanup/runs.js' +import * as schema from '../src/db/schema.js' +import { compactConfig } from '../src/push/compact.js' +import { parallelConfig } from '../src/push/parallel.js' +import { mergeRuns, type RunEntry } from '../src/push/runs.js' +import { sessionRuns } from '../src/push/session.js' +import { cleanup, type Harness, harness } from './harness.js' + +afterAll(cleanup) + +const defaults = { parallel: { ...parallelConfig }, compact: { ...compactConfig } } +afterEach(() => { + Object.assign(parallelConfig, defaults.parallel) + Object.assign(compactConfig, defaults.compact) +}) + +const sha = (s: string) => createHash('sha256').update(s).digest('hex') +const Doc = { type: 'object', properties: { title: { type: 'string' }, pdf: { type: 'object' } } } +const doc = (id: string, title: string, extra: Record = {}) => ({ + id, + type: 'Doc', + data: { title, ...extra }, +}) +const id = (i: number) => `d${String(i).padStart(5, '0')}` +const DAY = 24 * 60 * 60 * 1000 + +async function json(res: Response) { + return (await res.json()) as any +} + +async function open(h: Harness, user: string, slug: string) { + const res = await h.request(`/api/collections/org/${slug}/push`, { + method: 'POST', + user, + json: { schemas: { Doc } }, + }) + expect(res.status).toBe(200) + return (await json(res)).session_id as string +} + +async function upload(h: Harness, user: string, slug: string, sid: string, batch: object[]) { + const r = await h.request(`/api/collections/org/${slug}/push/${sid}/records`, { + method: 'POST', + user, + ndjson: batch, + }) + expect(r.status).toBe(200) +} + +async function commit(h: Harness, user: string, slug: string, sid: string) { + const res = await h.request(`/api/collections/org/${slug}/push/${sid}/commit`, { + method: 'POST', + user, + }) + await h.drain() + return res.status +} + +/** Pack files of a collection's set in the bucket. */ +const packKeys = (h: Harness, collectionId: string, set: RecordSet = 'public') => + [...h.bucket.objects.keys()].filter((k) => + k.startsWith(`repo/runs/${collectionId}/${set}/packs/`), + ) + +/** Every record of the head version: `set id title`. */ +async function headRecords(h: Harness, collectionId: string) { + const [c] = await h.ports.db + .select() + .from(schema.collections) + .where(eq(schema.collections.id, collectionId)) + const [v] = await h.ports.db + .select() + .from(schema.versions) + .where(eq(schema.versions.id, c!.headVersionId!)) + const repo = await h.ports.stores.forCollection(collectionId) + const root = await repo.root(v!.hash) + const out: string[] = [] + const sets: [RecordSet, string | null][] = [['public', root.public.types.Doc?.root ?? null]] + if (root.private) + sets.push(['private', (await repo.privateSet(root.private)).types.Doc?.root ?? null]) + for (const [set, tree] of sets) + for await (const e of iterate(new RepoSource(recordTree, repo, { set }), tree, { + payloads: true, + })) { + const body = JSON.parse(e.body!) as { data: { title: string } } + out.push(`${set} ${e.key} ${body.data.title}`) + } + return { v: v!, out } +} + +async function sessionEntries(h: Harness, sid: string) { + const out: RunEntry[] = [] + for await (const e of mergeRuns(h.ports.stores.internal, sid, await sessionRuns(h.ports, sid))) + out.push(e) + return out +} + +describe.runIf(process.env.UNDERLAY_TEST_LAYOUT === 'f')('uploads into layout F', () => { + it('write records into packs once, and runs carry places, not bodies', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('a', { recordLayout: 'f' }) + const sid = await open(h, user, 'a') + const title = 't'.repeat(2000) + // The last of a key in a batch wins: only it goes in a pack. + await upload(h, user, 'a', sid, [ + ...Array.from({ length: 300 }, (_, i) => doc(id(i), `${title}${i}`)), + doc(id(0), 'replaced'), + ]) + const packs = packKeys(h, c.id) + // A pack is its blocks and their directory. + expect(packs).toHaveLength(2) + const entries = await sessionEntries(h, sid) + expect(entries).toHaveLength(300) + expect(entries.every((e) => e.b === undefined && typeof e.l === 'string')).toBe(true) + // The run holds places: a small part of the records' bytes. + const [run] = await h.ports.db + .select() + .from(schema.pushRuns) + .where(eq(schema.pushRuns.sessionId, sid)) + expect(run!.bytes).toBeLessThan((300 * title.length) / 10) + const rows = await h.ports.db + .select() + .from(schema.pushPacks) + .where(eq(schema.pushPacks.sessionId, sid)) + const written = rows.filter((r) => r.records !== null) + expect(written).toHaveLength(1) + expect(written[0]!.records).toBe(300) + + // The commit indexes them where they are: no record is written again. + expect(await commit(h, user, 'a', sid)).toBe(201) + expect(packKeys(h, c.id)).toEqual(packs) + const { out } = await headRecords(h, c.id) + expect(out).toHaveLength(300) + expect(out[0]).toBe(`public ${id(0)} replaced`) + expect(out[299]).toBe(`public ${id(299)} ${title}299`) + }) + + it('keep large records in packs, without staging them apart', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('big', { recordLayout: 'f' }) + const sid = await open(h, user, 'big') + const big = doc('big', 'x'.repeat(70_000)) + await upload(h, user, 'big', sid, [big, doc('small', 's')]) + const staged = [...h.bucket.objects.keys()].filter((k) => + k.startsWith(`internal/sessions/${sid}/records/`), + ) + expect(staged).toEqual([]) + expect(await commit(h, user, 'big', sid)).toBe(201) + expect([...h.bucket.objects.keys()].filter((k) => k.startsWith('repo/records/'))).toEqual([]) + const { out } = await headRecords(h, c.id) + expect(out).toEqual([`public big ${'x'.repeat(70_000)}`, 'public small s']) + }) + + it('put private records in the private set’s packs', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('p', { recordLayout: 'f' }) + const sid = await open(h, user, 'p') + await upload(h, user, 'p', sid, [ + doc('pub', 'open'), + { ...doc('priv', 'hidden'), private: true }, + ]) + expect(packKeys(h, c.id, 'public')).toHaveLength(2) + expect(packKeys(h, c.id, 'private')).toHaveLength(2) + expect(await commit(h, user, 'p', sid)).toBe(201) + expect((await headRecords(h, c.id)).out).toEqual(['public pub open', 'private priv hidden']) + }) + + it('count the files records reference, read at upload', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('files', { recordLayout: 'f' }) + const file = 'the pdf' + const withFile = doc('d1', 'has a file', { pdf: { $file: `sha256:${sha(file)}` } }) + // Not uploaded: the commit refuses, so it saw the reference. + const missing = await open(h, user, 'files') + await upload(h, user, 'files', missing, [withFile]) + expect(await commit(h, user, 'files', missing)).toBe(422) + + const put = await h.request(`/api/collections/org/files/files/${sha(file)}`, { + method: 'PUT', + user, + body: file, + }) + expect(put.status).toBeLessThan(300) + const sid = await open(h, user, 'files') + await upload(h, user, 'files', sid, [withFile]) + expect(await commit(h, user, 'files', sid)).toBe(201) + // The version's file references: the record's file, from the run entry alone. + const { v } = await headRecords(h, c.id) + expect(v.publicRefsRoot).not.toBeNull() + const repo = await h.ports.stores.forCollection(c.id) + const refs: string[] = [] + for await (const e of iterate(new RepoSource(countTree, repo), v.publicRefsRoot)) + refs.push(`${e.key} ${e.n}`) + expect(refs.some((r) => r.includes(sha(file)) && r.endsWith(' 1'))).toBe(true) + }) + + it('commit in parallel after compaction, the same versions as today’s layout', async () => { + const h = await harness() + const user = await h.member() + const today = await h.collection('today', { recordLayout: 'today' }) + const f = await h.collection('f', { recordLayout: 'f' }) + await h.ports.db + .update(schema.collections) + .set({ privateSalt: today.privateSalt }) + .where(eq(schema.collections.id, f.id)) + // Six uploads into a collection with a base: compacted, then committed in units. + const batches = Array.from({ length: 6 }, (_, j) => + Array.from({ length: 200 }, (_, i) => { + const n = (j * 7919 + i * 104_729) % 2000 + return doc(id(n), `v${j}-${n}`) + }), + ) + for (const slug of ['today', 'f']) { + const first = await open(h, user, slug) + await upload(h, user, slug, first, [doc(id(0), 'base'), doc(id(5000), 'kept')]) + expect(await commit(h, user, slug, first)).toBe(201) + Object.assign(compactConfig, { commitRuns: 2 }) + Object.assign(parallelConfig, { aboveBytes: 0, unitBytes: 32_000, baseLevel: 0 }) + const sid = await open(h, user, slug) + for (const b of batches) await upload(h, user, slug, sid, b) + const before = packKeys(h, f.id).length + expect(await commit(h, user, slug, sid)).toBe(202) + const [s] = await h.ports.db + .select() + .from(schema.pushSessions) + .where(eq(schema.pushSessions.id, sid)) + expect(s!.status).toBe('committed') + // F's units wrote no pack: every record was in one already. + if (slug === 'f') expect(packKeys(h, f.id).length).toBe(before) + Object.assign(compactConfig, defaults.compact) + Object.assign(parallelConfig, defaults.parallel) + } + const a = await headRecords(h, today.id) + const b = await headRecords(h, f.id) + expect(b.v.hash).toBe(a.v.hash) + expect(b.out).toEqual(a.out) + }) + + it('are listed once in the registry, and merged when small', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('l', { recordLayout: 'f' }) + // A batch past UPLOAD_PACK_FINAL_RAW: listed in the registry, never merged. + const sid = await open(h, user, 'l') + await upload( + h, + user, + 'l', + sid, + Array.from({ length: 600 }, (_, i) => doc(id(i), `${'y'.repeat(2000)}${i}`)), + ) + expect(await commit(h, user, 'l', sid)).toBe(201) + const registry = `repo/runs/${c.id}/public/manifest/f/packs/upload-${sid}.json` + expect(h.bucket.objects.has(registry)).toBe(true) + // Small pushes: their packs go in the manifest, and merges re-point their records. + for (let j = 0; j < 10; j++) { + const s = await open(h, user, 'l') + await upload(h, user, 'l', s, [doc(id(1000 + j), `small${j}`), doc(id(j), `changed${j}`)]) + expect(await commit(h, user, 'l', s)).toBe(201) + } + const { out } = await headRecords(h, c.id) + expect(out).toHaveLength(610) + expect(out.filter((r) => /changed|small/.test(r))).toHaveLength(20) + const repo = await h.ports.stores.forCollection(c.id) + const layout = repo.records!.public as unknown as { + manifests: { latest(): Promise<{ packs: { id: string; tier: number }[] }> } + } + const m = await layout.manifests.latest() + // Every small pack listed, or merged into one above them. + expect(m.packs.some((p) => p.tier > 0) || m.packs.length >= 10).toBe(true) + expect(new Set(m.packs.map((p) => p.id)).size).toBe(m.packs.length) + }) + + it('of a push that never publishes are deleted by cleanup; a published one’s are kept', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('gone', { recordLayout: 'f' }) + const kept = await open(h, user, 'gone') + await upload(h, user, 'gone', kept, [doc('k', 'kept')]) + expect(await commit(h, user, 'gone', kept)).toBe(201) + const keptPacks = packKeys(h, c.id) + expect(keptPacks).toHaveLength(2) + + const sid = await open(h, user, 'gone') + await upload(h, user, 'gone', sid, [doc('x', 'abandoned')]) + expect(packKeys(h, c.id)).toHaveLength(4) + const abort = await h.request(`/api/collections/org/gone/push/${sid}`, { + method: 'DELETE', + user, + }) + expect(abort.status).toBeLessThan(300) + // Past their sessions' grace. + await h.ports.db + .update(schema.pushSessions) + .set({ expiresAt: new Date(Date.now() - cleanupConfig.sessionGraceMs - DAY) }) + const dry = await startRun(h.ports, 'internal', { trigger: 'manual', dryRun: true }) + await h.drain() + expect((await getRun(h.ports.db, dry.id))!.stats!.deleted['upload packs']).toMatchObject({ + objects: 2, + }) + expect(packKeys(h, c.id)).toHaveLength(4) + + const r = await startRun(h.ports, 'internal', { trigger: 'manual' }) + await h.drain() + expect((await getRun(h.ports.db, r.id))!.status).toBe('done') + expect(packKeys(h, c.id).sort()).toEqual(keptPacks.sort()) + expect(await h.ports.db.select().from(schema.pushPacks)).toEqual([]) + expect((await headRecords(h, c.id)).out).toEqual(['public k kept']) + }) + + it('of a push whose commit started and never published are listed, not deleted', async () => { + // The failed commit may have indexed their records (the same record as the head's, + // say, whose newest place is then in such a pack): deleting them could break reads. + const h = await harness() + const user = await h.member() + const c = await h.collection('half', { recordLayout: 'f' }) + const first = await open(h, user, 'half') + await upload(h, user, 'half', first, [doc('same', 'unchanged')]) + expect(await commit(h, user, 'half', first)).toBe(201) + + const sid = await open(h, user, 'half') + const file = { pdf: { $file: `sha256:${sha('never uploaded')}` } } + await upload(h, user, 'half', sid, [doc('same', 'unchanged'), doc('new', 'needs a file', file)]) + // Refused for the missing file: the session is open again, then abandoned. + expect(await commit(h, user, 'half', sid)).toBe(422) + const abort = await h.request(`/api/collections/org/half/push/${sid}`, { + method: 'DELETE', + user, + }) + expect(abort.status).toBeLessThan(300) + const packs = packKeys(h, c.id) + await h.ports.db + .update(schema.pushSessions) + .set({ expiresAt: new Date(Date.now() - cleanupConfig.sessionGraceMs - DAY) }) + const r = await startRun(h.ports, 'internal', { trigger: 'manual' }) + await h.drain() + expect((await getRun(h.ports.db, r.id))!.status).toBe('done') + expect(packKeys(h, c.id)).toEqual(packs) + expect(await h.ports.db.select().from(schema.pushPacks)).toEqual([]) + // Listed with the collection's packs, for reclaim. + const repo = await h.ports.stores.forCollection(c.id) + const layout = repo.records!.public as unknown as { + manifests: { latest(): Promise<{ adopted?: string[] }> } + } + expect((await layout.manifests.latest()).adopted).toContain(sid) + expect((await headRecords(h, c.id)).out).toEqual(['public same unchanged']) + }) +}) From d5dd5b9c8f3e44005a79b4314b9240ab5b798747 Mon Sep 17 00:00:00 2001 From: "Thomas F. K. Jorna" Date: Tue, 20 Oct 2026 21:36:11 +0200 Subject: [PATCH 2/2] Layout F: key-ordered sessions stream records into whole packs; count overlapping uploads; pack rows from write hooks --- docs/protocol-v2.md | 38 +++-- packages/protocol/src/index.ts | 3 +- packages/protocol/src/repo/layouts/index.ts | 3 +- packages/protocol/src/repo/layouts/keyed.ts | 66 ++++---- packages/protocol/src/repo/layouts/kit.ts | 21 ++- packages/protocol/src/repo/layouts/types.ts | 26 +++- packages/protocol/test/repo/layouts.test.ts | 58 +++++-- packages/server/drizzle/0026_push_packs.sql | 4 +- .../server/drizzle/meta/0026_snapshot.json | 18 ++- packages/server/drizzle/meta/_journal.json | 2 +- packages/server/src/api/push.ts | 22 ++- packages/server/src/db/schema.ts | 12 ++ packages/server/src/push/delta.ts | 64 +++++--- packages/server/src/push/packs.ts | 144 +++++++++++------- packages/server/src/push/session.ts | 37 ++++- packages/server/test/upload-packs.test.ts | 92 ++++++++++- 16 files changed, 444 insertions(+), 166 deletions(-) diff --git a/docs/protocol-v2.md b/docs/protocol-v2.md index 3fd33258..ba33acc1 100644 --- a/docs/protocol-v2.md +++ b/docs/protocol-v2.md @@ -778,23 +778,25 @@ DELETE /push/ abandon the session **Opening a session.** The request body is a JSON object. Every member is OPTIONAL. -| Member | Meaning | -| ---------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `base` | The semver of the version the changes are against. If it is not the collection's head, the server MUST answer 409 with `currentVersion`. `null` or absent: the head at the time of opening | -| `schemas` | The complete type set, slug → schema (Section 5). It replaces the base's type set; a type omitted is removed with its records. Absent: the base's type set | -| `metadata` | Replaces the base's metadata. An object or `null` | -| `metadata_patch` | An object whose top-level members are merged into the base's metadata. Ignored if `metadata` is present. With neither, the base's metadata is kept | -| `files` | `{"add": [fileHash, …], "remove": [fileHash, …]}`: files to declare or remove in addition to those records reference (Section 9) | -| `message` | A string recorded in the log entry | -| `app_id`, `actor_id` | Strings identifying the publishing application and actor | -| `strip_unknown_fields` | A boolean; see **Records** | - -The response is 200 with `{"session_id", "base", "needed_files", "expires_at", "limits"}`: +| Member | Meaning | +| ---------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `base` | The semver of the version the changes are against. If it is not the collection's head, the server MUST answer 409 with `currentVersion`. `null` or absent: the head at the time of opening | +| `schemas` | The complete type set, slug → schema (Section 5). It replaces the base's type set; a type omitted is removed with its records. Absent: the base's type set | +| `metadata` | Replaces the base's metadata. An object or `null` | +| `metadata_patch` | An object whose top-level members are merged into the base's metadata. Ignored if `metadata` is present. With neither, the base's metadata is kept | +| `files` | `{"add": [fileHash, …], "remove": [fileHash, …]}`: files to declare or remove in addition to those records reference (Section 9) | +| `message` | A string recorded in the log entry | +| `app_id`, `actor_id` | Strings identifying the publishing application and actor | +| `strip_unknown_fields` | A boolean; see **Records** | +| `in_key_order` | A boolean: the client sends each records request's lines in key order (type, then id); see **Records**. A server MAY decline it (the response's `in_key_order` says), and MAY then allow larger requests | + +The response is 200 with `{"session_id", "base", "needed_files", "expires_at", "in_key_order", "limits"}`: - `base` is the semver the session is against, or `null` if the collection has no versions; - `needed_files` lists the declared files the server does not hold for the collection; the client MUST upload them before committing; - `expires_at` is when the session expires; it is extended by each records or deletes request; +- `in_key_order` is whether the server took the session as key-ordered; - `limits` are the server's limits, by which a client MUST size its requests: | Limit | Meaning | @@ -819,6 +821,10 @@ type's schema (Section 5.1), and its type MUST be in the session's type set. identified by its 1-based `line` number among the request's non-empty lines, and MUST store nothing from the request. A server MAY list only the first failures, with `totalErrors` giving their number. +- In a session the server took as `in_key_order`, each request's lines MUST come in key order: + by type, then id, as UTF-8 bytes (equal keys allowed; the later wins). A line before the one + ahead of it fails the request with 422, and nothing from it is stored. Order across requests is + the client's to keep; a server MAY report how many requests overlapped earlier ones' key ranges. **Deletes.** `POST …/deletes` takes NDJSON lines `{"type", "id"}` and answers 200 with `{"received": n}`. Each line MUST satisfy the syntax, duplicate key, unsafe integer and lone surrogate rules, and its `id` and `type` the record id and type slug rules (Section 3); the type MUST be in the session's type set. Deleting a (type, id) the base does @@ -861,7 +867,10 @@ server's storage: a client can upload a whole corpus into one session over many days, and commit it as one version. A server keeps the uploads in storage rather than memory, and a background commit's status MAY carry `progress` (underlay.org: `runs`, `run_bytes`, `runs_compacting`, `phase` of `compacting`, `planning`, `units` or `assembling`, `units` done of -`total`, and `progress_at`). An open session expires only after `session_idle_seconds` without an +`total`, and `progress_at`). underlay.org's status also carries `in_key_order` and +`overlapping_uploads`, the records requests whose key range overlapped an earlier one's: a corpus +sent in key order, with none overlapping, is stored with each record next to its neighbours, and +reads back with the fewest requests. An open session expires only after `session_idle_seconds` without an upload, so a client that stops can pick the same session up again by its id. Each version rewrites the leaves its changes land in, so one large version costs far less than the same records in many small ones. @@ -909,7 +918,8 @@ The reference implementation defines these in `packages/protocol/src/constants.t test vectors repeat them. Server limits (Section 11.4) are not protocol constants. Informative: underlay.org advertises -`open_bytes` 8 MiB, `batch_bytes` 8 MiB, `batch_lines` 10,000, `session_idle_seconds` 43,200, +`open_bytes` 8 MiB, `batch_bytes` 8 MiB (64 MiB in an `in_key_order` session), `batch_lines` +10,000 (50,000 in one), `session_idle_seconds` 43,200, `open_sessions` 20 and `file_bytes` 32 MiB; caps log pages at 1,000 entries, manifest pages at 25,000 records and diff pages at 5,000 records or 4 MiB of records; stores records over 64 KiB out of line; and commits in the background when a session uploaded more than 100,000 records or a diff --git a/packages/protocol/src/index.ts b/packages/protocol/src/index.ts index 5cb5a657..827fc821 100644 --- a/packages/protocol/src/index.ts +++ b/packages/protocol/src/index.ts @@ -20,8 +20,9 @@ export { type LiveSet, type MergePlan, type PackInfo, - packIdsFor, + type PackHooks, type PackRecord, + type UploadWriter, type Placed, MissingRecordError, RECLAIM_MAX_RUNS, diff --git a/packages/protocol/src/repo/layouts/index.ts b/packages/protocol/src/repo/layouts/index.ts index 2b3b6f5d..259f3e76 100644 --- a/packages/protocol/src/repo/layouts/index.ts +++ b/packages/protocol/src/repo/layouts/index.ts @@ -12,12 +12,11 @@ export { type MergePlan, MissingRecordError, RECLAIM_MAX_RUNS, - packIdsFor, type ReclaimPlan, TooManyRunsError, UPLOAD_PACK_FINAL_RAW, } from './keyed.js' -export { headConfig, type LayerInfo, type PackInfo, pendingConfig } from './kit.js' +export { headConfig, type LayerInfo, type PackHooks, type PackInfo, pendingConfig } from './kit.js' export * from './types.js' /** A set's home under the collection's: `runs///` holds its files. */ diff --git a/packages/protocol/src/repo/layouts/keyed.ts b/packages/protocol/src/repo/layouts/keyed.ts index bf191151..91a3ed31 100644 --- a/packages/protocol/src/repo/layouts/keyed.ts +++ b/packages/protocol/src/repo/layouts/keyed.ts @@ -38,7 +38,7 @@ import { newId, type PackInfo, PackReader, - PACK_RAW, + type PackHooks, PackWriter, packDirKey, packKey, @@ -57,6 +57,7 @@ import { type RecordStore, type RecordWriter, type TypedLeaf, + type UploadWriter, } from './types.js' /** Index entries a reclaim range job works on: its live set is 8 bytes an entry. */ @@ -142,18 +143,6 @@ export const UPLOAD_PACK_FINAL_RAW = 1024 * 1024 /** Commits whose uploads' packs a manifest remembers listing (`Manifest.adopted`). */ const ADOPTED_KEPT = 1000 -/** - * Ids for the packs `writePacks` makes of these records, recorded before they're - * written (so an upload that dies leaves nothing unknown). A pack closes only once - * it holds PACK_RAW (PackWriter, the same count of a line), so this many at most. - */ -export function packIdsFor(records: readonly PackRecord[]): string[] { - let raw = 0 - // A line is `\t` and its newline, the key `recordKey`'s: type, id, hash and two NULs. - for (const r of records) raw += r.type.length + r.id.length + r.hash.length + r.line.length + 4 - return Array.from({ length: Math.floor(raw / PACK_RAW) + 1 }, newId) -} - /** How long a replaced object stays readable (readers holding an older manifest). */ export const layoutConfig = { graceMs: 60_000, @@ -302,31 +291,36 @@ export class KeyedLayout implements RecordStore { } /** - * An upload's records into packs of their own, in key order, outside the index and - * the manifest until a commit names them (adoptPacks). Their places, in the order - * given, go in the push's runs, so a commit indexes the records without moving them. + * A writer of an upload's records into packs of its own, outside the index and the + * manifest until a commit names them (adoptPacks). Their places go in the push's + * runs, so a commit indexes the records without moving them. Records arriving in + * key order (an upload sent sorted) stream straight into whole packs; one below the + * last starts a new pack. */ + uploadWriter(hooks: PackHooks = {}): UploadWriter { + const pw = new PackWriter(this.store, { hooks }) + return { + append: (r) => pw.append(recordKey(r.type, r.id, r.hash), r.line), + ready: () => pw.ready(), + finish: () => pw.finish(), + } + } + + /** An upload's records into packs, sorted first: their places, in the order given. */ async writePacks( records: readonly PackRecord[], - ids: readonly string[], + hooks: PackHooks = {}, ): Promise<{ places: string[]; packs: PackInfo[] }> { - let next = 0 - const pw = new PackWriter(this.store, { - ids: () => { - const id = ids[next++] - if (id === undefined) throw new Error('writePacks: the records need more packs than ids') - return id - }, - }) + const w = this.uploadWriter(hooks) const keys = records.map((r) => recordKey(r.type, r.id, r.hash)) const order = keys.map((_, i) => i).sort((a, b) => compareUtf8(keys[a]!, keys[b]!)) const places: string[] = Array.from({ length: records.length }) let n = 0 for (const i of order) { - places[i] = pw.append(keys[i]!, records[i]!.line) - if (++n % 1024 === 0) await pw.ready() + places[i] = w.append(records[i]!) + if (++n % 1024 === 0) await w.ready() } - return { places, packs: await pw.finish() } + return { places, packs: await w.finish() } } /** @@ -353,16 +347,16 @@ export class KeyedLayout implements RecordStore { } /** - * Delete packs an upload wrote for a commit `name` that never named them (an - * abandoned push). A commit that named them and then didn't publish (a conflict, - * then the session failed) leaves them: the manifest or registry lists them, so - * they go only by reclaim. False when kept. + * Delete packs an upload wrote for commit `name`, but none the collection lists: + * a pack `adoptPacks` named (in the manifest, its retired files, or the registry + * object it wrote) goes only by reclaim. The ids deleted. */ - async dropPacks(name: string, ids: readonly string[]): Promise { + async dropPacks(name: string, ids: readonly string[]): Promise { const m = await this.manifests.latest() - if (m.adopted?.includes(name)) return false - if (await this.store.head(this.registry.keyFor(`upload-${name}`))) return false const listed = new Set([...m.packs.map((p) => p.id), ...m.retired.map((r) => r.key)]) + const registered = await this.store.get(this.registry.keyFor(`upload-${name}`)) + if (registered) + for (const p of JSON.parse(await registered.text()) as PackInfo[]) listed.add(p.id) const gone = ids.filter((id) => !listed.has(id) && !listed.has(packKey(id))) for (let i = 0; i < gone.length; i += 16) await Promise.all( @@ -370,7 +364,7 @@ export class KeyedLayout implements RecordStore { .slice(i, i + 16) .flatMap((id) => [this.store.delete(packKey(id)), this.store.delete(packDirKey(id))]), ) - return true + return gone } /** A commit's runs its units held back (RecordWriter.finish), as one pending object. */ diff --git a/packages/protocol/src/repo/layouts/kit.ts b/packages/protocol/src/repo/layouts/kit.ts index df5366c9..15919e66 100644 --- a/packages/protocol/src/repo/layouts/kit.ts +++ b/packages/protocol/src/repo/layouts/kit.ts @@ -76,6 +76,15 @@ export interface PackInfo { raw: number } +/** + * Called around each pack's write. An upload records a pack before it's written, so + * cleanup can find it if the upload dies, and its stats once it is. + */ +export interface PackHooks { + before?(id: string): Promise + after?(pack: PackInfo): Promise +} + export class PackWriter { #id = '' #blocks: Promise[] = [] @@ -90,15 +99,14 @@ export class PackWriter { constructor( readonly store: Store, - /** `ids` names each new pack (an upload's, recorded before they're written); else random. */ - readonly opts: { tier?: number; ids?: () => string } = {}, + readonly opts: { tier?: number; hooks?: PackHooks } = {}, ) {} /** Add a record line under `key`; returns its place. A key below the last starts a new pack. */ append(key: string, line: string): string { if (this.#lastKey !== null && compareUtf8(key, this.#lastKey) < 0) this.#close() this.#lastKey = key - if (!this.#id) this.#id = this.opts.ids ? this.opts.ids() : newId() + if (!this.#id) this.#id = newId() this.#lines.push(`${key}\t${line}`) const at = loc(this.#id, this.#blocks.length, this.#lines.length - 1) this.#raw += key.length + line.length + 2 @@ -152,15 +160,18 @@ export class PackWriter { const parts = await Promise.all(job.blocks) const offsets = [0] for (const p of parts) offsets.push(offsets.at(-1)! + p.byteLength) + await this.opts.hooks?.before?.(job.id) await putJoined(this.store, packKey(job.id), parts, { ifAbsent: true }) await this.store.put(packDirKey(job.id), JSON.stringify(offsets), { ifAbsent: true }) - this.done.push({ + const pack = { id: job.id, tier: this.opts.tier ?? 0, records: job.records, bytes: offsets.at(-1)!, raw: job.raw, - }) + } + this.done.push(pack) + await this.opts.hooks?.after?.(pack) } /** Close the current pack and wait for every write. */ diff --git a/packages/protocol/src/repo/layouts/types.ts b/packages/protocol/src/repo/layouts/types.ts index 7ece86e4..55c78b28 100644 --- a/packages/protocol/src/repo/layouts/types.ts +++ b/packages/protocol/src/repo/layouts/types.ts @@ -5,7 +5,7 @@ * holds. Neither changes a node, a root or any hash. */ import type { RecordEntry } from '../../tree/node.js' -import type { PackInfo, PendingRun } from './kit.js' +import type { PackHooks, PackInfo, PendingRun } from './kit.js' export const LAYOUTS = ['today', 'f'] as const export type LayoutName = (typeof LAYOUTS)[number] @@ -25,6 +25,16 @@ export interface PackRecord { line: string } +/** Writes an upload's records into packs (RecordStore.uploadWriter). */ +export interface UploadWriter { + /** Write one record: its place. */ + append(record: PackRecord): string + /** Wait while too much is held (call between records): memory stays bounded. */ + ready(): Promise + /** Write the last pack and wait for every write: the packs written. */ + finish(): Promise +} + /** Records still in use, from a mark over every retained version (storage reclaim). */ export interface LiveSet { /** `recordKey(type, id, hash)` of every record in a retained version. */ @@ -73,13 +83,15 @@ export interface RecordStore { /** Give a fork its records: it lists the parent's files and reads them where they are. */ forkInto?(target: RecordStore): Promise> /** - * Write an upload's records into packs of their own, named by `ids` (more than it - * needs; see `packIdsFor`), outside the index and the manifest: their places, in - * the order given, and the packs written. + * A writer of an upload's records into packs of its own, outside the index and the + * manifest until a commit names them (`adoptPacks`). Records in key order fill whole + * packs; one below the last starts a new pack. */ + uploadWriter?(hooks?: PackHooks): UploadWriter + /** An upload's records into packs, sorted first: their places, in the order given. */ writePacks?( records: readonly PackRecord[], - ids: readonly string[], + hooks?: PackHooks, ): Promise<{ places: string[]; packs: PackInfo[] }> /** * List packs uploads wrote, once the commit's index names their records: large @@ -87,8 +99,8 @@ export interface RecordStore { * commit's, so a retry lists nothing twice. */ adoptPacks?(name: string, packs: readonly PackInfo[]): Promise - /** Delete packs uploads wrote for commit `name`, unless it named them (false then). */ - dropPacks?(name: string, ids: readonly string[]): Promise + /** Delete packs uploads wrote for commit `name`, but none the collection lists: the ids deleted. */ + dropPacks?(name: string, ids: readonly string[]): Promise } export interface RecordWriter { diff --git a/packages/protocol/test/repo/layouts.test.ts b/packages/protocol/test/repo/layouts.test.ts index db5a43a7..6fa35320 100644 --- a/packages/protocol/test/repo/layouts.test.ts +++ b/packages/protocol/test/repo/layouts.test.ts @@ -25,7 +25,6 @@ import { mergeTree, type NodeSource, openRepo, - packIdsFor, type RecordEntry, recordCanonical, RECLAIM_MAX_RUNS, @@ -648,10 +647,17 @@ describe("an upload's packs", () => { // Ten small uploads, each into a pack of its own; their commits index them. for (let u = 0; u < 10; u++) { const recs = upload(u * 100, u * 100 + 100) - const ids_ = packIdsFor(recs) - expect(ids_).toHaveLength(1) - const { places, packs } = await layout.writePacks(recs, ids_) - expect(packs.map((p) => p.id)).toEqual(ids_) + // Each pack is announced before it's written, and reported once it is. + const seen: string[] = [] + const { places, packs } = await layout.writePacks(recs, { + before: async (id) => { + seen.push(`before ${id}`) + expect(await root.head(`runs/c1/packs/${id}.pk`)).toBeNull() + }, + after: async (p) => void seen.push(`after ${p.id} ${p.records}`), + }) + expect(packs).toHaveLength(1) + expect(seen).toEqual([`before ${packs[0]!.id}`, `after ${packs[0]!.id} 100`]) const before = await packsNow() const w = layout.writer() recs.forEach((_, i) => w.addPlaced('T', rec(u * 100 + (99 - i)), places[i]!)) @@ -677,6 +683,33 @@ describe("an upload's packs", () => { } }) + it('stream into whole packs when they arrive in key order', async () => { + // A sorted upload's records go straight into packs as they arrive: 16 MB raw + // each, not one pack a request. + const layout = fLayout(memoryStore()) + const big = (i: number) => ({ + type: 'T', + id: `k${String(i).padStart(5, '0')}`, + hash: `h${i}`, + line: `${i}:${'x'.repeat(20_000)}`, + }) + const sorted = layout.uploadWriter() + const places: string[] = [] + for (let i = 0; i < 1000; i++) { + places.push(sorted.append(big(i))) + await sorted.ready() + } + const packs = await sorted.finish() + // 20 MB of records: one full pack and the rest. + expect(packs).toHaveLength(2) + expect(packs[0]!.raw).toBeGreaterThanOrEqual(16 * 1024 * 1024) + expect(new Set(places.map((p) => p.split('.')[0])).size).toBe(2) + // Out of order, each step down starts a new pack. + const shuffled = layout.uploadWriter() + for (const i of [5, 6, 1, 2, 9, 3]) shuffled.append(big(i)) + expect(await shuffled.finish()).toHaveLength(3) + }) + it("count toward a commit writer's runs, written as it goes", async () => { // A commit of packed entries alone must not hold every index entry to its end (#63). const saved = layoutConfig.writerRunEntries @@ -684,7 +717,7 @@ describe("an upload's packs", () => { try { const layout = fLayout(memoryStore()) const recs = upload(0, 350) - const { places } = await layout.writePacks(recs, packIdsFor(recs)) + const { places } = await layout.writePacks(recs) const w = layout.writer({ hold: true }) for (let j = 0; j < 350; j++) { w.addPlaced('T', rec(349 - j), places[j]!) @@ -708,7 +741,7 @@ describe("an upload's packs", () => { const root = memoryStore() const layout = fLayout(root) const recs = upload(0, 50) - const { packs } = await layout.writePacks(recs, packIdsFor(recs)) + const { packs } = await layout.writePacks(recs) await layout.adoptPacks('s1', packs) await layout.adoptPacks('s1', packs) expect((await layout.manifests.latest()).packs.map((p) => p.id)).toEqual([packs[0]!.id]) @@ -728,14 +761,17 @@ describe("an upload's packs", () => { const root = memoryStore() const layout = fLayout(root) const named = upload(0, 20) - const a = await layout.writePacks(named, packIdsFor(named)) + const a = await layout.writePacks(named) await layout.adoptPacks('kept', a.packs) const left = upload(20, 40) - const b = await layout.writePacks(left, packIdsFor(left)) + const b = await layout.writePacks(left) const files = async () => (await root.list('runs/c1/packs/')).keys expect(await files()).toHaveLength(4) - expect(await layout.dropPacks('kept', [a.packs[0]!.id])).toBe(false) - expect(await layout.dropPacks('left', [b.packs[0]!.id, 'neverwritten'])).toBe(true) + expect(await layout.dropPacks('kept', [a.packs[0]!.id])).toEqual([]) + expect(await layout.dropPacks('left', [b.packs[0]!.id, 'neverwritten'])).toEqual([ + b.packs[0]!.id, + 'neverwritten', + ]) expect((await files()).sort()).toEqual( [`runs/c1/packs/${a.packs[0]!.id}.dir`, `runs/c1/packs/${a.packs[0]!.id}.pk`].sort(), ) diff --git a/packages/server/drizzle/0026_push_packs.sql b/packages/server/drizzle/0026_push_packs.sql index 70d99ec2..f0d9856c 100644 --- a/packages/server/drizzle/0026_push_packs.sql +++ b/packages/server/drizzle/0026_push_packs.sql @@ -10,4 +10,6 @@ CREATE TABLE `push_packs` ( FOREIGN KEY (`session_id`) REFERENCES `push_sessions`(`id`) ON UPDATE no action ON DELETE cascade ); --> statement-breakpoint -ALTER TABLE `push_sessions` ADD `commit_started` integer DEFAULT false NOT NULL; \ No newline at end of file +ALTER TABLE `push_sessions` ADD `commit_started` integer DEFAULT false NOT NULL;--> statement-breakpoint +ALTER TABLE `push_sessions` ADD `in_key_order` integer DEFAULT false NOT NULL;--> statement-breakpoint +ALTER TABLE `push_sessions` ADD `overlapping_uploads` integer DEFAULT 0 NOT NULL; \ No newline at end of file diff --git a/packages/server/drizzle/meta/0026_snapshot.json b/packages/server/drizzle/meta/0026_snapshot.json index 5a88d9a7..6ababddb 100644 --- a/packages/server/drizzle/meta/0026_snapshot.json +++ b/packages/server/drizzle/meta/0026_snapshot.json @@ -1,7 +1,7 @@ { "version": "6", "dialect": "sqlite", - "id": "da1a4cfe-5e4b-4fb0-9240-470d050217f9", + "id": "b7f7c1c5-2c58-4d38-a78e-dec398756445", "prevId": "42b4a563-0b06-417e-a83f-2216ffba5eb8", "tables": { "abuse_reports": { @@ -2316,6 +2316,22 @@ "autoincrement": false, "default": false }, + "in_key_order": { + "name": "in_key_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": false + }, + "overlapping_uploads": { + "name": "overlapping_uploads", + "type": "integer", + "primaryKey": false, + "notNull": true, + "autoincrement": false, + "default": 0 + }, "cleaned_at": { "name": "cleaned_at", "type": "integer", diff --git a/packages/server/drizzle/meta/_journal.json b/packages/server/drizzle/meta/_journal.json index 225cdc62..e60d3a5b 100644 --- a/packages/server/drizzle/meta/_journal.json +++ b/packages/server/drizzle/meta/_journal.json @@ -187,7 +187,7 @@ { "idx": 26, "version": "6", - "when": 1792518549719, + "when": 1792524725308, "tag": "0026_push_packs", "breakpoints": true } diff --git a/packages/server/src/api/push.ts b/packages/server/src/api/push.ts index adceb665..30f1368b 100644 --- a/packages/server/src/api/push.ts +++ b/packages/server/src/api/push.ts @@ -22,9 +22,9 @@ import type { Ports } from '../ports.js' import { headBase, ingestDeletes, + batchLimits, ingestRecords, MAX_BATCH_BYTES, - MAX_BATCH_LINES, } from '../push/delta.js' import { finalizeSession } from '../push/finalize.js' import { unitProgress } from '../push/recover.js' @@ -54,10 +54,10 @@ export const commitConfig = { asyncAbove: 100_000 } * This node's push limits, advertised when a session opens (docs/protocol-v2.md, * section 11.4). Read on each call: tests lower some of them. */ -export const pushLimits = () => ({ +export const pushLimits = (session: Pick = { inKeyOrder: false }) => ({ open_bytes: MAX_OPEN_BYTES, - batch_bytes: MAX_BATCH_BYTES, - batch_lines: MAX_BATCH_LINES, + batch_bytes: batchLimits(session).bytes, + batch_lines: batchLimits(session).lines, session_idle_seconds: SESSION_TTL_MS / 1000, open_sessions: limits.openSessions, file_bytes: SMALL_UPLOAD_BYTES, @@ -248,6 +248,10 @@ async function statusBody(ports: Ports, s: SessionRow) { expires_at: s.expiresAt, created_at: s.createdAt, finalize_started_at: s.finalizeStartedAt, + // Whether the uploads came in key order: an upload that overlapped an earlier + // one's key range leaves records scattered across packs (layout F). + in_key_order: s.inKeyOrder, + overlapping_uploads: s.overlappingUploads, result: s.result ?? null, error: s.error ?? null, } @@ -356,6 +360,8 @@ export function pushRoutes() { appId: str(body.app_id), actorId: str(body.actor_id), stripUnknownFields: body.strip_unknown_fields === true, + // Only layout F packs records at upload: elsewhere it changes nothing. + inKeyOrder: body.in_key_order === true && access.collection.recordLayout === 'f', }, inputs, ) @@ -369,7 +375,8 @@ export function pushRoutes() { inputs.files && 'add' in inputs.files ? inputs.files.add : [], ), expires_at: session.expiresAt, - limits: pushLimits(), + in_key_order: session.inKeyOrder, + limits: pushLimits(session), }) }) @@ -379,12 +386,13 @@ export function pushRoutes() { const session = await ownSession(c, access) if (session instanceof Response) return session // Parsed line by line as the body arrives (v2-scale-review.md S6). + const max = batchLimits(session).bytes let r try { - r = await ingestRecords(c.var.ports, session, readLines(c, MAX_BATCH_BYTES)) + r = await ingestRecords(c.var.ports, session, readLines(c, max)) } catch (err) { if (err instanceof BodyTooLarge) - return jsonError(c, 413, `Batches are limited to ${MAX_BATCH_BYTES} bytes`) + return jsonError(c, 413, `Batches are limited to ${max} bytes`) throw err } return r.ok diff --git a/packages/server/src/db/schema.ts b/packages/server/src/db/schema.ts index 39d87f99..f59a0be9 100644 --- a/packages/server/src/db/schema.ts +++ b/packages/server/src/db/schema.ts @@ -613,6 +613,18 @@ export const pushSessions = sqliteTable( * (push/packs.ts), so cleanup leaves those to reclaim instead of deleting them. */ commitStarted: bool('commit_started').notNull().default(false), + /** + * Opened `in_key_order` on a layout F collection: each batch's records come by + * type, then id, and stream straight into packs (push/packs.ts), so batches + * may be larger (push/delta.ts `batchLimits`). + */ + inKeyOrder: bool('in_key_order').notNull().default(false), + /** + * Uploads whose key range overlapped an earlier upload's (or a compaction's) + * when they landed: 0 means the session's uploads came in key order, and its + * packs hold neighbouring records. + */ + overlappingUploads: integer('overlapping_uploads').notNull().default(0), /** When storage cleanup deleted the session's objects (sessions//) and run rows. */ cleanedAt: ts('cleaned_at'), /** diff --git a/packages/server/src/push/delta.ts b/packages/server/src/push/delta.ts index 153b3b98..4f8237d6 100644 --- a/packages/server/src/push/delta.ts +++ b/packages/server/src/push/delta.ts @@ -31,7 +31,7 @@ import type { Ports } from '../ports.js' import { type BaseVersion, commitVersion, type TypeInput } from '../versions/commit.js' import { deltaChanges, isPrivateSchema, stageOutOfLine } from './changes.js' import type { SessionCommitResult } from './outcome.js' -import { packEntries, packStats } from './packs.js' +import { keyOrderPlacer, OutOfKeyOrder, packEntries } from './packs.js' import { planParallel, SerialTooLargeError } from './parallel.js' import { type RunEntry, type RunIndex, writeRun } from './runs.js' import { @@ -49,6 +49,19 @@ import { /** 8 MB: an upload request holds its batch several times over (text, records, run lines). */ export const MAX_BATCH_BYTES = 8 * 1024 * 1024 export const MAX_BATCH_LINES = 10_000 +/** + * A key-ordered session's batches (layout F, `in_key_order`): records stream into + * packs as they arrive, so a request holds only its run entries (about 150 bytes a + * record) and the pack filling. Larger batches mean fewer runs to compact. + */ +export const KEY_ORDER_BATCH_BYTES = 64 * 1024 * 1024 +export const KEY_ORDER_BATCH_LINES = 50_000 + +/** A session's batch limits: larger for one whose records stream into packs. */ +export const batchLimits = (session: Pick) => + session.inKeyOrder + ? { bytes: KEY_ORDER_BATCH_BYTES, lines: KEY_ORDER_BATCH_LINES } + : { bytes: MAX_BATCH_BYTES, lines: MAX_BATCH_LINES } const MAX_REPORTED = 100 export interface LineError { @@ -83,7 +96,13 @@ export async function prepareRecords( inputs: SessionInputs, input: string | AsyncIterable, /** `packed`: the records go into packs (layout F), so large ones aren't staged apart. */ - opts: { stripUnknownFields: boolean; packed?: boolean }, + opts: { + stripUnknownFields: boolean + packed?: boolean + maxLines?: number + /** Each accepted record as it's read (a key-ordered session's packing), with its line number. */ + place?: (e: RunEntry, line: number) => Promise + }, ): Promise< { entries: RunEntry[] } | { errors: LineError[]; total: number } | { tooManyLines: true } > { @@ -115,7 +134,7 @@ export async function prepareRecords( let i = -1 for await (const line of source) { i++ - if (i >= MAX_BATCH_LINES) return { tooManyLines: true } + if (i >= (opts.maxLines ?? MAX_BATCH_LINES)) return { tooManyLines: true } let rec try { rec = parseRecordLine(line) @@ -180,6 +199,8 @@ export async function prepareRecords( ...(isPrivate ? { p: true } : {}), ...(refs.length ? { r: refs } : {}), } + // A batch with an invalid record is refused whole: nothing more is packed. + if (opts.place && total === 0) await opts.place(entry, i + 1) entries.push(entry) } return total > 0 ? { errors, total } : { entries } @@ -191,14 +212,28 @@ export async function ingestRecords( input: string | AsyncIterable, ): Promise { const inputs = await loadInputs(ports, session.id) - // Layout F: the records go straight into the collection's packs. + // Layout F: the records go straight into the collection's packs; a key-ordered + // session's as they arrive. const records = (await ports.stores.forCollection(session.collectionId)).records - const prepared = await prepareRecords(ports, session.id, inputs, input, { - stripUnknownFields: session.stripUnknownFields, - packed: records !== null, - }) + const placer = records && session.inKeyOrder ? keyOrderPlacer(ports, session.id, records) : null + const limits = batchLimits(session) + let prepared + try { + prepared = await prepareRecords(ports, session.id, inputs, input, { + stripUnknownFields: session.stripUnknownFields, + packed: records !== null, + maxLines: limits.lines, + ...(placer ? { place: placer.place } : {}), + }) + } catch (err) { + // Packs already written are reported (their rows' stats), so cleanup settles them. + await placer?.finish().catch(() => {}) + if (err instanceof OutOfKeyOrder) return { ok: false, status: 422, error: err.message } + throw err + } + await placer?.finish() if ('tooManyLines' in prepared) { - return { ok: false, status: 413, error: `At most ${MAX_BATCH_LINES} records per batch` } + return { ok: false, status: 413, error: `At most ${limits.lines} records per batch` } } if ('errors' in prepared) { return { @@ -212,16 +247,9 @@ export async function ingestRecords( if (prepared.entries.length === 0) return { ok: false, status: 400, error: 'Empty batch' } const seq = await nextRunSeq(ports, session.id) if (seq === null) return { ok: false, status: 409, error: 'Session is not open' } - const packs = records ? await packEntries(ports, session.id, records, prepared.entries) : [] + if (records && !placer) await packEntries(ports, session.id, records, prepared.entries) const index = await writeRun(ports.stores.internal, session.id, seq, prepared.entries) - await recordRun( - ports, - session.id, - 'records', - index, - { records: prepared.entries.length }, - packStats(ports, session.id, packs), - ) + await recordRun(ports, session.id, 'records', index, { records: prepared.entries.length }) return { ok: true, received: prepared.entries.length } } diff --git a/packages/server/src/push/packs.ts b/packages/server/src/push/packs.ts index 8fd12631..dd5d8580 100644 --- a/packages/server/src/push/packs.ts +++ b/packages/server/src/push/packs.ts @@ -6,31 +6,48 @@ * compacting 206 GB, 2026-10-19), and commit units index the records without * reading or writing them again. * + * A session opened `in_key_order` (its uploads sorted by type and id) streams each + * record into a pack as it arrives (`keyOrderPlacer`): packs fill to 16 MB, hold + * neighbouring records, and a batch isn't held in memory, so batches can be larger. + * Otherwise a batch is sorted and packed once it's read (`packEntries`), a pack a + * batch, in whatever order the batches come. + * * A pack is outside the collection's index and manifest until a commit names it * (`adoptSessionPacks`, just before the version publishes). Each is recorded in - * `push_packs` before it's written, so storage cleanup can delete the packs of a - * push that never published (`dropSessionPacks`). + * `push_packs` before it's written and its stats once it is, so storage cleanup can + * settle the packs of a push that never published (`settleSessionPacks`). */ import { - packIdsFor, + type PackHooks, type PackInfo, type RecordSet, type RecordSets, type Repo, + type UploadWriter, } from '@underlay/protocol' import { and, eq, isNotNull } from 'drizzle-orm' import * as schema from '../db/schema.js' import type { Ports } from '../ports.js' -import { type RunEntry, runKey } from './runs.js' +import { compareRunKeys, type RunEntry, runKey } from './runs.js' const SETS = ['public', 'private'] as const -/** A pack an upload wrote, for its row's stats. */ -export interface UploadPack { - set: RecordSet - pack: PackInfo -} +/** An upload marks an entry private exactly when it goes to the private set (deltaChanges). */ +const setOf = (e: RunEntry): RecordSet => (e.p ? 'private' : 'public') + +/** Each pack recorded before it's written, and its stats once it is. */ +const hooksFor = (ports: Ports, sessionId: string, set: RecordSet): PackHooks => ({ + before: async (packId) => { + await ports.db.insert(schema.pushPacks).values({ sessionId, packId, set }) + }, + after: async (pack) => { + await ports.db + .update(schema.pushPacks) + .set({ records: pack.records, bytes: pack.bytes, raw: pack.raw }) + .where(and(eq(schema.pushPacks.sessionId, sessionId), eq(schema.pushPacks.packId, pack.id))) + }, +}) /** * Write a batch's records into packs, a set each, and give their entries places @@ -42,38 +59,58 @@ export async function packEntries( sessionId: string, records: RecordSets, entries: RunEntry[], -): Promise { +): Promise { const last = new Map() for (const e of entries) last.set(runKey(e), e) - const out: UploadPack[] = [] for (const set of SETS) { - // An upload marks an entry private exactly when it goes to the private set (deltaChanges). - const group = [...last.values()].filter((e) => (e.p === true) === (set === 'private')) + const group = [...last.values()].filter((e) => setOf(e) === set) if (!group.length) continue const recs = group.map((e) => ({ type: e.t, id: e.k, hash: e.h!, line: e.b! })) - const ids = packIdsFor(recs) - // Recorded before they're written: an upload that dies leaves no pack cleanup can't find. - await ports.db - .insert(schema.pushPacks) - .values(ids.map((packId) => ({ sessionId, packId, set }))) - const { places, packs } = await records[set].writePacks!(recs, ids) + const { places } = await records[set].writePacks!(recs, hooksFor(ports, sessionId, set)) group.forEach((e, i) => { e.l = places[i]! delete e.b }) - out.push(...packs.map((pack) => ({ set, pack }))) } - return out } -/** The statements that record written packs' stats (run with the upload's run row). */ -export const packStats = (ports: Ports, sessionId: string, packs: UploadPack[]) => - packs.map(({ pack }) => - ports.db - .update(schema.pushPacks) - .set({ records: pack.records, bytes: pack.bytes, raw: pack.raw }) - .where(and(eq(schema.pushPacks.sessionId, sessionId), eq(schema.pushPacks.packId, pack.id))), - ) +/** A record of a key-ordered upload that comes before the one ahead of it. */ +export class OutOfKeyOrder extends Error { + constructor(readonly line: number) { + super( + `Records out of key order at line ${line}: this session was opened in_key_order, so each batch's records go by type, then id`, + ) + } +} + +/** + * Places a key-ordered batch's records as they arrive: each straight into its set's + * pack, its entry left with a place instead of a body. `finish` writes the last + * packs. A record before the one ahead of it throws OutOfKeyOrder. + */ +export function keyOrderPlacer(ports: Ports, sessionId: string, records: RecordSets) { + const writers = new Map() + let last: RunEntry | null = null + let n = 0 + return { + async place(e: RunEntry, line: number): Promise { + if (last && compareRunKeys(e, last) < 0) throw new OutOfKeyOrder(line) + last = e + const set = setOf(e) + let w = writers.get(set) + if (!w) { + w = records[set].uploadWriter!(hooksFor(ports, sessionId, set)) + writers.set(set, w) + } + e.l = w.append({ type: e.t, id: e.k, hash: e.h!, line: e.b! }) + delete e.b + if (++n % 256 === 0) await w.ready() + }, + async finish(): Promise { + for (const w of writers.values()) await w.finish() + }, + } +} /** * List a session's packs in the collection's manifest or registry. Called once the @@ -96,14 +133,16 @@ export async function adoptSessionPacks(ports: Ports, repo: Repo, sessionId: str /** * At storage cleanup, settle a finished session's packs, then drop its rows: - * - a version names the session: its commit listed them, nothing to do; + * - a pack with no stats was never reported written, so no recorded run places a + * record in it: deleted, whatever became of the session; + * - a version names the session: its commit listed the rest, nothing to do; * - no commit of it ever started: nothing indexes their records, so they're deleted; * - a commit started and never published: its index may name their records (one the * collection already held, say, whose newest place is now such a pack), so they're * listed instead, and reclaim drops what no version uses. - * Returns the objects and bytes deleted (a pack is two objects), counting packs that - * were written. A collection whose repository won't open (deleted) is left to the - * sweep, which drops its files. + * A pack the collection lists is never deleted here (KeyedLayout.dropPacks). Returns + * the objects and bytes deleted (a pack is two objects). A collection whose + * repository won't open (deleted) is left to the sweep, which drops its files. */ export async function settleSessionPacks( ports: Ports, @@ -127,28 +166,25 @@ export async function settleSessionPacks( } catch { repo = null } - if (!published && repo?.records) { - if (session.commitStarted) { - if (!dryRun) await adoptSessionPacks(ports, repo, session.id) - } else { - for (const set of SETS) { - const mine = rows.filter((r) => r.set === set) - if (!mine.length) continue - // Refused if a commit listed them after all (KeyedLayout.dropPacks). - const dropped = - dryRun || - (await repo.records[set].dropPacks!( + if (repo?.records) { + const keepWritten = !!published || session.commitStarted + if (!published && session.commitStarted && !dryRun) + await adoptSessionPacks(ports, repo, session.id) + for (const set of SETS) { + const drop = rows.filter((r) => r.set === set && (r.records === null || !keepWritten)) + if (!drop.length) continue + const ids = dryRun + ? drop.map((r) => r.packId) + : await repo.records[set].dropPacks!( session.id, - mine.map((r) => r.packId), - )) - if (!dropped) continue - // A row without stats is a pack never written, or one whose upload died writing it. - for (const r of mine) - if (r.records !== null) { - gone.objects += 2 - gone.bytes += r.bytes ?? 0 - } - } + drop.map((r) => r.packId), + ) + const dropped = new Set(ids) + for (const r of drop) + if (dropped.has(r.packId) && r.records !== null) { + gone.objects += 2 + gone.bytes += r.bytes ?? 0 + } } } if (!dryRun) diff --git a/packages/server/src/push/session.ts b/packages/server/src/push/session.ts index f9f286eb..03a206c2 100644 --- a/packages/server/src/push/session.ts +++ b/packages/server/src/push/session.ts @@ -6,7 +6,7 @@ * list. */ import { checkSchemaFull, hashSchema } from '@underlay/protocol' -import { and, asc, count, eq, gt, or, sql } from 'drizzle-orm' +import { and, asc, count, eq, gt, gte, lte, ne, or, sql } from 'drizzle-orm' import * as schema from '../db/schema.js' import type { Ports } from '../ports.js' @@ -44,8 +44,6 @@ export interface SessionInputs { export type SessionRow = typeof schema.pushSessions.$inferSelect -type BatchStatement = Parameters[0][number] - const inputsKey = (id: string) => `sessions/${id}/inputs.json` export async function createSession( @@ -115,8 +113,6 @@ export async function recordRun( kind: 'records' | 'deletes', index: RunIndex, counters: { records?: number }, - /** More statements for the same batch (packs.ts: the upload's packs' stats). */ - extra: BatchStatement[] = [], ): Promise { const count = runCount(index) await ports.db.batch([ @@ -137,11 +133,40 @@ export async function recordRun( recordsReceived: sql`${schema.pushSessions.recordsReceived} + ${counters.records ?? 0}`, }) .where(eq(schema.pushSessions.id, sessionId)), - ...extra, ]) + if (kind === 'records' && index.parts.length) await noteOverlap(ports, sessionId, index) await maybeCompact(ports, sessionId, index.tier ?? 0) } +/** + * Count an upload whose key range overlaps another run of the session's: checked + * once the upload's own run is recorded, so of two that land together the later + * check sees the earlier (a compacted run's range spans its inputs', gaps included). + */ +async function noteOverlap(ports: Ports, sessionId: string, index: RunIndex): Promise { + const r = schema.pushRuns + const first = index.parts[0]!.first + const last = index.parts[index.parts.length - 1]!.last + const [hit] = await ports.db + .select({ seq: r.seq }) + .from(r) + .where( + and( + eq(r.sessionId, sessionId), + eq(r.kind, 'records'), + ne(r.seq, index.seq), + lte(r.firstKey, last), + gte(r.lastKey, first), + ), + ) + .limit(1) + if (!hit) return + await ports.db + .update(schema.pushSessions) + .set({ overlappingUploads: sql`${schema.pushSessions.overlappingUploads} + 1` }) + .where(eq(schema.pushSessions.id, sessionId)) +} + export async function sessionRuns( ports: Ports, sessionId: string, diff --git a/packages/server/test/upload-packs.test.ts b/packages/server/test/upload-packs.test.ts index 83f04eb9..2c3d7ba9 100644 --- a/packages/server/test/upload-packs.test.ts +++ b/packages/server/test/upload-packs.test.ts @@ -41,16 +41,20 @@ async function json(res: Response) { return (await res.json()) as any } -async function open(h: Harness, user: string, slug: string) { +async function open(h: Harness, user: string, slug: string, extra: object = {}) { const res = await h.request(`/api/collections/org/${slug}/push`, { method: 'POST', user, - json: { schemas: { Doc } }, + json: { schemas: { Doc }, ...extra }, }) expect(res.status).toBe(200) return (await json(res)).session_id as string } +async function status(h: Harness, user: string, slug: string, sid: string) { + return json(await h.request(`/api/collections/org/${slug}/push/${sid}`, { user })) +} + async function upload(h: Harness, user: string, slug: string, sid: string, batch: object[]) { const r = await h.request(`/api/collections/org/${slug}/push/${sid}/records`, { method: 'POST', @@ -362,4 +366,88 @@ describe.runIf(process.env.UNDERLAY_TEST_LAYOUT === 'f')('uploads into layout F' expect((await layout.manifests.latest()).adopted).toContain(sid) expect((await headRecords(h, c.id)).out).toEqual(['public same unchanged']) }) + describe('in key order', () => { + // A batch's records by id, from `from`; ids sort as their numbers do. + const sorted = (from: number, n: number, title = 'r') => + Array.from({ length: n }, (_, i) => doc(id(from + i), `${title}${from + i}`)) + + it('stream into whole packs, take larger batches, and count no overlaps', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('k', { recordLayout: 'f' }) + const res = await h.request('/api/collections/org/k/push', { + method: 'POST', + user, + json: { schemas: { Doc }, in_key_order: true }, + }) + const opened = await json(res) + expect(opened.in_key_order).toBe(true) + expect(opened.limits).toMatchObject({ batch_bytes: 64 * 1024 * 1024, batch_lines: 50_000 }) + const sid = opened.session_id as string + // One batch over the default 8 MB: a single pack, its records together. + const big = (from: number) => + Array.from({ length: 1100 }, (_, i) => doc(id(from + i), `${'z'.repeat(8000)}${from + i}`)) + await upload(h, user, 'k', sid, big(0)) + expect(packKeys(h, c.id)).toHaveLength(2) + await upload(h, user, 'k', sid, sorted(5000, 50)) + expect((await status(h, user, 'k', sid)).overlapping_uploads).toBe(0) + expect(await commit(h, user, 'k', sid)).toBe(201) + const { out } = await headRecords(h, c.id) + expect(out).toHaveLength(1150) + expect(out[1100]).toBe(`public ${id(5000)} r5000`) + }) + + it('refuse a batch out of order, and cleanup deletes what it packed', async () => { + const h = await harness() + const user = await h.member() + const c = await h.collection('o', { recordLayout: 'f' }) + const sid = await open(h, user, 'o', { in_key_order: true }) + const batch = [...sorted(0, 300), doc(id(10), 'behind')] + const r = await h.request(`/api/collections/org/o/push/${sid}/records`, { + method: 'POST', + user, + ndjson: batch, + }) + expect(r.status).toBe(422) + expect((await json(r)).error).toMatch(/out of key order at line 301/) + // What it packed before the refusal is reported, for cleanup. + const rows = await h.ports.db + .select() + .from(schema.pushPacks) + .where(eq(schema.pushPacks.sessionId, sid)) + expect(rows.length).toBeGreaterThan(0) + expect(rows.every((row) => row.records !== null)).toBe(true) + expect((await status(h, user, 'o', sid)).records_received).toBe(0) + await h.request(`/api/collections/org/o/push/${sid}`, { method: 'DELETE', user }) + await h.ports.db + .update(schema.pushSessions) + .set({ expiresAt: new Date(Date.now() - cleanupConfig.sessionGraceMs - DAY) }) + await startRun(h.ports, 'internal', { trigger: 'manual' }) + await h.drain() + expect(packKeys(h, c.id)).toEqual([]) + }) + + it('is counted when uploads overlap, and ignored on today’s layout', async () => { + const h = await harness() + const user = await h.member() + await h.collection('ov', { recordLayout: 'f' }) + const sid = await open(h, user, 'ov') + await upload(h, user, 'ov', sid, sorted(0, 100)) + await upload(h, user, 'ov', sid, sorted(200, 100)) + expect((await status(h, user, 'ov', sid)).overlapping_uploads).toBe(0) + // Ids 50–149: inside the first upload's range. + await upload(h, user, 'ov', sid, sorted(50, 100)) + expect((await status(h, user, 'ov', sid)).overlapping_uploads).toBe(1) + + await h.collection('t', { recordLayout: 'today' }) + const res = await h.request('/api/collections/org/t/push', { + method: 'POST', + user, + json: { schemas: { Doc }, in_key_order: true }, + }) + const opened = await json(res) + expect(opened.in_key_order).toBe(false) + expect(opened.limits.batch_bytes).toBe(8 * 1024 * 1024) + }) + }) })