diff --git a/.github/workflows/desktop-storage-soak.yml b/.github/workflows/desktop-storage-soak.yml index aca44f4..9dfc938 100644 --- a/.github/workflows/desktop-storage-soak.yml +++ b/.github/workflows/desktop-storage-soak.yml @@ -55,8 +55,8 @@ jobs: matrix: os: [macos-26, windows-latest, ubuntu-24.04] group: - - name: contract, T3–T6 - cases: contract,T3,T4,T5,T6 + - name: contract, T3–T6, quit during a copy + cases: contract,T3,T4,T4-copy,T5,T6 - name: T1 kill during append cases: T1,T1-dense - name: T2 kill during compaction diff --git a/CHANGELOG.md b/CHANGELOG.md index 7cb26d2..c7066a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,34 @@ since the previous tag. ## [Unreleased] +### Desktop P2.6 decision 5: daily copies never hold up a save or a quit + +A daily `VACUUM INTO` on the storage worker blocked every save behind it, for +up to 9 s on a Windows runner. That included the quit flush, so a quit during +a copy could drop the final batch of edits. The copy now runs in its own +thread and can be stopped. + +#### Changed + +- **Desktop:** the daily copy (`desktop/storage/engine/copy-library.ts`) + - copies the library in batches of at most 256 rows or about 4 MiB, from + one read-only snapshot, inside one transaction; + - creates the schema verbatim and copies `sqlite_sequence`, + `user_version` and `application_id`; + - checks row counts and `integrity_check` before the copy is renamed into + place, then runs a passive WAL checkpoint; + - on failure, keeps the previous copies and records `daily_copy_failed`. +- **Desktop:** the storage worker runs that copy in a second thread + (`desktop/storage/copy-worker.ts`), so saves never wait on it. On quit it + stops the copy in milliseconds and deletes the partial file; the previous + copy stays. Opening the library sweeps any partial copy left behind. +- **Tests:** soak case `T4-copy` quits during a copy on all three OSes. T8 + compares the copy's time and size with `VACUUM INTO`'s on the year-end + library. A unit test restores a copy and runs the `DocStore` contract on it. +- **Docs:** the architecture spec records the quit budget (3 s flush, then + 2 s worker shutdown), the batched copy, and `utilityProcess` as the + upgrade path once P2.5 exists. + ### Desktop P2.6: the library storage worker The SQLite engine behind the desktop library, where documents, folders and diff --git a/desktop/dev/bun-sqlite-driver.ts b/desktop/dev/bun-sqlite-driver.ts index 4e91520..08473de 100644 --- a/desktop/dev/bun-sqlite-driver.ts +++ b/desktop/dev/bun-sqlite-driver.ts @@ -8,7 +8,7 @@ * connection with live statements open as a zombie that keeps the file * handles, which on Windows blocks deleting or replacing the library. */ -import { Database } from "bun:sqlite"; +import { constants, Database } from "bun:sqlite"; import type { OpenSqlite, @@ -19,9 +19,16 @@ import type { } from "@/desktop/storage/engine/driver"; export const openBunSqlite: OpenSqlite = (path, { readOnly }) => { - const db = readOnly - ? new Database(path, { readonly: true }) - : new Database(path, { create: true }); + // SQLITE_OPEN_URI: the daily copy ATTACHes the library as + // `file:…?mode=ro`. node:sqlite accepts URIs by default; Bun's bundled + // SQLite (Linux, Windows) only with this flag. + const db = new Database( + path, + constants.SQLITE_OPEN_URI | + (readOnly + ? constants.SQLITE_OPEN_READONLY + : constants.SQLITE_OPEN_READWRITE | constants.SQLITE_OPEN_CREATE), + ); const cache = new Map(); const statements: { finalize(): void }[] = []; const driver: SqliteDriver = { diff --git a/desktop/storage-harness/build.ts b/desktop/storage-harness/build.ts index 194b77b..79d3f5c 100644 --- a/desktop/storage-harness/build.ts +++ b/desktop/storage-harness/build.ts @@ -10,6 +10,7 @@ const result = await Bun.build({ entrypoints: [ path.join(root, "main.ts"), path.join(root, "../storage/worker.ts"), + path.join(root, "../storage/copy-worker.ts"), ], outdir: path.join(root, "dist"), target: "node", diff --git a/desktop/storage-harness/main.ts b/desktop/storage-harness/main.ts index 2971574..e04544b 100644 --- a/desktop/storage-harness/main.ts +++ b/desktop/storage-harness/main.ts @@ -17,6 +17,8 @@ * - `readonly`: opens a newer library (T6) and tries to write. * - `commit-timing`: append round trips with `fullfsync` on or off (T3). * - `startup`: worker spawn → ready → sidebar rows (T8, the §10 budget). + * - `copy-quit`: quit while the daily copy runs in its thread (decision 5). + * - `copy-time`: how long the batched daily copy takes (decision 5). * * An uncaught throw in Electron main opens a modal dialog, so every failure * exits with `process.exit(1)` after printing `HARNESS_FAIL`. @@ -279,6 +281,87 @@ async function commitTiming() { await client.shutdown(); } +/** + * Decision 5: quit while the daily copy runs in its thread. Waits for the + * library to go idle (the copy starts), keeps typing through the copy (the + * writer must not wait on it), then runs the quit flush. + */ +async function copyQuit() { + const docId = args.doc ?? fail("--doc is required"); + const client = await StorageClient.start(workerPath(), { + dir: dir(), + trace: true, + idleMs: 300, + idlePollMs: 100, + }); + const owner = "copy-quit"; + const loaded = await client.call("load", { docId, owner }); + const doc = new Y.Doc(); + if (loaded.snapshot) Y.applyUpdate(doc, loaded.snapshot); + for (const update of loaded.updates) Y.applyUpdate(doc, update); + let k = + editCount(doc.getText(TEXT).toString()) ?? fail("stored text is broken"); + say(`READY ${k}`); + // Stay idle until the copy thread is running, then type through it. + const waitUntil = performance.now() + 10_000; + while (!(await client.call("stats", {})).dailyCopyRunning) { + if (performance.now() > waitUntil) fail("the daily copy never started"); + await new Promise((r) => setTimeout(r, 10)); + } + const appendMs: number[] = []; + const typeFor = Number(args["type-ms"] ?? 150); + const started = performance.now(); + while (performance.now() - started < typeFor) { + k += 1; + let update: Uint8Array = new Uint8Array(); + doc.once("update", (u: Uint8Array) => { + update = u; + }); + const text = doc.getText(TEXT); + text.insert(text.length, `${k};`); + const t0 = performance.now(); + await client.call("append", { + docId, + owner, + record: { origin: "local", update }, + }); + appendMs.push(performance.now() - t0); + await new Promise((r) => setTimeout(r, 20)); + } + const quitAt = performance.now(); + const outcome = await client.shutdown(); + say( + `QUIT ${outcome} ${k} ${Math.round(performance.now() - quitAt)} ${Math.round(Math.max(...appendMs))}`, + ); +} + +/** Decision 5: how long the batched daily copy takes on node:sqlite. */ +async function copyTime() { + const client = await StorageClient.start(workerPath(), { + dir: dir(), + idleMs: 0, + idlePollMs: 50, + }); + const today = (await client.call("stats", {})).today; + const deadline = performance.now() + 300_000; + for (;;) { + const stats = await client.call("stats", {}); + if (stats.lastDailyCopy === today || stats.lastDailyCopyError) { + say( + `COPYTIME ${JSON.stringify({ + ms: stats.lastDailyCopyMs, + error: stats.lastDailyCopyError, + libraryBytes: stats.bytes, + })}`, + ); + break; + } + if (performance.now() > deadline) fail("the daily copy never finished"); + await new Promise((r) => setTimeout(r, 100)); + } + await client.shutdown(); +} + async function startup() { // performance.now() counts from process start in Node and Electron main. const spawnAt = performance.now(); @@ -310,6 +393,8 @@ const MODES: Record Promise> = { readonly, "commit-timing": commitTiming, startup, + "copy-quit": copyQuit, + "copy-time": copyTime, }; app.whenReady().then(async () => { diff --git a/desktop/storage/client.ts b/desktop/storage/client.ts index 42129d8..890f510 100644 --- a/desktop/storage/client.ts +++ b/desktop/storage/client.ts @@ -8,6 +8,7 @@ * pending call rejects with `worker_restarting`, which the renderer's * batcher retries (resending an append is harmless). */ +import { dirname, join } from "node:path"; import { Worker } from "node:worker_threads"; import type { StorageErrorEnvelope } from "./engine/errors"; @@ -86,7 +87,12 @@ export class StorageClient { events: StorageClientEvents = {}, ): Promise { const started = performance.now(); - const worker = new Worker(workerPath, { workerData: options }); + const worker = new Worker(workerPath, { + workerData: { + copyWorkerPath: join(dirname(workerPath), "copy-worker.cjs"), + ...options, + }, + }); return new Promise((resolve, reject) => { const onError = (error: Error) => reject(error); worker.once("error", onError); diff --git a/desktop/storage/copy-worker.ts b/desktop/storage/copy-worker.ts new file mode 100644 index 0000000..e5b5359 --- /dev/null +++ b/desktop/storage/copy-worker.ts @@ -0,0 +1,36 @@ +/** + * The daily-copy thread (P2.6 decision 5). The storage worker starts one per + * copy; it reads the library through its own read-only connection, so the + * writer never waits on it, and it yields between batches, so the storage + * worker can `terminate()` it on quit within milliseconds. + * + * Replies once: `{ ok: true, report, ms }` or `{ ok: false, code, message }`. + */ +import { parentPort, workerData } from "node:worker_threads"; + +import { openNodeSqlite } from "./drivers/node-sqlite"; +import { copyLibraryBatched } from "./engine/copy-library"; +import { toEnvelope } from "./engine/errors"; + +export type CopyWorkerData = { + source: string; + partial: string; + batchRows?: number; +}; + +const port = parentPort; +if (!port) throw new Error("the copy worker must run in a worker_thread"); +const { source, partial, batchRows } = workerData as CopyWorkerData; +const started = performance.now(); + +copyLibraryBatched(openNodeSqlite, source, partial, { + batchRows, + yieldNow: () => new Promise((resolve) => setImmediate(resolve)), +}).then( + (report) => + port.postMessage({ ok: true, report, ms: performance.now() - started }), + (error) => { + const { code, message } = toEnvelope(error); + port.postMessage({ ok: false, code, message }); + }, +); diff --git a/desktop/storage/engine/copy-library.ts b/desktop/storage/engine/copy-library.ts new file mode 100644 index 0000000..1394888 --- /dev/null +++ b/desktop/storage/engine/copy-library.ts @@ -0,0 +1,223 @@ +/** + * The daily copy, made in batches so it can be stopped (P2.6 decision 5). + * + * `VACUUM INTO` is one native call: a worker thread running it can't be + * terminated, and Node joins worker threads on exit, so a quit during a copy + * waited for it (9 s for a 240 MB library on a Windows runner). This copy + * yields between batches, where `Worker.terminate()` takes effect. + * + * It must be as trustworthy as `VACUUM INTO`: + * - **one snapshot:** the library is attached read-only (`mode=ro`) to the + * copy's connection, and the whole copy is one transaction on that + * connection, so every batch reads the same snapshot of the library + * while the writer keeps committing (WAL); + * - **the schema verbatim:** tables from the library's `sqlite_master` + * first, then the rows, then indexes and triggers; `sqlite_sequence`, + * `user_version` and `application_id` are copied (`format_version` lives + * in `meta`, a table); + * - **verified before it is kept:** row counts per table against the + * snapshot and `integrity_check`, inside the transaction; any mismatch + * rolls back and nothing is committed. The caller renames the file into + * place only after this resolves. + * + * The result keeps every row, but isn't compacted the way `VACUUM INTO`'s + * is: free pages of the library are simply not copied, so sizes stay close. + */ +import type { OpenSqlite, SqliteDriver } from "./driver"; +import { StorageError } from "./errors"; +import { scalar } from "./sql"; + +export type CopyOptions = { + /** At most this many rows per batch; the copy yields after each one. */ + batchRows?: number; + /** + * At most about this many bytes per batch (a single larger row is its own + * batch). One batch is one native statement that `terminate()` can't + * interrupt, so this bounds how long a quit waits: 4 MiB is a few ms. + */ + batchBytes?: number; + /** Lets the event loop run (and `terminate()` land) between batches. */ + yieldNow: () => Promise; + /** Tests only: tamper with the copy before it is verified. */ + beforeVerify?: (dest: SqliteDriver) => void; +}; + +export type CopyReport = { tables: number; rows: number; batches: number }; + +/** A read-only SQLite URI for `path`, on every OS. */ +export function readOnlyUri(path: string): string { + const slashed = path.replace(/\\/g, "/"); + const encoded = encodeURI(slashed).replace(/\?/g, "%3F").replace(/#/g, "%23"); + return `file:${slashed.startsWith("/") ? "" : "///"}${encoded}?mode=ro`; +} + +const quote = (name: string) => `"${name.replace(/"/g, '""')}"`; + +type SchemaObject = { type: string; name: string; sql: string }; + +/** An SQL expression for a row's stored size, from every column's length. */ +function rowSize(db: SqliteDriver, table: string): string { + const columns = db + .prepare(`PRAGMA lib.table_info(${quote(table)})`) + .all() + .map((column) => `COALESCE(LENGTH(${quote(String(column.name))}), 0)`); + return columns.length > 0 ? columns.join(" + ") : "0"; +} + +/** Primary-key columns of a WITHOUT ROWID table, in key order. */ +function keyColumns(db: SqliteDriver, table: string): string[] { + return db + .prepare(`PRAGMA lib.table_info(${quote(table)})`) + .all() + .filter((column) => Number(column.pk) > 0) + .sort((a, b) => Number(a.pk) - Number(b.pk)) + .map((column) => quote(String(column.name))); +} + +async function copyRowidTable( + db: SqliteDriver, + table: string, + { batchRows, batchBytes }: { batchRows: number; batchBytes: number }, + yieldNow: () => Promise, +): Promise { + const name = quote(table); + // LENGTH() of a BLOB or TEXT comes from the record header: no payload read. + const sizes = db.prepare( + `SELECT rowid AS r, ${rowSize(db, table)} AS b FROM lib.${name} WHERE rowid > ? ORDER BY rowid LIMIT ?`, + ); + const insert = db.prepare( + `INSERT INTO main.${name} SELECT * FROM lib.${name} WHERE rowid > ? AND rowid <= ?`, + ); + let cursor = Number.MIN_SAFE_INTEGER; + let batches = 0; + for (;;) { + let last: number | null = null; + let bytes = 0; + for (const row of sizes.all(cursor, batchRows)) { + if (last !== null && bytes + Number(row.b) > batchBytes) break; + last = Number(row.r); + bytes += Number(row.b); + } + if (last === null) return batches; + insert.run(cursor, last); + cursor = last; + batches += 1; + await yieldNow(); + } +} + +async function copyKeyedTable( + db: SqliteDriver, + table: string, + { batchRows }: { batchRows: number; batchBytes: number }, + yieldNow: () => Promise, +): Promise { + const name = quote(table); + const order = keyColumns(db, table).join(", ") || "1"; + const insert = db.prepare( + `INSERT INTO main.${name} SELECT * FROM lib.${name} ORDER BY ${order} LIMIT ? OFFSET ?`, + ); + let batches = 0; + for (let offset = 0; ; offset += batchRows) { + if (insert.run(batchRows, offset).changes === 0) return batches; + batches += 1; + await yieldNow(); + } +} + +function count(db: SqliteDriver, schema: "main" | "lib", table: string) { + return Number(scalar(db, `SELECT COUNT(*) FROM ${schema}.${quote(table)}`)); +} + +/** + * Copies the library at `source` into a new file at `partial`. Resolves + * once the copy is committed and verified; rejects (with nothing committed) + * on a mismatch, an I/O error, or a throw from `yieldNow`. + */ +export async function copyLibraryBatched( + openSqlite: OpenSqlite, + source: string, + partial: string, + { + batchRows = 256, + batchBytes = 4 * 1024 * 1024, + yieldNow, + beforeVerify, + }: CopyOptions, +): Promise { + const dest = openSqlite(partial, { readOnly: false }); + let attached = false; + let open = false; + try { + dest.exec("PRAGMA journal_mode = DELETE"); + dest.exec("PRAGMA synchronous = FULL"); + dest.prepare("ATTACH DATABASE ? AS lib").run(readOnlyUri(source)); + attached = true; + dest.exec("BEGIN"); + open = true; + // The first read pins the library's snapshot for the whole copy. + const objects = dest + .prepare( + "SELECT type, name, sql FROM lib.sqlite_master WHERE sql IS NOT NULL ORDER BY rowid", + ) + .all() as SchemaObject[]; + const tables = objects.filter( + (o) => o.type === "table" && !o.name.startsWith("sqlite_"), + ); + for (const table of tables) dest.exec(table.sql); + for (const pragma of ["user_version", "application_id"]) { + const value = Number(scalar(dest, `PRAGMA lib.${pragma}`) ?? 0); + dest.exec(`PRAGMA main.${pragma} = ${Math.trunc(value)}`); + } + const report: CopyReport = { tables: tables.length, rows: 0, batches: 0 }; + for (const table of tables) { + const keyed = /\bWITHOUT\s+ROWID\b/i.test(table.sql); + report.batches += await (keyed ? copyKeyedTable : copyRowidTable)( + dest, + table.name, + { batchRows, batchBytes }, + yieldNow, + ); + } + const hasSequence = objects.some((o) => o.name === "sqlite_sequence"); + if (hasSequence) { + dest.exec("DELETE FROM main.sqlite_sequence"); + dest.exec( + "INSERT INTO main.sqlite_sequence (name, seq) SELECT name, seq FROM lib.sqlite_sequence", + ); + } + for (const object of objects) + if (object.type !== "table" && !object.name.startsWith("sqlite_")) + dest.exec(object.sql); + beforeVerify?.(dest); + for (const table of tables) { + const copied = count(dest, "main", table.name); + if (copied !== count(dest, "lib", table.name)) + throw new StorageError("corrupt", `copy of ${table.name} is short`); + report.rows += copied; + } + if ( + hasSequence && + count(dest, "main", "sqlite_sequence") !== + count(dest, "lib", "sqlite_sequence") + ) + throw new StorageError("corrupt", "copy of sqlite_sequence is short"); + const check = String(scalar(dest, "PRAGMA main.integrity_check")); + if (check !== "ok") + throw new StorageError( + "corrupt", + `copy failed integrity_check: ${check}`, + ); + dest.exec("COMMIT"); + open = false; + return report; + } finally { + try { + if (open) dest.exec("ROLLBACK"); + if (attached) dest.exec("DETACH DATABASE lib"); + } catch { + // The connection closes below either way. + } + dest.close(); + } +} diff --git a/desktop/storage/engine/daily-copy.ts b/desktop/storage/engine/daily-copy.ts index 884a4a7..6cb43d1 100644 --- a/desktop/storage/engine/daily-copy.ts +++ b/desktop/storage/engine/daily-copy.ts @@ -6,8 +6,13 @@ * library. * * Each copy is written to `.partial` and renamed, so a kill mid-copy - * never leaves a half file under a real name. Retention is by count, so a - * clock jump can't delete good copies early. + * never leaves a half file under a real name; the next open sweeps it. + * Retention is by count, so a clock jump can't delete good copies early. + * + * The copy itself is `copy-library.ts`: batched, from one read-only snapshot, + * verified before it is kept. The worker runs it in a second thread + * (`copy-worker.ts`) so the writer never waits on it, and stops that thread + * on quit. */ import type { SqliteDriver } from "./driver"; import type { StorageHost } from "./host"; @@ -39,31 +44,63 @@ export function isDailyCopyDue(db: SqliteDriver, now: number): boolean { return readMeta(db, "last_daily_copy") !== localDate(now); } -/** - * Writes today's copy if there isn't one yet. Returns the copy's file name, - * or null when it was already taken today. - */ -export function takeDailyCopy( +/** One daily copy in flight. */ +export type DailyCopyJob = { + date: string; + /** The library file the copy reads. */ + source: string; + /** Where `VACUUM INTO` writes; renamed to `target` once complete. */ + partial: string; + target: string; +}; + +/** Deletes leftover `.partial` files (a killed copy). Best effort. */ +export function sweepPartialCopies(host: StorageHost, dir: string) { + const folder = host.join(dir, DAILY_DIR); + for (const file of host.list(folder)) { + if (!file.endsWith(".partial")) continue; + try { + host.remove(host.join(folder, file)); + } catch { + // Still open elsewhere (Windows); the next open sweeps it. + } + } +} + +/** Today's copy job, or null when today's copy exists. */ +export function planDailyCopy( db: SqliteDriver, host: StorageHost, dir: string, + source: string, now: number, -): string | null { +): DailyCopyJob | null { if (!isDailyCopyDue(db, now)) return null; const date = localDate(now); const folder = host.join(dir, DAILY_DIR); host.mkdirp(folder); - for (const file of host.list(folder)) - if (file.endsWith(".partial")) host.remove(host.join(folder, file)); - const name = dailyCopyName(date); - const target = host.join(folder, name); - const partial = `${target}.partial`; - db.prepare("VACUUM INTO ?").run(partial); - if (host.exists(target)) host.remove(target); - host.rename(partial, target); - writeMeta(db, "last_daily_copy", date); - const copies = listDailyCopies(host, dir); - for (const old of copies.slice(0, -DAILY_KEEP)) + sweepPartialCopies(host, dir); + const target = host.join(folder, dailyCopyName(date)); + return { date, source, partial: `${target}.partial`, target }; +} + +/** + * Publishes a finished copy: rename into place, record the day, keep the + * newest {@link DAILY_KEEP}, then a passive checkpoint, since the copy's + * long read held the WAL from shrinking. + */ +export function finishDailyCopy( + db: SqliteDriver, + host: StorageHost, + dir: string, + job: DailyCopyJob, +): string { + if (host.exists(job.target)) host.remove(job.target); + host.rename(job.partial, job.target); + writeMeta(db, "last_daily_copy", job.date); + const folder = host.join(dir, DAILY_DIR); + for (const old of listDailyCopies(host, dir).slice(0, -DAILY_KEEP)) host.remove(host.join(folder, old)); - return name; + db.exec("PRAGMA wal_checkpoint(PASSIVE)"); + return dailyCopyName(job.date); } diff --git a/desktop/storage/engine/library.ts b/desktop/storage/engine/library.ts index e36025d..7e1b5f7 100644 --- a/desktop/storage/engine/library.ts +++ b/desktop/storage/engine/library.ts @@ -37,13 +37,16 @@ import { pendingRows, stateOf, } from "./content"; +import { copyLibraryBatched } from "./copy-library"; import { DAILY_DIR, + type DailyCopyJob, dailyCopyName, - isDailyCopyDue, + finishDailyCopy, listDailyCopies, localDate, - takeDailyCopy, + planDailyCopy, + sweepPartialCopies, } from "./daily-copy"; import type { OpenSqlite, SqliteDriver } from "./driver"; import { storageErrorFrom } from "./driver"; @@ -91,6 +94,8 @@ export type OpenLibraryOptions = { formatVersion?: number; formatSteps?: readonly FormatStep[]; trace?: Trace; + /** How long without a write counts as idle; default {@link IDLE_MS}. Soak only. */ + idleMs?: number; }; function applyPragmas( @@ -155,6 +160,9 @@ export class Library { private lastWriteAt: number; private changes = new Map(); private lastDailyCopyError: string | null = null; + private lastDailyCopyMs: number | null = null; + private copyJob: DailyCopyJob | null = null; + private readonly idleMs: number; private constructor(fields: { dir: string; @@ -165,6 +173,7 @@ export class Library { openSqlite: OpenSqlite; lockPath: string; trace: Trace | undefined; + idleMs: number; }) { this.dir = fields.dir; this.path = fields.path; @@ -174,6 +183,7 @@ export class Library { this.openSqlite = fields.openSqlite; this.lockPath = fields.lockPath; this.trace = fields.trace; + this.idleMs = fields.idleMs; this.lastWriteAt = fields.host.now(); } @@ -228,7 +238,10 @@ export class Library { status.formatVersion = readFormatVersion(db); } } - if (!status.readOnly) writeMeta(db, "unclean_exit", "1"); + if (!status.readOnly) { + writeMeta(db, "unclean_exit", "1"); + sweepPartialCopies(host, dir); + } } return new Library({ dir, @@ -239,6 +252,7 @@ export class Library { openSqlite, lockPath, trace: options.trace, + idleMs: options.idleMs ?? IDLE_MS, }); } catch (error) { try { @@ -484,48 +498,128 @@ export class Library { // ── Jobs and lifecycle ──────────────────────────────────────────────── + private isIdle(): boolean { + return ( + !this.status.readOnly && + this.db !== null && + this.host.now() - this.lastWriteAt >= this.idleMs + ); + } + /** - * Runs the idle jobs when nothing was written for {@link IDLE_MS}: today's - * daily copy, then Trash older than 30 days (skipping open documents). - * Returns what ran, for the worker's diagnostics. A failed daily copy is - * kept for `stats()` (app health `backups_failing`) and retried on the - * next idle pass; it never stops the purge. + * Starts today's daily copy when the library is idle and today has none: + * the caller runs `copyLibraryBatched` on `job` (the worker, in its copy + * thread), then calls {@link completeDailyCopy} or {@link failDailyCopy}. + * One job at a time; null otherwise. */ - runIdleJobs(): { + beginDailyCopy(): DailyCopyJob | null { + if (this.copyJob || !this.isIdle()) return null; + this.copyJob = planDailyCopy( + this.conn, + this.host, + this.dir, + this.path, + this.host.now(), + ); + if (this.copyJob) this.trace?.(`COPY_BEGIN ${this.copyJob.date}`); + return this.copyJob; + } + + completeDailyCopy(job: DailyCopyJob, ms: number | null = null): string { + if (job !== this.copyJob) + throw new StorageError("invalid_argument", "not the running copy"); + this.copyJob = null; + const name = finishDailyCopy(this.conn, this.host, this.dir, job); + this.lastDailyCopyError = null; + this.lastDailyCopyMs = ms; + this.trace?.(`COPY_DONE ${job.date}`); + return name; + } + + /** + * Drops a copy that failed or was abandoned (quit): its partial file is + * removed when it can be (the next open sweeps it otherwise), earlier + * copies stay, and the next idle pass tries again. + */ + failDailyCopy(job: DailyCopyJob, reason: string): void { + if (job === this.copyJob) this.copyJob = null; + try { + this.host.remove(job.partial); + } catch { + // Still open in the copy thread (Windows); swept on the next open. + } + this.lastDailyCopyError = reason; + this.trace?.(`COPY_ABANDONED ${job.date} ${reason}`); + } + + /** Purges Trash older than 30 days, skipping open documents. */ + purgeExpiredTrash(): number { + if (!this.isIdle()) return 0; + let purged = 0; + for (const id of index.expiredTrash(this.conn, this.host.now())) { + if (this.holders.has(id)) continue; + index.purgeDoc(this.conn, id); + this.changed("doc", id); + purged += 1; + } + return purged; + } + + /** + * Runs the idle jobs inline, on this thread: today's daily copy, then the + * Trash purge. Tests, the dev host and the soak's year use it; the worker + * runs the copy in its copy thread instead. + */ + async runIdleJobs(): Promise<{ dailyCopy: string | null; dailyCopyMs: number | null; purged: number; - } { - const now = this.host.now(); + }> { const result = { dailyCopy: null as string | null, dailyCopyMs: null as number | null, purged: 0, }; - if (this.status.readOnly || now - this.lastWriteAt < IDLE_MS) return result; - if (isDailyCopyDue(this.conn, now)) { + const job = this.beginDailyCopy(); + if (job) { const started = performance.now(); - try { - result.dailyCopy = takeDailyCopy(this.conn, this.host, this.dir, now); - this.lastDailyCopyError = null; - } catch (error) { - this.lastDailyCopyError = storageErrorFrom(error).code; - } + result.dailyCopy = await this.copyInline(job); result.dailyCopyMs = performance.now() - started; } - for (const id of index.expiredTrash(this.conn, now)) { - if (this.holders.has(id)) continue; - index.purgeDoc(this.conn, id); - this.changed("doc", id); - result.purged += 1; - } + result.purged = this.purgeExpiredTrash(); return result; } - /** Forces today's daily copy now (diagnostics, tests). */ - takeDailyCopyNow(): string | null { + private async copyInline(job: DailyCopyJob): Promise { + const started = performance.now(); + try { + await copyLibraryBatched(this.openSqlite, job.source, job.partial, { + yieldNow: () => new Promise((resolve) => setTimeout(resolve, 0)), + }); + return this.completeDailyCopy(job, performance.now() - started); + } catch (error) { + this.failDailyCopy( + job, + `daily_copy_failed: ${storageErrorFrom(error).code}`, + ); + return null; + } + } + + /** Forces today's daily copy now, ignoring idle (diagnostics, tests). */ + async takeDailyCopyNow(): Promise { this.assertWritable(); - return takeDailyCopy(this.conn, this.host, this.dir, this.host.now()); + if (this.copyJob) return null; + const job = planDailyCopy( + this.conn, + this.host, + this.dir, + this.path, + this.host.now(), + ); + if (!job) return null; + this.copyJob = job; + return this.copyInline(job); } stats() { @@ -542,6 +636,8 @@ export class Library { ), lastDailyCopy: readMeta(db, "last_daily_copy"), lastDailyCopyError: this.lastDailyCopyError, + lastDailyCopyMs: this.lastDailyCopyMs, + dailyCopyRunning: this.copyJob !== null, today: localDate(this.host.now()), // Read back, as set (D-A6; S2 confirmed them on all three OSes). pragmas: { @@ -564,6 +660,8 @@ export class Library { shutdown(): void { const db = this.db; if (!db) return; + // Quit never waits for a daily copy (its thread is the worker's to stop). + if (this.copyJob) this.failDailyCopy(this.copyJob, "quit"); try { if (!this.status.readOnly) { writeMeta(db, "unclean_exit", "0"); diff --git a/desktop/storage/protocol.ts b/desktop/storage/protocol.ts index 1d238cd..4f211be 100644 --- a/desktop/storage/protocol.ts +++ b/desktop/storage/protocol.ts @@ -58,10 +58,6 @@ export type StorageOps = { setDocTags: { args: { docId: string; tagIds: string[] }; result: number }; listDailyCopies: { args: Record; result: string[] }; readDailyCopy: { args: { date: string; docId: string }; result: Uint8Array }; - runIdleJobs: { - args: Record; - result: ReturnType; - }; stats: { args: Record; result: ReturnType }; /** The quit flush's last step: checkpoint, clear `unclean_exit`, close. */ shutdown: { args: Record; result: null }; @@ -95,6 +91,13 @@ export type StorageWorkerOptions = { trace?: boolean; /** How often the worker looks for idle work, in ms (default 5000). */ idlePollMs?: number; + /** Soak only: how long without a write counts as idle (default 30 s). */ + idleMs?: number; + /** + * The daily-copy thread's bundle (`copy-worker.cjs`). `StorageClient` + * defaults it to the worker's sibling. Without it, no daily copy runs. + */ + copyWorkerPath?: string; }; function run(library: Library, request: StorageRequest): unknown { @@ -155,8 +158,6 @@ function run(library: Library, request: StorageRequest): unknown { return library.listDailyCopies(); case "readDailyCopy": return library.readDailyCopy(request.args.date, request.args.docId); - case "runIdleJobs": - return library.runIdleJobs(); case "stats": return library.stats(); case "shutdown": diff --git a/desktop/storage/worker.ts b/desktop/storage/worker.ts index f402001..469ba71 100644 --- a/desktop/storage/worker.ts +++ b/desktop/storage/worker.ts @@ -6,17 +6,20 @@ * behind appends runs after they have committed. * * Messages: `ready` (or `open_failed`) once, then one `reply` per request, - * and `index_changed` after a metadata write. Idle jobs (daily copy, Trash) - * run between requests after 30 s without a write. + * and `index_changed` after a metadata write. Idle jobs run between + * requests after 30 s without a write: the Trash purge here, and the daily + * copy in its own thread (`copy-worker.ts`), so a long copy never holds an + * append or the quit flush. * * With `trace`, the engine's protocol lines (`BATCH_BEGIN`, `ACK `, * `COMPACT_BEGIN`, `COMPACT_ACK`) go straight to fd 1 with `writeSync`, so * the durability soak knows what committed before it kills the process. */ import { writeSync } from "node:fs"; -import { parentPort, workerData } from "node:worker_threads"; +import { parentPort, Worker, workerData } from "node:worker_threads"; import { openNodeSqlite } from "./drivers/node-sqlite"; +import type { DailyCopyJob } from "./engine/daily-copy"; import { toEnvelope } from "./engine/errors"; import { Library } from "./engine/library"; import { createNodeHost } from "./node-host"; @@ -28,8 +31,9 @@ import { transferablesOf, } from "./protocol"; +if (!parentPort) + throw new Error("the storage worker must run in a worker_thread"); const port = parentPort; -if (!port) throw new Error("the storage worker must run in a worker_thread"); const options = workerData as StorageWorkerOptions; @@ -56,6 +60,7 @@ try { fullfsync: options.fullfsync, synchronous: options.synchronous, trace: options.trace ? line : undefined, + idleMs: options.idleMs, }); post({ type: "ready", @@ -72,14 +77,71 @@ function announceChanges(open: Library) { if (changed) post({ type: "index_changed", ...changed }); } +/** The daily copy running in its own thread, if any (decision 5). */ +let copy: { job: DailyCopyJob; thread: Worker } | null = null; + +function startCopy(open: Library, job: DailyCopyJob, copyWorkerPath: string) { + const thread = new Worker(copyWorkerPath, { + workerData: { source: job.source, partial: job.partial }, + }); + copy = { job, thread }; + const settle = (reason: string | null, ms?: number) => { + if (copy?.thread !== thread) return; + copy = null; + if (!library) return; // shut down meanwhile; shutdown abandoned the job + try { + if (reason === null) { + open.completeDailyCopy(job, ms ?? null); + } else open.failDailyCopy(job, reason); + } catch (error) { + open.failDailyCopy(job, `daily_copy_failed: ${toEnvelope(error).code}`); + } + announceChanges(open); + }; + thread.once("message", (reply: { ok: boolean; code?: string; ms?: number }) => + settle(reply.ok ? null : `daily_copy_failed: ${reply.code}`, reply.ms), + ); + thread.once("error", (error) => + settle(`daily_copy_failed: ${String(error)}`), + ); + thread.once("exit", (code) => settle(`daily_copy_failed: exit ${code}`)); +} + +/** + * Stops a running copy for quit (decision 5): the thread yields between + * batches, so this takes milliseconds, and waiting for it closes the + * partial file before `shutdown` deletes it. Never longer than 500 ms. + */ +async function stopCopy() { + const running = copy; + if (!running) return; + copy = null; + const started = performance.now(); + let guard: ReturnType | undefined; + const outcome = await Promise.race([ + running.thread.terminate().then(() => "stopped"), + new Promise((resolve) => { + guard = setTimeout(resolve, 500, "timed_out"); + }), + ]); + // A pending timer would keep this thread alive after shutdown. + clearTimeout(guard); + if (options.trace) + line(`COPY_STOPPED ${outcome} ${Math.round(performance.now() - started)}`); +} + let idleTimer: ReturnType | null = null; if (library) { const open = library; idleTimer = setInterval(() => { try { - open.runIdleJobs(); + if (!copy && options.copyWorkerPath) { + const job = open.beginDailyCopy(); + if (job) startCopy(open, job, options.copyWorkerPath); + } + open.purgeExpiredTrash(); } catch { - // Only the Trash purge can throw here; it retries on the next pass. + // A failed purge or copy start retries on the next pass. } announceChanges(open); }, options.idlePollMs ?? 5000); @@ -103,13 +165,25 @@ port.on("message", (request: StorageRequest) => { return; } const open = library; - const result = dispatch(open, request); - post({ type: "reply", id: request.id, result }); if (request.op === "shutdown") { - if (idleTimer) clearInterval(idleTimer); - library = null; - port.close(); + void shutDown(open, request); return; } + const result = dispatch(open, request); + post({ type: "reply", id: request.id, result }); announceChanges(open); }); + +/** + * The quit flush's last step. Every request sent before it has committed: + * the writer never waited on the daily copy. Stop the copy, then + * checkpoint, clear `unclean_exit`, close and release the lock. + */ +async function shutDown(open: Library, request: StorageRequest) { + if (idleTimer) clearInterval(idleTimer); + await stopCopy(); + const result = dispatch(open, request); + library = null; + post({ type: "reply", id: request.id, result }); + port.close(); +} diff --git a/docs/superpowers/specs/2026-11-15-desktop-architecture.md b/docs/superpowers/specs/2026-11-15-desktop-architecture.md index a477516..2022e0b 100644 --- a/docs/superpowers/specs/2026-11-15-desktop-architecture.md +++ b/docs/superpowers/specs/2026-11-15-desktop-architecture.md @@ -301,12 +301,19 @@ INDEX import_sources(source_hash) | `migrations.ts` | `format_version` runner:
- **newer than the app:** read-only, with `library_too_new`
- **older:** take `VACUUM INTO library.pre-v-.sqlite` (keep 2), then run each step in its own transaction; a failure means read-only with `migration_failed` | | `content.ts` | `load`, `append` (an INSERT plus the `docs` title and `modified_at`, in one transaction), `compact`, `stateOf` (`Y.mergeUpdates`, for `.tinydoc` and restore) | | `index.ts` | CRUD on `docs` and `folders`, `rev` and the change feed, metadata search (D-A8: titles, tags, folder names and properties, matched on normalised strings; never body text), purge of Trash after 30 days | -| `daily-copy.ts` | Once a day on the first idle: `VACUUM INTO daily/library-.sqlite`, keeping 7. "Restore from a daily snapshot" opens a copy read-only and returns one document's state, which the renderer applies to the open document as a forward edit through `updateYFragment` (the `schema-repair.ts` approach). | +| `daily-copy.ts` | Once a day on the first idle: a copy to `daily/library-.sqlite`, keeping 7. Since decision 5 (P2.6), the copy is `copy-library.ts`, run in its own thread; see "Daily copies off the writer" below. "Restore from a daily snapshot" opens a copy read-only and returns one document's state, which the renderer applies to the open document as a forward edit through `updateYFragment` (the `schema-repair.ts` approach). | | `mirror.ts` | For each document where `modified_at > mirrored_at` and the renderer has sent Markdown: write `-.tinydoc` (`TINYDOC2`: magic, JSON header with `v`, `minReader`, `enc: null` and `lineage`, then `stateOf`; v8, S8) and `.md`, writing each to a temp file and then renaming it. It keeps one current copy per document. When a document is permanently deleted, its copies move to `Deleted/` and are removed after 30 days. Image files go once to `assets/.` beside them, and backup `.tinydoc` files reference them as external assets (image store §5.4). No build writes or reads `TINYDOC1` (D40). | | `filenames.ts` | Cleans names: path separators, `..`, Windows reserved names, trailing dots and spaces, and paths over 240 characters | | `assets.ts` (P2.6b) | Owns `/assets/`: the temp-fsync-rename write order, dedupe, the startup `tmp/` sweep, `recount` and GC with a 30-day grace (image store §3.2–§3.4). | | `jobs.ts` | One queue. Foreground calls run first; daily copy and mirror run on idle, meaning 30 s without an append. Each job yields between documents. | +**Daily copies off the writer (P2.6 decision 5).** `VACUUM INTO` on the storage worker blocked every append and the quit flush behind it: up to 9 s for a 240 MB library on `windows-latest`. Moved to a second `worker_thread`, it still couldn't be stopped: `terminate()` doesn't interrupt a native SQLite call, Node joins worker threads on exit, and Electron 45's `node:sqlite` has no `interrupt()`. (`backup()` restarts on every write from the writer, so it never finishes while someone types.) So: +- `copy-library.ts` copies in batches from one snapshot. The library is attached read-only (`mode=ro`) to the copy's own connection, and the whole copy is one transaction there. Tables are created from `sqlite_master` verbatim, then the rows, then indexes and triggers. `sqlite_sequence`, `user_version` and `application_id` are copied too. Batches hold at most 256 rows or about 4 MiB, sized by `LENGTH()` from the record headers, and the copy yields between them. +- The copy is verified before it is kept: row counts per table against the snapshot, and `integrity_check`, all inside the transaction. On a mismatch it rolls back. Only a verified copy is renamed over the day's file, then a passive WAL checkpoint runs. A failure keeps the previous copies, records `daily_copy_failed` and retries on the next idle pass. +- `copy-worker.ts` runs the copy in a thread that the storage worker starts. The writer never waits on it. +- On quit, the worker terminates the copy thread (6–9 ms measured), waits for it, deletes the partial file and keeps the previous copy. The next open sweeps any partial left behind. Soak `T4-copy` checks this on all three OSes: every typed edit stored, a clean shutdown, no partial, and the previous copy byte-identical. +- **Upgrade path (option B):** once P2.5's main process exists, `VACUUM INTO` can run in an Electron `utilityProcess`, which `kill()` stops at once. That would give back `VACUUM INTO`'s compaction of free pages. The batched copy keeps every row and skips the library's free pages, so its size is close; T8 reports both. + **As built in P2.6.** - **Modules.** `index.ts` is `library-index.ts` (not a barrel). `library.ts` holds the open order, the registry and the lifecycle; `lock.ts`, `cloud-folder.ts`, `sql.ts`, `driver.ts` (the `SqliteDriver` seam and SQLite error mapping) and `host.ts` (an injected `StorageHost` for files, the pid check and the clock) keep the engine free of `node:*`. `jobs.ts` is two lines in the worker: idle jobs run between requests after 30 s without a write. `mirror.ts`, `filenames.ts` and `assets.ts` come with P2.10 and P2.6b. - **Open order.** Cloud-folder refusal, lock, pragmas, schema creation for a new file, then `format_version` before any other write (a newer library is reopened read-only and never written), then `quick_check` if `unclean_exit` was set (a failure is read-only, `corrupt`), then migrations, then `unclean_exit` is set. @@ -415,6 +422,7 @@ stateDiagram-v2 Ready --> Degraded: "3 crashes in 1 min → read-only, Library unavailable" ``` +- **Quit budget:** `Flushing` gets 3 s (every window flushes its batches), then `WorkerShutdown` gets 2 s (checkpoint, clear `unclean_exit`, close, release the lock), and then the worker is terminated. The WAL keeps every committed row. A daily copy never counts against this budget: it runs in its own thread, which shutdown stops in milliseconds (§5.4, decision 5). The worker's appends commit throughout, so the final batch is durable. - **Re-entrancy:** `quitAndInstall` fires `before-quit` again, and the coordinator ignores it once it is past `Flushing`. - **No `beforeunload` on desktop.** It would silently block quitting (`editor-app.tsx:58-62`). diff --git a/tests/load/desktop-durability-soak.ts b/tests/load/desktop-durability-soak.ts index 5fa349e..b72c746 100644 --- a/tests/load/desktop-durability-soak.ts +++ b/tests/load/desktop-durability-soak.ts @@ -8,7 +8,7 @@ * the two drivers read each other's files. The shape follows spike S2's * kill harness. * - * bun tests/load/desktop-durability-soak.ts --case=all|contract|T1|T2|T3|T4|T5|T6|T8 + * bun tests/load/desktop-durability-soak.ts --case=all|contract|T1|T2|T3|T4|T4-copy|T5|T6|T8 * [--kills=200] [--out=soak-out] [--verbose] * * T7 (slow-hydrate overwrite) is a web-library case: it tests the session @@ -28,6 +28,7 @@ import { mkdirSync, mkdtempSync, readFileSync, + renameSync, rmSync, statSync, writeFileSync, @@ -695,7 +696,7 @@ async function caseT8(): Promise { } // Idle: the daily copy and the Trash purge. clock.now += IDLE_MS; - const idle = library.runIdleJobs(); + const idle = await library.runIdleJobs(); if (idle.dailyCopyMs !== null) copyMs.push(idle.dailyCopyMs); if (idle.dailyCopy === null && idle.dailyCopyMs !== null) failures.push( @@ -791,11 +792,15 @@ async function caseT8(): Promise { const sortedCopies = [...copyMs].sort((a, b) => a - b); const copyMedian = sortedCopies[Math.floor(sortedCopies.length / 2)] ?? 0; const copyMax = sortedCopies.at(-1) ?? 0; + // Decision 5: the batched copy in its thread on node:sqlite, against + // VACUUM INTO on the same year-end library. + const comparison = args["no-electron"] ? null : await compareCopies(dir); + if (comparison?.error) failures.push(`batched copy: ${comparison.error}`); removeDir(dir); return { name: "T8", pass: failures.length === 0, - summary: `${days} days, ${appends} appends, ${compactions} compactions, ${trashed} trashed; ${(seededBytes / 1048576).toFixed(0)} → ${(stats.bytes / 1048576).toFixed(0)} MB; ${dailies.length} daily copies (VACUUM INTO median ${copyMedian.toFixed(0)} ms, max ${copyMax.toFixed(0)} ms); ${liveDocs} documents: worker ready ${toWorkerReady.toFixed(0)} ms and sidebar rows ${toSidebar.toFixed(0)} ms after process start (median of 5; §10 ${budget.workerReadyMs} / ${budget.sidebarMs} ms; excludes renderer IPC and paint); unclean-exit open with quick_check ${unclean ? unclean.openMs.toFixed(0) : "n/a"} ms`, + summary: `${days} days, ${appends} appends, ${compactions} compactions, ${trashed} trashed; ${(seededBytes / 1048576).toFixed(0)} → ${(stats.bytes / 1048576).toFixed(0)} MB; ${dailies.length} daily copies (bun inline median ${copyMedian.toFixed(0)} ms, max ${copyMax.toFixed(0)} ms)${comparison ? `; batched copy on node:sqlite ${comparison.batchedMs?.toFixed(0)} ms, ${(comparison.batchedBytes / 1048576).toFixed(1)} MB vs VACUUM INTO ${comparison.vacuumMs.toFixed(0)} ms, ${(comparison.vacuumBytes / 1048576).toFixed(1)} MB` : ""}; ${liveDocs} documents: worker ready ${toWorkerReady.toFixed(0)} ms and sidebar rows ${toSidebar.toFixed(0)} ms after process start (median of 5; §10 ${budget.workerReadyMs} / ${budget.sidebarMs} ms; excludes renderer IPC and paint); unclean-exit open with quick_check ${unclean ? unclean.openMs.toFixed(0) : "n/a"} ms`, details: { days, docs: liveDocs, @@ -809,12 +814,164 @@ async function caseT8(): Promise { unclean, budget, dailyCopyMs: { median: copyMedian, max: copyMax, count: copyMs.length }, + comparison, simMs, }, failures, }; } +/** Batched copy (Electron, its thread) vs VACUUM INTO (bun), same library. */ +async function compareCopies(dir: string) { + const daily = join(dir, "daily"); + // The year's copies are dated ahead of today's real clock: today's copy + // would sort oldest and be pruned at once, so start from an empty folder. + removeDir(daily); + const before = new Set(); + const run = await runHarness(["--mode=copy-time", `--dir=${dir}`]); + const line = run.lines.find((l) => l.startsWith("COPYTIME ")); + if (!line) + return { + error: run.stderr.slice(0, 500), + batchedMs: null, + batchedBytes: 0, + vacuumMs: 0, + vacuumBytes: 0, + }; + const timing = JSON.parse(line.slice(9)) as { + ms: number | null; + error: string | null; + }; + const made = createNodeHost() + .list(daily) + .find((file) => !before.has(file) && file.endsWith(".sqlite")); + const batchedBytes = made ? statSync(join(daily, made)).size : 0; + const vacuumPath = join(dir, "vacuum-into.sqlite"); + // Read-write: a read-only open of a WAL file with no -shm can't start. + const db = new Database(join(dir, LIBRARY_FILE)); + const t0 = performance.now(); + db.prepare("VACUUM INTO ?").run(vacuumPath); + const vacuumMs = performance.now() - t0; + db.close(); + return { + error: timing.error ?? (made ? null : "no copy file"), + batchedMs: timing.ms, + batchedBytes, + vacuumMs, + vacuumBytes: statSync(vacuumPath).size, + }; +} + +// ── T4-copy: quit while the daily copy runs (decision 5) ────────────── + +function yesterday(): string { + const date = new Date(Date.now() - 24 * 60 * 60 * 1000); + const pad = (n: number) => String(n).padStart(2, "0"); + return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`; +} + +async function caseQuitDuringCopy(runs: number): Promise { + const dir = mkdtempSync(join(tmpdir(), "soak-t4c-")); + const docs = Number(args["copy-docs"] ?? 1000); + const failures: string[] = []; + // A library big enough that the copy outlasts the typing: ~200 MB. + const library = openEngine(dir); + const pad = new Y.Doc(); + pad.getText(PAD).insert(0, "p".repeat(200 * 1024)); + const padState = Y.encodeStateAsUpdate(pad); + for (let i = 0; i < docs; i += 1) { + const { id } = library.createDoc({ title: `Doc ${i}` }); + library.load(id, "seed"); + library.compact(id, "seed", padState, 0); + library.close(id, "seed"); + } + const { id: docId } = library.createDoc({ title: "Typed" }); + // The previous good copy, dated yesterday: it must survive every quit. + const previous = await library.takeDailyCopyNow(); + const copyError = library.stats().lastDailyCopyError; + library.shutdown(); + if (!previous) throw new Error(`setup copy failed: ${copyError}`); + const daily = join(dir, "daily"); + const previousPath = join(daily, `library-${yesterday()}.sqlite`); + renameSync(join(daily, previous), previousPath); + const resetDay = () => { + const db = new Database(join(dir, LIBRARY_FILE)); + db.prepare("UPDATE meta SET value = ? WHERE key = 'last_daily_copy'").run( + yesterday(), + ); + db.close(); + }; + resetDay(); + const hashOf = (path: string) => + createHash("sha256").update(readFileSync(path)).digest("hex"); + const previousHash = hashOf(previousPath); + let midCopy = 0; + const quitMs: number[] = []; + const exitLagMs: number[] = []; + const appendMaxMs: number[] = []; + for (let i = 1; i <= runs; i += 1) { + let quitSeenAt = 0; + let began = false; + let done = false; + const run = await runHarness( + ["--mode=copy-quit", `--dir=${dir}`, `--doc=${docId}`], + (line) => { + if (line.startsWith("COPY_BEGIN")) began = true; + if (line.startsWith("COPY_DONE")) done = true; + if (line.startsWith("QUIT ")) quitSeenAt = performance.now(); + }, + ); + const exitedAt = performance.now(); + note(run.lines.filter((l) => !/^(BATCH_BEGIN|ACK)/.test(l)).join(" | ")); + const quit = run.lines.find((l) => l.startsWith("QUIT ")); + if (run.exitCode !== 0 || !quit) { + failures.push( + `run ${i}: exit ${run.exitCode} ${run.stderr.slice(0, 500)}`, + ); + continue; + } + const [, outcome, kText, quitText, appendText] = quit.split(" "); + if (began && !done) midCopy += 1; + quitMs.push(Number(quitText)); + appendMaxMs.push(Number(appendText)); + exitLagMs.push(exitedAt - quitSeenAt); + if (outcome !== "clean") failures.push(`run ${i}: shutdown ${outcome}`); + if (Number(quitText) > 2000) + failures.push(`run ${i}: shutdown took ${quitText} ms (budget 2 s)`); + const partials = createNodeHost() + .list(daily) + .filter((file) => file.endsWith(".partial")); + if (partials.length > 0) + failures.push( + `run ${i}: partial copy left behind: ${partials.join(", ")}`, + ); + if (!existsSync(previousPath) || hashOf(previousPath) !== previousHash) + failures.push(`run ${i}: the previous good copy changed`); + const reopened = openEngine(dir); + if (reopened.status.uncleanExit) + failures.push(`run ${i}: unclean_exit set`); + const k = editCount(textOf(reopened, docId).text); + reopened.shutdown(); + if (k !== Number(kText)) + failures.push(`run ${i}: ${k} edits stored, ${kText} typed before quit`); + // A run whose copy finished leaves today's copy: undo it for the next run. + for (const file of createNodeHost().list(daily)) + if (file.endsWith(".sqlite") && join(daily, file) !== previousPath) + removeDir(join(daily, file)); + resetDay(); + } + removeDir(dir); + const max = (values: number[]) => Math.round(Math.max(0, ...values)); + if (midCopy === 0) failures.push("no quit landed during a copy"); + return { + name: "T4-copy", + pass: failures.length === 0, + summary: `${runs} quits, ${midCopy} during a daily copy: every typed edit stored, clean shutdowns (max ${max(quitMs)} ms), process gone ${max(exitLagMs)} ms after the quit at worst, no partial copy left, the previous copy byte-identical; slowest append during a copy ${max(appendMaxMs)} ms`, + details: { runs, midCopy, quitMs, exitLagMs, appendMaxMs }, + failures, + }; +} + // ── Main ──────────────────────────────────────────────────────────────── const CASES: Record Promise> = { @@ -825,6 +982,7 @@ const CASES: Record Promise> = { T2: () => killLoop("T2", "compaction", KILLS), T3: caseT3, T4: () => caseT4(Number(args.quits ?? 20)), + "T4-copy": () => caseQuitDuringCopy(Number(args["copy-quits"] ?? 10)), T5: () => caseT5(Number(args["migration-kills"] ?? 40)), T6: caseT6, T8: caseT8, diff --git a/tests/unit/desktop/storage/copy-library.test.ts b/tests/unit/desktop/storage/copy-library.test.ts new file mode 100644 index 0000000..64b598e --- /dev/null +++ b/tests/unit/desktop/storage/copy-library.test.ts @@ -0,0 +1,307 @@ +import { Database } from "bun:sqlite"; +import { afterEach, beforeEach, describe, expect, it, test } from "bun:test"; +import { copyFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; + +import * as Y from "yjs"; + +import { openBunSqlite } from "@/desktop/dev/bun-sqlite-driver"; +import { copyLibraryBatched } from "@/desktop/storage/engine/copy-library"; +import { LIBRARY_FILE, type Library } from "@/desktop/storage/engine/library"; +import { defineDocStoreContract } from "@/tests/helpers/doc-store-contract"; +import { createEngineDocStore } from "@/tests/helpers/engine-doc-store"; +import { + createClock, + createTempDir, + createTestHost, + openTestLibrary, + type TestClock, +} from "@/tests/helpers/storage-library"; + +let temp: ReturnType; +let out: ReturnType; +let clock: TestClock; +let library: Library; + +beforeEach(() => { + temp = createTempDir(); + out = createTempDir("tinydocy-copy-"); + clock = createClock(); + library = openTestLibrary(temp.dir, createTestHost(clock)); +}); +afterEach(() => { + library.shutdown(); + temp.cleanUp(); + out.cleanUp(); +}); + +const tick = () => new Promise((resolve) => setImmediate(resolve)); + +function write(docId: string, text: string) { + if (!library.isOpen(docId)) library.load(docId, "w1"); + const doc = new Y.Doc(); + doc.getText("t").insert(0, text); + return library.append(docId, "w1", { + origin: "local", + update: Y.encodeStateAsUpdate(doc), + }); +} + +/** A library with every table populated, and a compacted document. */ +function populate() { + const folder = library.createFolder({ name: "Work" }); + const docs = Array.from({ length: 12 }, (_, i) => + library.createDoc({ title: `Doc ${i}`, folderId: folder.id }), + ); + const tags = Array.from({ length: 5 }, (_, i) => library.createTag(`t${i}`)); + for (const doc of docs.slice(0, 6)) + library.setDocTags( + doc.id, + tags.map((t) => t.id), + ); + for (const [i, doc] of docs.entries()) { + for (let n = 0; n < 5; n += 1) write(doc.id, `${i}:${n} `); + } + const first = docs[0]?.id ?? ""; + const state = new Y.Doc(); + state.getText("t").insert(0, "compacted"); + const seq = write(first, "x"); + library.compact(first, "w1", Y.encodeStateAsUpdate(state), seq); + return docs.map((d) => d.id); +} + +function describeFile(path: string) { + const db = new Database(path, { readonly: true }); + const schema = db + .query( + "SELECT type, name, tbl_name, sql FROM sqlite_master ORDER BY type, name", + ) + .all(); + const counts = Object.fromEntries( + ( + db + .query( + "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name", + ) + .all() as { name: string }[] + ).map(({ name }) => [ + name, + (db.query(`SELECT COUNT(*) AS n FROM "${name}"`).get() as { n: number }) + .n, + ]), + ); + const sequence = db.query("SELECT * FROM sqlite_sequence").all(); + const meta = db.query("SELECT key, value FROM meta ORDER BY key").all(); + const pragmas = { + user_version: db.query("PRAGMA user_version").get(), + application_id: db.query("PRAGMA application_id").get(), + integrity: db.query("PRAGMA integrity_check").get(), + }; + db.close(); + return { schema, counts, sequence, meta, pragmas }; +} + +describe("copyLibraryBatched", () => { + test("copies the schema verbatim, every row, the sequence and the pragmas", async () => { + populate(); + const raw = new Database(library.path); + raw.exec("PRAGMA user_version = 7; PRAGMA application_id = 1952807268"); + raw.close(); + const partial = join(out.dir, "copy.sqlite.partial"); + const report = await copyLibraryBatched( + openBunSqlite, + library.path, + partial, + { + batchRows: 3, + yieldNow: tick, + }, + ); + const source = describeFile(library.path); + const copy = describeFile(partial); + expect(copy).toEqual(source); + expect(copy.pragmas.integrity).toEqual({ integrity_check: "ok" }); + expect(report.rows).toBe( + Object.entries(source.counts) + .filter(([name]) => name !== "sqlite_sequence") + .reduce((sum, [, n]) => sum + n, 0), + ); + expect(report.batches).toBeGreaterThan(10); + }); + + test("is one snapshot: rows the writer commits during the copy aren't in it", async () => { + const [docId = ""] = populate(); + const before = describeFile(library.path).counts; + let writes = 0; + const partial = join(out.dir, "copy.sqlite.partial"); + await copyLibraryBatched(openBunSqlite, library.path, partial, { + batchRows: 2, + yieldNow: async () => { + // The writer keeps committing between batches. + write(docId, `during ${writes++}`); + await tick(); + }, + }); + expect(writes).toBeGreaterThan(5); + expect(describeFile(partial).counts).toEqual(before); + expect(describeFile(library.path).counts.updates).toBe( + (before.updates ?? 0) + writes, + ); + }); + + test("refuses a copy whose rows don't match the snapshot, leaving nothing committed", async () => { + populate(); + const partial = join(out.dir, "copy.sqlite.partial"); + await expect( + copyLibraryBatched(openBunSqlite, library.path, partial, { + yieldNow: tick, + beforeVerify: (dest) => dest.exec("DELETE FROM main.doc_tags"), + }), + ).rejects.toMatchObject({ code: "corrupt" }); + const db = new Database(partial, { readonly: true }); + expect(db.query("SELECT COUNT(*) AS n FROM sqlite_master").get()).toEqual({ + n: 0, + }); + db.close(); + }); + + test("stops between batches when told to, and releases the files", async () => { + populate(); + const partial = join(out.dir, "copy.sqlite.partial"); + let batches = 0; + await expect( + copyLibraryBatched(openBunSqlite, library.path, partial, { + batchRows: 2, + yieldNow: async () => { + if (++batches === 4) throw new Error("terminated"); + }, + }), + ).rejects.toThrow("terminated"); + const lsof = Bun.which("lsof"); + if (lsof) { + const open = Bun.spawnSync([lsof, "-p", String(process.pid)]) + .stdout.toString() + .split("\n") + .filter((line) => line.includes(out.dir)); + expect(open).toEqual([]); + } + }); + + test("writes a URI-safe read-only attach for paths with spaces and #", async () => { + library.shutdown(); + const odd = createTempDir("tiny docy #1 "); + library = openTestLibrary(odd.dir, createTestHost(clock)); + populate(); + const partial = join(out.dir, "copy.sqlite.partial"); + await copyLibraryBatched(openBunSqlite, library.path, partial, { + yieldNow: tick, + }); + expect(describeFile(partial).counts).toEqual( + describeFile(library.path).counts, + ); + library.shutdown(); + odd.cleanUp(); + library = openTestLibrary(temp.dir, createTestHost(clock)); + }); +}); + +describe("a daily copy restores as a working library", () => { + test("every document and the list come back identical", async () => { + const ids = populate(); + clock.advance(60_000); + const result = await library.runIdleJobs(); + expect(result.dailyCopy).not.toBeNull(); + const copyPath = join(temp.dir, "daily", result.dailyCopy ?? ""); + copyFileSync(copyPath, join(out.dir, LIBRARY_FILE)); + const sidebar = library.sidebar(); + const restored = openTestLibrary(out.dir, createTestHost(clock)); + try { + expect(restored.status.formatVersion).toBe(library.status.formatVersion); + expect(restored.sidebar()).toEqual(sidebar); + for (const id of ids) { + const hash = (lib: Library) => { + const doc = new Y.Doc(); + Y.applyUpdate(doc, lib.stateOf(id)); + return `${doc.getText("t").toString()}|${Y.encodeStateVector(doc).join(",")}`; + }; + expect(hash(restored)).toBe(hash(library)); + } + // AUTOINCREMENT survived: the copy never reuses a seq. + const next = (() => { + restored.load(ids[1] ?? "", "w9"); + const doc = new Y.Doc(); + doc.getText("t").insert(0, "after restore"); + return restored.append(ids[1] ?? "", "w9", { + origin: "local", + update: Y.encodeStateAsUpdate(doc), + }); + })(); + expect(next).toBeGreaterThan(library.load(ids[1] ?? "", "w1").maxSeq); + } finally { + restored.shutdown(); + } + expect(existsSync(`${copyPath}.partial`)).toBe(false); + }); +}); + +// P2.1's DocStore contract on libraries that are daily copies of others. +defineDocStoreContract( + { describe, it }, + "DocStore contract on a restored daily copy (bun:sqlite)", + { + exclusiveOpen: true, + async createLibrary() { + const source = createTempDir(); + const target = createTempDir(); + const host = createTestHost(createClock()); + const original = openTestLibrary(source.dir, host); + original.createDoc({ title: "seed" }); + const name = await original.takeDailyCopyNow(); + original.shutdown(); + copyFileSync( + join(source.dir, "daily", name ?? ""), + join(target.dir, LIBRARY_FILE), + ); + const copy = openTestLibrary(target.dir, host); + let clients = 0; + return { + connect: () => createEngineDocStore(copy, `client-${++clients}`), + async createDoc(docId, schema) { + copy.createDoc({ id: docId, schema }); + }, + async dispose() { + copy.shutdown(); + source.cleanUp(); + target.cleanUp(); + }, + }; + }, + }, +); + +test("batches are bounded by bytes, so a stop never waits on a big batch", async () => { + populate(); + const partial = join(out.dir, "bytes.sqlite.partial"); + const rowBound = await copyLibraryBatched( + openBunSqlite, + library.path, + partial, + { + yieldNow: tick, + }, + ); + const partial2 = join(out.dir, "bytes2.sqlite.partial"); + const byteBound = await copyLibraryBatched( + openBunSqlite, + library.path, + partial2, + { + batchBytes: 1, + yieldNow: tick, + }, + ); + // One row per batch when every row is over the budget. + expect(byteBound.batches).toBeGreaterThan(rowBound.batches); + expect(byteBound.rows).toBe(rowBound.rows); + expect(describeFile(partial2).counts).toEqual(describeFile(partial).counts); +}); diff --git a/tests/unit/desktop/storage/daily-copy.test.ts b/tests/unit/desktop/storage/daily-copy.test.ts index fc4ba1a..436639a 100644 --- a/tests/unit/desktop/storage/daily-copy.test.ts +++ b/tests/unit/desktop/storage/daily-copy.test.ts @@ -2,6 +2,8 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import * as Y from "yjs"; +import { openBunSqlite } from "@/desktop/dev/bun-sqlite-driver"; +import { copyLibraryBatched } from "@/desktop/storage/engine/copy-library"; import { DAILY_KEEP, localDate } from "@/desktop/storage/engine/daily-copy"; import { IDLE_MS, type Library } from "@/desktop/storage/engine/library"; import { @@ -39,36 +41,36 @@ function write(docId: string, text: string) { } describe("daily copies (D-A6)", () => { - test("waits for 30 s without a write", () => { + test("waits for 30 s without a write", async () => { write(library.createDoc().id, "x"); clock.advance(IDLE_MS - 1); - expect(library.runIdleJobs().dailyCopy).toBeNull(); + expect((await library.runIdleJobs()).dailyCopy).toBeNull(); clock.advance(1); - expect(library.runIdleJobs().dailyCopy).toBe( + expect((await library.runIdleJobs()).dailyCopy).toBe( `library-${localDate(clock.now)}.sqlite`, ); }); - test("takes one copy a day and keeps the newest 7", () => { + test("takes one copy a day and keeps the newest 7", async () => { const docId = library.createDoc().id; const taken: string[] = []; for (let day = 0; day < 10; day += 1) { write(docId, `day ${day}`); clock.advance(IDLE_MS); - const first = library.runIdleJobs().dailyCopy; + const first = (await library.runIdleJobs()).dailyCopy; expect(first).not.toBeNull(); taken.push(localDate(clock.now)); - expect(library.runIdleJobs().dailyCopy).toBeNull(); + expect((await library.runIdleJobs()).dailyCopy).toBeNull(); clock.advance(DAY); } expect(library.listDailyCopies()).toEqual(taken.slice(-DAILY_KEEP)); }); - test("a copy restores one document's state as of that day", () => { + test("a copy restores one document's state as of that day", async () => { const docId = library.createDoc().id; write(docId, "before"); clock.advance(IDLE_MS); - library.runIdleJobs(); + await library.runIdleJobs(); const day = localDate(clock.now); write(docId, "after"); const restored = new Y.Doc(); @@ -76,7 +78,7 @@ describe("daily copies (D-A6)", () => { expect(restored.getText("t").toString()).toBe("before"); }); - test("survives a leftover partial copy", () => { + test("survives a leftover partial copy", async () => { const host = createTestHost(clock); host.mkdirp(host.join(temp.dir, "daily")); const partial = host.join( @@ -86,13 +88,83 @@ describe("daily copies (D-A6)", () => { ); host.createExclusive(partial, "junk"); clock.advance(IDLE_MS); - expect(library.runIdleJobs().dailyCopy).not.toBeNull(); + expect((await library.runIdleJobs()).dailyCopy).not.toBeNull(); expect(host.exists(partial)).toBe(false); }); - test("refuses a malformed date", () => { + test("refuses a malformed date", async () => { expect(() => library.readDailyCopy("../../etc", crypto.randomUUID()), ).toThrow(); }); }); + +describe("daily copy off the writer (begin, copy, complete or fail)", () => { + function idle() { + clock.advance(IDLE_MS); + } + + test("begin waits for idle, hands out one job at a time, and nothing until it ends", async () => { + write(library.createDoc().id, "x"); + expect(library.beginDailyCopy()).toBeNull(); + idle(); + const job = library.beginDailyCopy(); + expect(job?.partial).toEndWith(".sqlite.partial"); + expect(job?.source).toBe(library.path); + expect(library.beginDailyCopy()).toBeNull(); + }); + + test("a copy made on its own read-only connection completes like the inline one", async () => { + const docId = library.createDoc().id; + write(docId, "copied"); + idle(); + const job = library.beginDailyCopy(); + if (!job) throw new Error("no job"); + await copyLibraryBatched(openBunSqlite, job.source, job.partial, { + yieldNow: () => new Promise((resolve) => setImmediate(resolve)), + }); + // The writer keeps committing while the copy is out. + write(docId, " later"); + expect(library.completeDailyCopy(job)).toBe( + `library-${localDate(clock.now)}.sqlite`, + ); + expect(library.listDailyCopies()).toEqual([localDate(clock.now)]); + const restored = new Y.Doc(); + Y.applyUpdate(restored, library.readDailyCopy(localDate(clock.now), docId)); + expect(restored.getText("t").toString()).toBe("copied"); + expect(library.beginDailyCopy()).toBeNull(); // done for today + }); + + test("a failed or abandoned copy leaves no partial, keeps older copies and retries", async () => { + const docId = library.createDoc().id; + write(docId, "day one"); + idle(); + await library.runIdleJobs(); + const kept = library.listDailyCopies(); + clock.advance(DAY); + write(docId, " day two"); + idle(); + const job = library.beginDailyCopy(); + if (!job) throw new Error("no job"); + createTestHost(clock).createExclusive(job.partial, "half a copy"); + library.failDailyCopy(job, "quit"); + expect(createTestHost(clock).exists(job.partial)).toBe(false); + expect(library.listDailyCopies()).toEqual(kept); + expect(library.stats().lastDailyCopyError).toBe("quit"); + expect(library.beginDailyCopy()).not.toBeNull(); + }); + + test("opening the library sweeps partial copies a killed process left", async () => { + const host = createTestHost(clock); + library.shutdown(); + host.mkdirp(host.join(temp.dir, "daily")); + const partial = host.join( + temp.dir, + "daily", + "library-2026-11-16.sqlite.partial", + ); + host.createExclusive(partial, "junk"); + library = openTestLibrary(temp.dir, host); + expect(host.exists(partial)).toBe(false); + }); +}); diff --git a/tests/unit/desktop/storage/doc-store-engine.test.ts b/tests/unit/desktop/storage/doc-store-engine.test.ts index 5c1840b..b772e8c 100644 --- a/tests/unit/desktop/storage/doc-store-engine.test.ts +++ b/tests/unit/desktop/storage/doc-store-engine.test.ts @@ -25,7 +25,7 @@ afterEach(() => { }); describe("dispatch", () => { - test("a failure comes back as an envelope, never a throw", () => { + test("a failure comes back as an envelope, never a throw", async () => { expect( dispatch(library, { id: 2, @@ -40,7 +40,7 @@ describe("dispatch", () => { }); describe("the index change feed", () => { - test("an append that changes the title moves rev and names the document", () => { + test("an append that changes the title moves rev and names the document", async () => { const { id } = library.createDoc(); library.load(id, "w1"); library.takeIndexChanges(); @@ -59,7 +59,7 @@ describe("the index change feed", () => { expect(library.takeIndexChanges()).toBeNull(); }); - test("index writes report each changed row once", () => { + test("index writes report each changed row once", async () => { const folder = library.createFolder({ name: "A" }); const { id } = library.createDoc({ folderId: folder.id }); library.renameDoc(id, "x"); @@ -72,7 +72,7 @@ describe("the index change feed", () => { ]); }); - test("the idle Trash purge is announced", () => { + test("the idle Trash purge is announced", async () => { const clock = createClock(); library.shutdown(); library = openTestLibrary(temp.dir, createTestHost(clock)); @@ -80,7 +80,7 @@ describe("the index change feed", () => { library.trashDoc(id); library.takeIndexChanges(); clock.advance(TRASH_RETENTION_MS + IDLE_MS); - expect(library.runIdleJobs().purged).toBe(1); + expect((await library.runIdleJobs()).purged).toBe(1); expect(library.takeIndexChanges()?.changes).toEqual([{ kind: "doc", id }]); }); }); diff --git a/tests/unit/desktop/storage/library-index.test.ts b/tests/unit/desktop/storage/library-index.test.ts index 0947972..1daf981 100644 --- a/tests/unit/desktop/storage/library-index.test.ts +++ b/tests/unit/desktop/storage/library-index.test.ts @@ -44,7 +44,7 @@ function codeOf(run: () => unknown) { } describe("creating documents", () => { - test("mints a UUID, a lineage and a 32-hex link key at insert", () => { + test("mints a UUID, a lineage and a 32-hex link key at insert", async () => { const a = library.createDoc(); const b = library.createDoc(); expect(a.id).toMatch(/^[0-9a-f-]{36}$/); @@ -54,7 +54,7 @@ describe("creating documents", () => { expect(a.linkKey).not.toBe(b.linkKey); }); - test("keeps a lineage and link key only when given (backup restore)", () => { + test("keeps a lineage and link key only when given (backup restore)", async () => { const lineage = crypto.randomUUID(); const linkKey = "a".repeat(32); expect(library.createDoc({ lineage, linkKey })).toMatchObject({ @@ -66,7 +66,7 @@ describe("creating documents", () => { ); }); - test("refuses malformed ids, link keys and kinds", () => { + test("refuses malformed ids, link keys and kinds", async () => { expect(codeOf(() => library.createDoc({ id: "../x" }))).toBe( "invalid_argument", ); @@ -78,14 +78,14 @@ describe("creating documents", () => { ); }); - test("templates are stored with kind 'template'", () => { + test("templates are stored with kind 'template'", async () => { library.createDoc({ kind: "template", title: "Weekly" }); expect(library.sidebar().docs[0]?.kind).toBe("template"); }); }); describe("the sidebar snapshot", () => { - test("lists live folders, documents and tags; rev moves on every write", () => { + test("lists live folders, documents and tags; rev moves on every write", async () => { const start = library.sidebar().rev; const folder = library.createFolder({ name: "Work" }); const doc = library.createDoc({ title: "Plan", folderId: folder.id }); @@ -107,19 +107,19 @@ describe("the sidebar snapshot", () => { ]); }); - test("tag names are unique regardless of case", () => { + test("tag names are unique regardless of case", async () => { library.createTag("Urgent"); expect(codeOf(() => library.createTag("urgent"))).toBe("invalid_argument"); }); - test("opening a document records opened_at", () => { + test("opening a document records opened_at", async () => { const doc = library.createDoc(); clock.advance(1000); library.load(doc.id, "w1"); expect(library.sidebar().docs[0]?.openedAt).toBe(clock.now); }); - test("rename and move", () => { + test("rename and move", async () => { const folder = library.createFolder({ name: "A" }); const doc = library.createDoc({ title: "x" }); library.renameDoc(doc.id, " Renamed "); @@ -136,7 +136,7 @@ describe("the sidebar snapshot", () => { }); describe("Trash and purge", () => { - test("trash hides a document and restore brings it back to its folder", () => { + test("trash hides a document and restore brings it back to its folder", async () => { const folder = library.createFolder({ name: "A" }); const doc = library.createDoc({ folderId: folder.id }); library.trashDoc(doc.id); @@ -145,7 +145,7 @@ describe("Trash and purge", () => { expect(library.sidebar().docs[0]?.folderId).toBe(folder.id); }); - test("purge cascades to rows, snapshot, tags, view and import source", () => { + test("purge cascades to rows, snapshot, tags, view and import source", async () => { const doc = library.createDoc(); const tag = library.createTag("t"); library.setDocTags(doc.id, [tag.id]); @@ -188,7 +188,7 @@ describe("Trash and purge", () => { library = openTestLibrary(temp.dir, createTestHost(clock)); }); - test("idle jobs empty Trash after 30 days, skipping open documents", () => { + test("idle jobs empty Trash after 30 days, skipping open documents", async () => { const old = library.createDoc(); const open = library.createDoc(); const recent = library.createDoc(); @@ -198,18 +198,18 @@ describe("Trash and purge", () => { library.trashDoc(recent.id); library.load(open.id, "w1"); clock.advance(IDLE_MS + 1000); - expect(library.runIdleJobs().purged).toBe(1); + expect((await library.runIdleJobs()).purged).toBe(1); expect(library.stats().docs).toBe(2); }); }); describe("ids and schema at creation", () => { - test("accepts any safe id and stores the given schema", () => { + test("accepts any safe id and stores the given schema", async () => { const { id } = library.createDoc({ id: "doc-a_1", schema: 3 }); expect(library.load(id, "w1").schema).toBe(3); }); - test("refuses ids that aren't safe in SQL, paths or logs", () => { + test("refuses ids that aren't safe in SQL, paths or logs", async () => { for (const id of ["", "../x", "a b", "a/b", "-x", "x".repeat(65)]) expect(codeOf(() => library.createDoc({ id }))).toBe("invalid_argument"); expect(codeOf(() => library.createDoc({ schema: -1 }))).toBe(