From d318c3d6f917b8f6d93e418b63e122b03586c135 Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 14:05:49 +0330 Subject: [PATCH 1/7] refactor(desktop): split the daily copy into begin, copy and complete The copy now reads through its own read-only connection, so it can run off the writer: in WAL a reader never blocks the writer. beginDailyCopy hands out one job when the library is idle, completeDailyCopy publishes it and runs a passive checkpoint, and failDailyCopy drops it, keeping older copies for a retry. Shutdown abandons a running copy and opening the library sweeps partial copies a killed process left. --- desktop/storage/engine/daily-copy.ts | 97 ++++++++++--- desktop/storage/engine/library.ts | 137 ++++++++++++++---- tests/unit/desktop/storage/daily-copy.test.ts | 75 +++++++++- 3 files changed, 262 insertions(+), 47 deletions(-) diff --git a/desktop/storage/engine/daily-copy.ts b/desktop/storage/engine/daily-copy.ts index 884a4a7..069d28b 100644 --- a/desktop/storage/engine/daily-copy.ts +++ b/desktop/storage/engine/daily-copy.ts @@ -6,10 +6,15 @@ * library. * * Each copy is written to `.partial` and renamed, so a kill mid-copy - * never leaves a half file under a real name. Retention is by count, so a - * clock jump can't delete good copies early. + * never leaves a half file under a real name; the next open sweeps it. + * Retention is by count, so a clock jump can't delete good copies early. + * + * The copy reads through its own read-only connection in a second thread + * (`copy-worker.ts`): `VACUUM INTO` of a 240 MB library took up to 9 s on a + * Windows runner, and on the writer it would hold every append, and the + * quit flush, behind it. */ -import type { SqliteDriver } from "./driver"; +import type { OpenSqlite, SqliteDriver } from "./driver"; import type { StorageHost } from "./host"; import { readMeta, writeMeta } from "./sql"; @@ -39,31 +44,81 @@ export function isDailyCopyDue(db: SqliteDriver, now: number): boolean { return readMeta(db, "last_daily_copy") !== localDate(now); } -/** - * Writes today's copy if there isn't one yet. Returns the copy's file name, - * or null when it was already taken today. - */ -export function takeDailyCopy( +/** One daily copy in flight. */ +export type DailyCopyJob = { + date: string; + /** The library file the copy reads. */ + source: string; + /** Where `VACUUM INTO` writes; renamed to `target` once complete. */ + partial: string; + target: string; +}; + +/** Deletes leftover `.partial` files (a killed copy). Best effort. */ +export function sweepPartialCopies(host: StorageHost, dir: string) { + const folder = host.join(dir, DAILY_DIR); + for (const file of host.list(folder)) { + if (!file.endsWith(".partial")) continue; + try { + host.remove(host.join(folder, file)); + } catch { + // Still open elsewhere (Windows); the next open sweeps it. + } + } +} + +/** Today's copy job, or null when today's copy exists. */ +export function planDailyCopy( db: SqliteDriver, host: StorageHost, dir: string, + source: string, now: number, -): string | null { +): DailyCopyJob | null { if (!isDailyCopyDue(db, now)) return null; const date = localDate(now); const folder = host.join(dir, DAILY_DIR); host.mkdirp(folder); - for (const file of host.list(folder)) - if (file.endsWith(".partial")) host.remove(host.join(folder, file)); - const name = dailyCopyName(date); - const target = host.join(folder, name); - const partial = `${target}.partial`; - db.prepare("VACUUM INTO ?").run(partial); - if (host.exists(target)) host.remove(target); - host.rename(partial, target); - writeMeta(db, "last_daily_copy", date); - const copies = listDailyCopies(host, dir); - for (const old of copies.slice(0, -DAILY_KEEP)) + sweepPartialCopies(host, dir); + const target = host.join(folder, dailyCopyName(date)); + return { date, source, partial: `${target}.partial`, target }; +} + +/** + * Writes the copy from its own read-only connection, so the writer never + * waits on it: in WAL a reader never blocks the writer. Runs in the copy + * thread (`desktop/storage/copy-worker.ts`); tests run it inline. + */ +export function copyLibraryInto( + openSqlite: OpenSqlite, + source: string, + partial: string, +) { + const reader = openSqlite(source, { readOnly: true }); + try { + reader.prepare("VACUUM INTO ?").run(partial); + } finally { + reader.close(); + } +} + +/** + * Publishes a finished copy: rename into place, record the day, keep the + * newest {@link DAILY_KEEP}, then a passive checkpoint, since the copy's + * long read held the WAL from shrinking. + */ +export function finishDailyCopy( + db: SqliteDriver, + host: StorageHost, + dir: string, + job: DailyCopyJob, +): string { + if (host.exists(job.target)) host.remove(job.target); + host.rename(job.partial, job.target); + writeMeta(db, "last_daily_copy", job.date); + const folder = host.join(dir, DAILY_DIR); + for (const old of listDailyCopies(host, dir).slice(0, -DAILY_KEEP)) host.remove(host.join(folder, old)); - return name; + db.exec("PRAGMA wal_checkpoint(PASSIVE)"); + return dailyCopyName(job.date); } diff --git a/desktop/storage/engine/library.ts b/desktop/storage/engine/library.ts index e36025d..9b6db33 100644 --- a/desktop/storage/engine/library.ts +++ b/desktop/storage/engine/library.ts @@ -38,12 +38,15 @@ import { stateOf, } from "./content"; import { + copyLibraryInto, DAILY_DIR, + type DailyCopyJob, dailyCopyName, - isDailyCopyDue, + finishDailyCopy, listDailyCopies, localDate, - takeDailyCopy, + planDailyCopy, + sweepPartialCopies, } from "./daily-copy"; import type { OpenSqlite, SqliteDriver } from "./driver"; import { storageErrorFrom } from "./driver"; @@ -91,6 +94,8 @@ export type OpenLibraryOptions = { formatVersion?: number; formatSteps?: readonly FormatStep[]; trace?: Trace; + /** How long without a write counts as idle; default {@link IDLE_MS}. Soak only. */ + idleMs?: number; }; function applyPragmas( @@ -155,6 +160,8 @@ export class Library { private lastWriteAt: number; private changes = new Map(); private lastDailyCopyError: string | null = null; + private copyJob: DailyCopyJob | null = null; + private readonly idleMs: number; private constructor(fields: { dir: string; @@ -165,6 +172,7 @@ export class Library { openSqlite: OpenSqlite; lockPath: string; trace: Trace | undefined; + idleMs: number; }) { this.dir = fields.dir; this.path = fields.path; @@ -174,6 +182,7 @@ export class Library { this.openSqlite = fields.openSqlite; this.lockPath = fields.lockPath; this.trace = fields.trace; + this.idleMs = fields.idleMs; this.lastWriteAt = fields.host.now(); } @@ -228,7 +237,10 @@ export class Library { status.formatVersion = readFormatVersion(db); } } - if (!status.readOnly) writeMeta(db, "unclean_exit", "1"); + if (!status.readOnly) { + writeMeta(db, "unclean_exit", "1"); + sweepPartialCopies(host, dir); + } } return new Library({ dir, @@ -239,6 +251,7 @@ export class Library { openSqlite, lockPath, trace: options.trace, + idleMs: options.idleMs ?? IDLE_MS, }); } catch (error) { try { @@ -484,48 +497,120 @@ export class Library { // ── Jobs and lifecycle ──────────────────────────────────────────────── + private isIdle(): boolean { + return ( + !this.status.readOnly && + this.db !== null && + this.host.now() - this.lastWriteAt >= this.idleMs + ); + } + + /** + * Starts today's daily copy when the library is idle and today has none: + * the caller runs {@link copyLibraryInto} on `job` (the worker, in its copy + * thread), then calls {@link completeDailyCopy} or {@link failDailyCopy}. + * One job at a time; null otherwise. + */ + beginDailyCopy(): DailyCopyJob | null { + if (this.copyJob || !this.isIdle()) return null; + this.copyJob = planDailyCopy( + this.conn, + this.host, + this.dir, + this.path, + this.host.now(), + ); + if (this.copyJob) this.trace?.(`COPY_BEGIN ${this.copyJob.date}`); + return this.copyJob; + } + + completeDailyCopy(job: DailyCopyJob): string { + if (job !== this.copyJob) + throw new StorageError("invalid_argument", "not the running copy"); + this.copyJob = null; + const name = finishDailyCopy(this.conn, this.host, this.dir, job); + this.lastDailyCopyError = null; + this.trace?.(`COPY_DONE ${job.date}`); + return name; + } + /** - * Runs the idle jobs when nothing was written for {@link IDLE_MS}: today's - * daily copy, then Trash older than 30 days (skipping open documents). - * Returns what ran, for the worker's diagnostics. A failed daily copy is - * kept for `stats()` (app health `backups_failing`) and retried on the - * next idle pass; it never stops the purge. + * Drops a copy that failed or was abandoned (quit): its partial file is + * removed when it can be (the next open sweeps it otherwise), earlier + * copies stay, and the next idle pass tries again. + */ + failDailyCopy(job: DailyCopyJob, reason: string): void { + if (job === this.copyJob) this.copyJob = null; + try { + this.host.remove(job.partial); + } catch { + // Still open in the copy thread (Windows); swept on the next open. + } + this.lastDailyCopyError = reason; + } + + /** Purges Trash older than 30 days, skipping open documents. */ + purgeExpiredTrash(): number { + if (!this.isIdle()) return 0; + let purged = 0; + for (const id of index.expiredTrash(this.conn, this.host.now())) { + if (this.holders.has(id)) continue; + index.purgeDoc(this.conn, id); + this.changed("doc", id); + purged += 1; + } + return purged; + } + + /** + * Runs the idle jobs inline, on this thread: today's daily copy, then the + * Trash purge. Tests, the dev host and the soak's year use it; the worker + * runs the copy in its copy thread instead. */ runIdleJobs(): { dailyCopy: string | null; dailyCopyMs: number | null; purged: number; } { - const now = this.host.now(); const result = { dailyCopy: null as string | null, dailyCopyMs: null as number | null, purged: 0, }; - if (this.status.readOnly || now - this.lastWriteAt < IDLE_MS) return result; - if (isDailyCopyDue(this.conn, now)) { + const job = this.beginDailyCopy(); + if (job) { const started = performance.now(); - try { - result.dailyCopy = takeDailyCopy(this.conn, this.host, this.dir, now); - this.lastDailyCopyError = null; - } catch (error) { - this.lastDailyCopyError = storageErrorFrom(error).code; - } + result.dailyCopy = this.copyInline(job); result.dailyCopyMs = performance.now() - started; } - for (const id of index.expiredTrash(this.conn, now)) { - if (this.holders.has(id)) continue; - index.purgeDoc(this.conn, id); - this.changed("doc", id); - result.purged += 1; - } + result.purged = this.purgeExpiredTrash(); return result; } - /** Forces today's daily copy now (diagnostics, tests). */ + private copyInline(job: DailyCopyJob): string | null { + try { + copyLibraryInto(this.openSqlite, job.source, job.partial); + return this.completeDailyCopy(job); + } catch (error) { + this.failDailyCopy(job, storageErrorFrom(error).code); + return null; + } + } + + /** Forces today's daily copy now, ignoring idle (diagnostics, tests). */ takeDailyCopyNow(): string | null { this.assertWritable(); - return takeDailyCopy(this.conn, this.host, this.dir, this.host.now()); + if (this.copyJob) return null; + const job = planDailyCopy( + this.conn, + this.host, + this.dir, + this.path, + this.host.now(), + ); + if (!job) return null; + this.copyJob = job; + return this.copyInline(job); } stats() { @@ -564,6 +649,8 @@ export class Library { shutdown(): void { const db = this.db; if (!db) return; + // Quit never waits for a daily copy (its thread is the worker's to stop). + if (this.copyJob) this.failDailyCopy(this.copyJob, "quit"); try { if (!this.status.readOnly) { writeMeta(db, "unclean_exit", "0"); diff --git a/tests/unit/desktop/storage/daily-copy.test.ts b/tests/unit/desktop/storage/daily-copy.test.ts index fc4ba1a..52975ea 100644 --- a/tests/unit/desktop/storage/daily-copy.test.ts +++ b/tests/unit/desktop/storage/daily-copy.test.ts @@ -2,7 +2,12 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import * as Y from "yjs"; -import { DAILY_KEEP, localDate } from "@/desktop/storage/engine/daily-copy"; +import { openBunSqlite } from "@/desktop/dev/bun-sqlite-driver"; +import { + copyLibraryInto, + DAILY_KEEP, + localDate, +} from "@/desktop/storage/engine/daily-copy"; import { IDLE_MS, type Library } from "@/desktop/storage/engine/library"; import { createClock, @@ -96,3 +101,71 @@ describe("daily copies (D-A6)", () => { ).toThrow(); }); }); + +describe("daily copy off the writer (begin, copy, complete or fail)", () => { + function idle() { + clock.advance(IDLE_MS); + } + + test("begin waits for idle, hands out one job at a time, and nothing until it ends", () => { + write(library.createDoc().id, "x"); + expect(library.beginDailyCopy()).toBeNull(); + idle(); + const job = library.beginDailyCopy(); + expect(job?.partial).toEndWith(".sqlite.partial"); + expect(job?.source).toBe(library.path); + expect(library.beginDailyCopy()).toBeNull(); + }); + + test("a copy made on its own read-only connection completes like the inline one", () => { + const docId = library.createDoc().id; + write(docId, "copied"); + idle(); + const job = library.beginDailyCopy(); + if (!job) throw new Error("no job"); + copyLibraryInto(openBunSqlite, job.source, job.partial); + // The writer keeps committing while the copy is out. + write(docId, " later"); + expect(library.completeDailyCopy(job)).toBe( + `library-${localDate(clock.now)}.sqlite`, + ); + expect(library.listDailyCopies()).toEqual([localDate(clock.now)]); + const restored = new Y.Doc(); + Y.applyUpdate(restored, library.readDailyCopy(localDate(clock.now), docId)); + expect(restored.getText("t").toString()).toBe("copied"); + expect(library.beginDailyCopy()).toBeNull(); // done for today + }); + + test("a failed or abandoned copy leaves no partial, keeps older copies and retries", () => { + const docId = library.createDoc().id; + write(docId, "day one"); + idle(); + library.runIdleJobs(); + const kept = library.listDailyCopies(); + clock.advance(DAY); + write(docId, " day two"); + idle(); + const job = library.beginDailyCopy(); + if (!job) throw new Error("no job"); + createTestHost(clock).createExclusive(job.partial, "half a copy"); + library.failDailyCopy(job, "quit"); + expect(createTestHost(clock).exists(job.partial)).toBe(false); + expect(library.listDailyCopies()).toEqual(kept); + expect(library.stats().lastDailyCopyError).toBe("quit"); + expect(library.beginDailyCopy()).not.toBeNull(); + }); + + test("opening the library sweeps partial copies a killed process left", () => { + const host = createTestHost(clock); + library.shutdown(); + host.mkdirp(host.join(temp.dir, "daily")); + const partial = host.join( + temp.dir, + "daily", + "library-2026-11-16.sqlite.partial", + ); + host.createExclusive(partial, "junk"); + library = openTestLibrary(temp.dir, host); + expect(host.exists(partial)).toBe(false); + }); +}); From 2f4c89123f1e90feba3269aab560c7f767997fdc Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 14:15:52 +0330 Subject: [PATCH 2/7] feat(desktop): copy the library in batches from one verified snapshot The daily copy no longer uses VACUUM INTO, one native call that a thread can't be stopped in. copyLibraryBatched attaches the library read-only to the copy's connection and copies it in one transaction: the schema from sqlite_master verbatim, the rows in batches of at most 256 rows or about 4 MiB (yielding between them), then indexes and triggers, sqlite_sequence, user_version and application_id. Row counts and integrity_check are checked before it commits; only then is the copy renamed into place. runIdleJobs and takeDailyCopyNow become async. --- desktop/storage/engine/copy-library.ts | 223 +++++++++++++ desktop/storage/engine/daily-copy.ts | 28 +- desktop/storage/engine/library.ts | 33 +- .../unit/desktop/storage/copy-library.test.ts | 307 ++++++++++++++++++ tests/unit/desktop/storage/daily-copy.test.ts | 43 ++- .../desktop/storage/doc-store-engine.test.ts | 10 +- .../desktop/storage/library-index.test.ts | 28 +- 7 files changed, 597 insertions(+), 75 deletions(-) create mode 100644 desktop/storage/engine/copy-library.ts create mode 100644 tests/unit/desktop/storage/copy-library.test.ts diff --git a/desktop/storage/engine/copy-library.ts b/desktop/storage/engine/copy-library.ts new file mode 100644 index 0000000..1394888 --- /dev/null +++ b/desktop/storage/engine/copy-library.ts @@ -0,0 +1,223 @@ +/** + * The daily copy, made in batches so it can be stopped (P2.6 decision 5). + * + * `VACUUM INTO` is one native call: a worker thread running it can't be + * terminated, and Node joins worker threads on exit, so a quit during a copy + * waited for it (9 s for a 240 MB library on a Windows runner). This copy + * yields between batches, where `Worker.terminate()` takes effect. + * + * It must be as trustworthy as `VACUUM INTO`: + * - **one snapshot:** the library is attached read-only (`mode=ro`) to the + * copy's connection, and the whole copy is one transaction on that + * connection, so every batch reads the same snapshot of the library + * while the writer keeps committing (WAL); + * - **the schema verbatim:** tables from the library's `sqlite_master` + * first, then the rows, then indexes and triggers; `sqlite_sequence`, + * `user_version` and `application_id` are copied (`format_version` lives + * in `meta`, a table); + * - **verified before it is kept:** row counts per table against the + * snapshot and `integrity_check`, inside the transaction; any mismatch + * rolls back and nothing is committed. The caller renames the file into + * place only after this resolves. + * + * The result keeps every row, but isn't compacted the way `VACUUM INTO`'s + * is: free pages of the library are simply not copied, so sizes stay close. + */ +import type { OpenSqlite, SqliteDriver } from "./driver"; +import { StorageError } from "./errors"; +import { scalar } from "./sql"; + +export type CopyOptions = { + /** At most this many rows per batch; the copy yields after each one. */ + batchRows?: number; + /** + * At most about this many bytes per batch (a single larger row is its own + * batch). One batch is one native statement that `terminate()` can't + * interrupt, so this bounds how long a quit waits: 4 MiB is a few ms. + */ + batchBytes?: number; + /** Lets the event loop run (and `terminate()` land) between batches. */ + yieldNow: () => Promise; + /** Tests only: tamper with the copy before it is verified. */ + beforeVerify?: (dest: SqliteDriver) => void; +}; + +export type CopyReport = { tables: number; rows: number; batches: number }; + +/** A read-only SQLite URI for `path`, on every OS. */ +export function readOnlyUri(path: string): string { + const slashed = path.replace(/\\/g, "/"); + const encoded = encodeURI(slashed).replace(/\?/g, "%3F").replace(/#/g, "%23"); + return `file:${slashed.startsWith("/") ? "" : "///"}${encoded}?mode=ro`; +} + +const quote = (name: string) => `"${name.replace(/"/g, '""')}"`; + +type SchemaObject = { type: string; name: string; sql: string }; + +/** An SQL expression for a row's stored size, from every column's length. */ +function rowSize(db: SqliteDriver, table: string): string { + const columns = db + .prepare(`PRAGMA lib.table_info(${quote(table)})`) + .all() + .map((column) => `COALESCE(LENGTH(${quote(String(column.name))}), 0)`); + return columns.length > 0 ? columns.join(" + ") : "0"; +} + +/** Primary-key columns of a WITHOUT ROWID table, in key order. */ +function keyColumns(db: SqliteDriver, table: string): string[] { + return db + .prepare(`PRAGMA lib.table_info(${quote(table)})`) + .all() + .filter((column) => Number(column.pk) > 0) + .sort((a, b) => Number(a.pk) - Number(b.pk)) + .map((column) => quote(String(column.name))); +} + +async function copyRowidTable( + db: SqliteDriver, + table: string, + { batchRows, batchBytes }: { batchRows: number; batchBytes: number }, + yieldNow: () => Promise, +): Promise { + const name = quote(table); + // LENGTH() of a BLOB or TEXT comes from the record header: no payload read. + const sizes = db.prepare( + `SELECT rowid AS r, ${rowSize(db, table)} AS b FROM lib.${name} WHERE rowid > ? ORDER BY rowid LIMIT ?`, + ); + const insert = db.prepare( + `INSERT INTO main.${name} SELECT * FROM lib.${name} WHERE rowid > ? AND rowid <= ?`, + ); + let cursor = Number.MIN_SAFE_INTEGER; + let batches = 0; + for (;;) { + let last: number | null = null; + let bytes = 0; + for (const row of sizes.all(cursor, batchRows)) { + if (last !== null && bytes + Number(row.b) > batchBytes) break; + last = Number(row.r); + bytes += Number(row.b); + } + if (last === null) return batches; + insert.run(cursor, last); + cursor = last; + batches += 1; + await yieldNow(); + } +} + +async function copyKeyedTable( + db: SqliteDriver, + table: string, + { batchRows }: { batchRows: number; batchBytes: number }, + yieldNow: () => Promise, +): Promise { + const name = quote(table); + const order = keyColumns(db, table).join(", ") || "1"; + const insert = db.prepare( + `INSERT INTO main.${name} SELECT * FROM lib.${name} ORDER BY ${order} LIMIT ? OFFSET ?`, + ); + let batches = 0; + for (let offset = 0; ; offset += batchRows) { + if (insert.run(batchRows, offset).changes === 0) return batches; + batches += 1; + await yieldNow(); + } +} + +function count(db: SqliteDriver, schema: "main" | "lib", table: string) { + return Number(scalar(db, `SELECT COUNT(*) FROM ${schema}.${quote(table)}`)); +} + +/** + * Copies the library at `source` into a new file at `partial`. Resolves + * once the copy is committed and verified; rejects (with nothing committed) + * on a mismatch, an I/O error, or a throw from `yieldNow`. + */ +export async function copyLibraryBatched( + openSqlite: OpenSqlite, + source: string, + partial: string, + { + batchRows = 256, + batchBytes = 4 * 1024 * 1024, + yieldNow, + beforeVerify, + }: CopyOptions, +): Promise { + const dest = openSqlite(partial, { readOnly: false }); + let attached = false; + let open = false; + try { + dest.exec("PRAGMA journal_mode = DELETE"); + dest.exec("PRAGMA synchronous = FULL"); + dest.prepare("ATTACH DATABASE ? AS lib").run(readOnlyUri(source)); + attached = true; + dest.exec("BEGIN"); + open = true; + // The first read pins the library's snapshot for the whole copy. + const objects = dest + .prepare( + "SELECT type, name, sql FROM lib.sqlite_master WHERE sql IS NOT NULL ORDER BY rowid", + ) + .all() as SchemaObject[]; + const tables = objects.filter( + (o) => o.type === "table" && !o.name.startsWith("sqlite_"), + ); + for (const table of tables) dest.exec(table.sql); + for (const pragma of ["user_version", "application_id"]) { + const value = Number(scalar(dest, `PRAGMA lib.${pragma}`) ?? 0); + dest.exec(`PRAGMA main.${pragma} = ${Math.trunc(value)}`); + } + const report: CopyReport = { tables: tables.length, rows: 0, batches: 0 }; + for (const table of tables) { + const keyed = /\bWITHOUT\s+ROWID\b/i.test(table.sql); + report.batches += await (keyed ? copyKeyedTable : copyRowidTable)( + dest, + table.name, + { batchRows, batchBytes }, + yieldNow, + ); + } + const hasSequence = objects.some((o) => o.name === "sqlite_sequence"); + if (hasSequence) { + dest.exec("DELETE FROM main.sqlite_sequence"); + dest.exec( + "INSERT INTO main.sqlite_sequence (name, seq) SELECT name, seq FROM lib.sqlite_sequence", + ); + } + for (const object of objects) + if (object.type !== "table" && !object.name.startsWith("sqlite_")) + dest.exec(object.sql); + beforeVerify?.(dest); + for (const table of tables) { + const copied = count(dest, "main", table.name); + if (copied !== count(dest, "lib", table.name)) + throw new StorageError("corrupt", `copy of ${table.name} is short`); + report.rows += copied; + } + if ( + hasSequence && + count(dest, "main", "sqlite_sequence") !== + count(dest, "lib", "sqlite_sequence") + ) + throw new StorageError("corrupt", "copy of sqlite_sequence is short"); + const check = String(scalar(dest, "PRAGMA main.integrity_check")); + if (check !== "ok") + throw new StorageError( + "corrupt", + `copy failed integrity_check: ${check}`, + ); + dest.exec("COMMIT"); + open = false; + return report; + } finally { + try { + if (open) dest.exec("ROLLBACK"); + if (attached) dest.exec("DETACH DATABASE lib"); + } catch { + // The connection closes below either way. + } + dest.close(); + } +} diff --git a/desktop/storage/engine/daily-copy.ts b/desktop/storage/engine/daily-copy.ts index 069d28b..6cb43d1 100644 --- a/desktop/storage/engine/daily-copy.ts +++ b/desktop/storage/engine/daily-copy.ts @@ -9,12 +9,12 @@ * never leaves a half file under a real name; the next open sweeps it. * Retention is by count, so a clock jump can't delete good copies early. * - * The copy reads through its own read-only connection in a second thread - * (`copy-worker.ts`): `VACUUM INTO` of a 240 MB library took up to 9 s on a - * Windows runner, and on the writer it would hold every append, and the - * quit flush, behind it. + * The copy itself is `copy-library.ts`: batched, from one read-only snapshot, + * verified before it is kept. The worker runs it in a second thread + * (`copy-worker.ts`) so the writer never waits on it, and stops that thread + * on quit. */ -import type { OpenSqlite, SqliteDriver } from "./driver"; +import type { SqliteDriver } from "./driver"; import type { StorageHost } from "./host"; import { readMeta, writeMeta } from "./sql"; @@ -84,24 +84,6 @@ export function planDailyCopy( return { date, source, partial: `${target}.partial`, target }; } -/** - * Writes the copy from its own read-only connection, so the writer never - * waits on it: in WAL a reader never blocks the writer. Runs in the copy - * thread (`desktop/storage/copy-worker.ts`); tests run it inline. - */ -export function copyLibraryInto( - openSqlite: OpenSqlite, - source: string, - partial: string, -) { - const reader = openSqlite(source, { readOnly: true }); - try { - reader.prepare("VACUUM INTO ?").run(partial); - } finally { - reader.close(); - } -} - /** * Publishes a finished copy: rename into place, record the day, keep the * newest {@link DAILY_KEEP}, then a passive checkpoint, since the copy's diff --git a/desktop/storage/engine/library.ts b/desktop/storage/engine/library.ts index 9b6db33..7e1b5f7 100644 --- a/desktop/storage/engine/library.ts +++ b/desktop/storage/engine/library.ts @@ -37,8 +37,8 @@ import { pendingRows, stateOf, } from "./content"; +import { copyLibraryBatched } from "./copy-library"; import { - copyLibraryInto, DAILY_DIR, type DailyCopyJob, dailyCopyName, @@ -160,6 +160,7 @@ export class Library { private lastWriteAt: number; private changes = new Map(); private lastDailyCopyError: string | null = null; + private lastDailyCopyMs: number | null = null; private copyJob: DailyCopyJob | null = null; private readonly idleMs: number; @@ -507,7 +508,7 @@ export class Library { /** * Starts today's daily copy when the library is idle and today has none: - * the caller runs {@link copyLibraryInto} on `job` (the worker, in its copy + * the caller runs `copyLibraryBatched` on `job` (the worker, in its copy * thread), then calls {@link completeDailyCopy} or {@link failDailyCopy}. * One job at a time; null otherwise. */ @@ -524,12 +525,13 @@ export class Library { return this.copyJob; } - completeDailyCopy(job: DailyCopyJob): string { + completeDailyCopy(job: DailyCopyJob, ms: number | null = null): string { if (job !== this.copyJob) throw new StorageError("invalid_argument", "not the running copy"); this.copyJob = null; const name = finishDailyCopy(this.conn, this.host, this.dir, job); this.lastDailyCopyError = null; + this.lastDailyCopyMs = ms; this.trace?.(`COPY_DONE ${job.date}`); return name; } @@ -547,6 +549,7 @@ export class Library { // Still open in the copy thread (Windows); swept on the next open. } this.lastDailyCopyError = reason; + this.trace?.(`COPY_ABANDONED ${job.date} ${reason}`); } /** Purges Trash older than 30 days, skipping open documents. */ @@ -567,11 +570,11 @@ export class Library { * Trash purge. Tests, the dev host and the soak's year use it; the worker * runs the copy in its copy thread instead. */ - runIdleJobs(): { + async runIdleJobs(): Promise<{ dailyCopy: string | null; dailyCopyMs: number | null; purged: number; - } { + }> { const result = { dailyCopy: null as string | null, dailyCopyMs: null as number | null, @@ -580,25 +583,31 @@ export class Library { const job = this.beginDailyCopy(); if (job) { const started = performance.now(); - result.dailyCopy = this.copyInline(job); + result.dailyCopy = await this.copyInline(job); result.dailyCopyMs = performance.now() - started; } result.purged = this.purgeExpiredTrash(); return result; } - private copyInline(job: DailyCopyJob): string | null { + private async copyInline(job: DailyCopyJob): Promise { + const started = performance.now(); try { - copyLibraryInto(this.openSqlite, job.source, job.partial); - return this.completeDailyCopy(job); + await copyLibraryBatched(this.openSqlite, job.source, job.partial, { + yieldNow: () => new Promise((resolve) => setTimeout(resolve, 0)), + }); + return this.completeDailyCopy(job, performance.now() - started); } catch (error) { - this.failDailyCopy(job, storageErrorFrom(error).code); + this.failDailyCopy( + job, + `daily_copy_failed: ${storageErrorFrom(error).code}`, + ); return null; } } /** Forces today's daily copy now, ignoring idle (diagnostics, tests). */ - takeDailyCopyNow(): string | null { + async takeDailyCopyNow(): Promise { this.assertWritable(); if (this.copyJob) return null; const job = planDailyCopy( @@ -627,6 +636,8 @@ export class Library { ), lastDailyCopy: readMeta(db, "last_daily_copy"), lastDailyCopyError: this.lastDailyCopyError, + lastDailyCopyMs: this.lastDailyCopyMs, + dailyCopyRunning: this.copyJob !== null, today: localDate(this.host.now()), // Read back, as set (D-A6; S2 confirmed them on all three OSes). pragmas: { diff --git a/tests/unit/desktop/storage/copy-library.test.ts b/tests/unit/desktop/storage/copy-library.test.ts new file mode 100644 index 0000000..64b598e --- /dev/null +++ b/tests/unit/desktop/storage/copy-library.test.ts @@ -0,0 +1,307 @@ +import { Database } from "bun:sqlite"; +import { afterEach, beforeEach, describe, expect, it, test } from "bun:test"; +import { copyFileSync, existsSync } from "node:fs"; +import { join } from "node:path"; + +import * as Y from "yjs"; + +import { openBunSqlite } from "@/desktop/dev/bun-sqlite-driver"; +import { copyLibraryBatched } from "@/desktop/storage/engine/copy-library"; +import { LIBRARY_FILE, type Library } from "@/desktop/storage/engine/library"; +import { defineDocStoreContract } from "@/tests/helpers/doc-store-contract"; +import { createEngineDocStore } from "@/tests/helpers/engine-doc-store"; +import { + createClock, + createTempDir, + createTestHost, + openTestLibrary, + type TestClock, +} from "@/tests/helpers/storage-library"; + +let temp: ReturnType; +let out: ReturnType; +let clock: TestClock; +let library: Library; + +beforeEach(() => { + temp = createTempDir(); + out = createTempDir("tinydocy-copy-"); + clock = createClock(); + library = openTestLibrary(temp.dir, createTestHost(clock)); +}); +afterEach(() => { + library.shutdown(); + temp.cleanUp(); + out.cleanUp(); +}); + +const tick = () => new Promise((resolve) => setImmediate(resolve)); + +function write(docId: string, text: string) { + if (!library.isOpen(docId)) library.load(docId, "w1"); + const doc = new Y.Doc(); + doc.getText("t").insert(0, text); + return library.append(docId, "w1", { + origin: "local", + update: Y.encodeStateAsUpdate(doc), + }); +} + +/** A library with every table populated, and a compacted document. */ +function populate() { + const folder = library.createFolder({ name: "Work" }); + const docs = Array.from({ length: 12 }, (_, i) => + library.createDoc({ title: `Doc ${i}`, folderId: folder.id }), + ); + const tags = Array.from({ length: 5 }, (_, i) => library.createTag(`t${i}`)); + for (const doc of docs.slice(0, 6)) + library.setDocTags( + doc.id, + tags.map((t) => t.id), + ); + for (const [i, doc] of docs.entries()) { + for (let n = 0; n < 5; n += 1) write(doc.id, `${i}:${n} `); + } + const first = docs[0]?.id ?? ""; + const state = new Y.Doc(); + state.getText("t").insert(0, "compacted"); + const seq = write(first, "x"); + library.compact(first, "w1", Y.encodeStateAsUpdate(state), seq); + return docs.map((d) => d.id); +} + +function describeFile(path: string) { + const db = new Database(path, { readonly: true }); + const schema = db + .query( + "SELECT type, name, tbl_name, sql FROM sqlite_master ORDER BY type, name", + ) + .all(); + const counts = Object.fromEntries( + ( + db + .query( + "SELECT name FROM sqlite_master WHERE type = 'table' ORDER BY name", + ) + .all() as { name: string }[] + ).map(({ name }) => [ + name, + (db.query(`SELECT COUNT(*) AS n FROM "${name}"`).get() as { n: number }) + .n, + ]), + ); + const sequence = db.query("SELECT * FROM sqlite_sequence").all(); + const meta = db.query("SELECT key, value FROM meta ORDER BY key").all(); + const pragmas = { + user_version: db.query("PRAGMA user_version").get(), + application_id: db.query("PRAGMA application_id").get(), + integrity: db.query("PRAGMA integrity_check").get(), + }; + db.close(); + return { schema, counts, sequence, meta, pragmas }; +} + +describe("copyLibraryBatched", () => { + test("copies the schema verbatim, every row, the sequence and the pragmas", async () => { + populate(); + const raw = new Database(library.path); + raw.exec("PRAGMA user_version = 7; PRAGMA application_id = 1952807268"); + raw.close(); + const partial = join(out.dir, "copy.sqlite.partial"); + const report = await copyLibraryBatched( + openBunSqlite, + library.path, + partial, + { + batchRows: 3, + yieldNow: tick, + }, + ); + const source = describeFile(library.path); + const copy = describeFile(partial); + expect(copy).toEqual(source); + expect(copy.pragmas.integrity).toEqual({ integrity_check: "ok" }); + expect(report.rows).toBe( + Object.entries(source.counts) + .filter(([name]) => name !== "sqlite_sequence") + .reduce((sum, [, n]) => sum + n, 0), + ); + expect(report.batches).toBeGreaterThan(10); + }); + + test("is one snapshot: rows the writer commits during the copy aren't in it", async () => { + const [docId = ""] = populate(); + const before = describeFile(library.path).counts; + let writes = 0; + const partial = join(out.dir, "copy.sqlite.partial"); + await copyLibraryBatched(openBunSqlite, library.path, partial, { + batchRows: 2, + yieldNow: async () => { + // The writer keeps committing between batches. + write(docId, `during ${writes++}`); + await tick(); + }, + }); + expect(writes).toBeGreaterThan(5); + expect(describeFile(partial).counts).toEqual(before); + expect(describeFile(library.path).counts.updates).toBe( + (before.updates ?? 0) + writes, + ); + }); + + test("refuses a copy whose rows don't match the snapshot, leaving nothing committed", async () => { + populate(); + const partial = join(out.dir, "copy.sqlite.partial"); + await expect( + copyLibraryBatched(openBunSqlite, library.path, partial, { + yieldNow: tick, + beforeVerify: (dest) => dest.exec("DELETE FROM main.doc_tags"), + }), + ).rejects.toMatchObject({ code: "corrupt" }); + const db = new Database(partial, { readonly: true }); + expect(db.query("SELECT COUNT(*) AS n FROM sqlite_master").get()).toEqual({ + n: 0, + }); + db.close(); + }); + + test("stops between batches when told to, and releases the files", async () => { + populate(); + const partial = join(out.dir, "copy.sqlite.partial"); + let batches = 0; + await expect( + copyLibraryBatched(openBunSqlite, library.path, partial, { + batchRows: 2, + yieldNow: async () => { + if (++batches === 4) throw new Error("terminated"); + }, + }), + ).rejects.toThrow("terminated"); + const lsof = Bun.which("lsof"); + if (lsof) { + const open = Bun.spawnSync([lsof, "-p", String(process.pid)]) + .stdout.toString() + .split("\n") + .filter((line) => line.includes(out.dir)); + expect(open).toEqual([]); + } + }); + + test("writes a URI-safe read-only attach for paths with spaces and #", async () => { + library.shutdown(); + const odd = createTempDir("tiny docy #1 "); + library = openTestLibrary(odd.dir, createTestHost(clock)); + populate(); + const partial = join(out.dir, "copy.sqlite.partial"); + await copyLibraryBatched(openBunSqlite, library.path, partial, { + yieldNow: tick, + }); + expect(describeFile(partial).counts).toEqual( + describeFile(library.path).counts, + ); + library.shutdown(); + odd.cleanUp(); + library = openTestLibrary(temp.dir, createTestHost(clock)); + }); +}); + +describe("a daily copy restores as a working library", () => { + test("every document and the list come back identical", async () => { + const ids = populate(); + clock.advance(60_000); + const result = await library.runIdleJobs(); + expect(result.dailyCopy).not.toBeNull(); + const copyPath = join(temp.dir, "daily", result.dailyCopy ?? ""); + copyFileSync(copyPath, join(out.dir, LIBRARY_FILE)); + const sidebar = library.sidebar(); + const restored = openTestLibrary(out.dir, createTestHost(clock)); + try { + expect(restored.status.formatVersion).toBe(library.status.formatVersion); + expect(restored.sidebar()).toEqual(sidebar); + for (const id of ids) { + const hash = (lib: Library) => { + const doc = new Y.Doc(); + Y.applyUpdate(doc, lib.stateOf(id)); + return `${doc.getText("t").toString()}|${Y.encodeStateVector(doc).join(",")}`; + }; + expect(hash(restored)).toBe(hash(library)); + } + // AUTOINCREMENT survived: the copy never reuses a seq. + const next = (() => { + restored.load(ids[1] ?? "", "w9"); + const doc = new Y.Doc(); + doc.getText("t").insert(0, "after restore"); + return restored.append(ids[1] ?? "", "w9", { + origin: "local", + update: Y.encodeStateAsUpdate(doc), + }); + })(); + expect(next).toBeGreaterThan(library.load(ids[1] ?? "", "w1").maxSeq); + } finally { + restored.shutdown(); + } + expect(existsSync(`${copyPath}.partial`)).toBe(false); + }); +}); + +// P2.1's DocStore contract on libraries that are daily copies of others. +defineDocStoreContract( + { describe, it }, + "DocStore contract on a restored daily copy (bun:sqlite)", + { + exclusiveOpen: true, + async createLibrary() { + const source = createTempDir(); + const target = createTempDir(); + const host = createTestHost(createClock()); + const original = openTestLibrary(source.dir, host); + original.createDoc({ title: "seed" }); + const name = await original.takeDailyCopyNow(); + original.shutdown(); + copyFileSync( + join(source.dir, "daily", name ?? ""), + join(target.dir, LIBRARY_FILE), + ); + const copy = openTestLibrary(target.dir, host); + let clients = 0; + return { + connect: () => createEngineDocStore(copy, `client-${++clients}`), + async createDoc(docId, schema) { + copy.createDoc({ id: docId, schema }); + }, + async dispose() { + copy.shutdown(); + source.cleanUp(); + target.cleanUp(); + }, + }; + }, + }, +); + +test("batches are bounded by bytes, so a stop never waits on a big batch", async () => { + populate(); + const partial = join(out.dir, "bytes.sqlite.partial"); + const rowBound = await copyLibraryBatched( + openBunSqlite, + library.path, + partial, + { + yieldNow: tick, + }, + ); + const partial2 = join(out.dir, "bytes2.sqlite.partial"); + const byteBound = await copyLibraryBatched( + openBunSqlite, + library.path, + partial2, + { + batchBytes: 1, + yieldNow: tick, + }, + ); + // One row per batch when every row is over the budget. + expect(byteBound.batches).toBeGreaterThan(rowBound.batches); + expect(byteBound.rows).toBe(rowBound.rows); + expect(describeFile(partial2).counts).toEqual(describeFile(partial).counts); +}); diff --git a/tests/unit/desktop/storage/daily-copy.test.ts b/tests/unit/desktop/storage/daily-copy.test.ts index 52975ea..436639a 100644 --- a/tests/unit/desktop/storage/daily-copy.test.ts +++ b/tests/unit/desktop/storage/daily-copy.test.ts @@ -3,11 +3,8 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; import * as Y from "yjs"; import { openBunSqlite } from "@/desktop/dev/bun-sqlite-driver"; -import { - copyLibraryInto, - DAILY_KEEP, - localDate, -} from "@/desktop/storage/engine/daily-copy"; +import { copyLibraryBatched } from "@/desktop/storage/engine/copy-library"; +import { DAILY_KEEP, localDate } from "@/desktop/storage/engine/daily-copy"; import { IDLE_MS, type Library } from "@/desktop/storage/engine/library"; import { createClock, @@ -44,36 +41,36 @@ function write(docId: string, text: string) { } describe("daily copies (D-A6)", () => { - test("waits for 30 s without a write", () => { + test("waits for 30 s without a write", async () => { write(library.createDoc().id, "x"); clock.advance(IDLE_MS - 1); - expect(library.runIdleJobs().dailyCopy).toBeNull(); + expect((await library.runIdleJobs()).dailyCopy).toBeNull(); clock.advance(1); - expect(library.runIdleJobs().dailyCopy).toBe( + expect((await library.runIdleJobs()).dailyCopy).toBe( `library-${localDate(clock.now)}.sqlite`, ); }); - test("takes one copy a day and keeps the newest 7", () => { + test("takes one copy a day and keeps the newest 7", async () => { const docId = library.createDoc().id; const taken: string[] = []; for (let day = 0; day < 10; day += 1) { write(docId, `day ${day}`); clock.advance(IDLE_MS); - const first = library.runIdleJobs().dailyCopy; + const first = (await library.runIdleJobs()).dailyCopy; expect(first).not.toBeNull(); taken.push(localDate(clock.now)); - expect(library.runIdleJobs().dailyCopy).toBeNull(); + expect((await library.runIdleJobs()).dailyCopy).toBeNull(); clock.advance(DAY); } expect(library.listDailyCopies()).toEqual(taken.slice(-DAILY_KEEP)); }); - test("a copy restores one document's state as of that day", () => { + test("a copy restores one document's state as of that day", async () => { const docId = library.createDoc().id; write(docId, "before"); clock.advance(IDLE_MS); - library.runIdleJobs(); + await library.runIdleJobs(); const day = localDate(clock.now); write(docId, "after"); const restored = new Y.Doc(); @@ -81,7 +78,7 @@ describe("daily copies (D-A6)", () => { expect(restored.getText("t").toString()).toBe("before"); }); - test("survives a leftover partial copy", () => { + test("survives a leftover partial copy", async () => { const host = createTestHost(clock); host.mkdirp(host.join(temp.dir, "daily")); const partial = host.join( @@ -91,11 +88,11 @@ describe("daily copies (D-A6)", () => { ); host.createExclusive(partial, "junk"); clock.advance(IDLE_MS); - expect(library.runIdleJobs().dailyCopy).not.toBeNull(); + expect((await library.runIdleJobs()).dailyCopy).not.toBeNull(); expect(host.exists(partial)).toBe(false); }); - test("refuses a malformed date", () => { + test("refuses a malformed date", async () => { expect(() => library.readDailyCopy("../../etc", crypto.randomUUID()), ).toThrow(); @@ -107,7 +104,7 @@ describe("daily copy off the writer (begin, copy, complete or fail)", () => { clock.advance(IDLE_MS); } - test("begin waits for idle, hands out one job at a time, and nothing until it ends", () => { + test("begin waits for idle, hands out one job at a time, and nothing until it ends", async () => { write(library.createDoc().id, "x"); expect(library.beginDailyCopy()).toBeNull(); idle(); @@ -117,13 +114,15 @@ describe("daily copy off the writer (begin, copy, complete or fail)", () => { expect(library.beginDailyCopy()).toBeNull(); }); - test("a copy made on its own read-only connection completes like the inline one", () => { + test("a copy made on its own read-only connection completes like the inline one", async () => { const docId = library.createDoc().id; write(docId, "copied"); idle(); const job = library.beginDailyCopy(); if (!job) throw new Error("no job"); - copyLibraryInto(openBunSqlite, job.source, job.partial); + await copyLibraryBatched(openBunSqlite, job.source, job.partial, { + yieldNow: () => new Promise((resolve) => setImmediate(resolve)), + }); // The writer keeps committing while the copy is out. write(docId, " later"); expect(library.completeDailyCopy(job)).toBe( @@ -136,11 +135,11 @@ describe("daily copy off the writer (begin, copy, complete or fail)", () => { expect(library.beginDailyCopy()).toBeNull(); // done for today }); - test("a failed or abandoned copy leaves no partial, keeps older copies and retries", () => { + test("a failed or abandoned copy leaves no partial, keeps older copies and retries", async () => { const docId = library.createDoc().id; write(docId, "day one"); idle(); - library.runIdleJobs(); + await library.runIdleJobs(); const kept = library.listDailyCopies(); clock.advance(DAY); write(docId, " day two"); @@ -155,7 +154,7 @@ describe("daily copy off the writer (begin, copy, complete or fail)", () => { expect(library.beginDailyCopy()).not.toBeNull(); }); - test("opening the library sweeps partial copies a killed process left", () => { + test("opening the library sweeps partial copies a killed process left", async () => { const host = createTestHost(clock); library.shutdown(); host.mkdirp(host.join(temp.dir, "daily")); diff --git a/tests/unit/desktop/storage/doc-store-engine.test.ts b/tests/unit/desktop/storage/doc-store-engine.test.ts index 5c1840b..b772e8c 100644 --- a/tests/unit/desktop/storage/doc-store-engine.test.ts +++ b/tests/unit/desktop/storage/doc-store-engine.test.ts @@ -25,7 +25,7 @@ afterEach(() => { }); describe("dispatch", () => { - test("a failure comes back as an envelope, never a throw", () => { + test("a failure comes back as an envelope, never a throw", async () => { expect( dispatch(library, { id: 2, @@ -40,7 +40,7 @@ describe("dispatch", () => { }); describe("the index change feed", () => { - test("an append that changes the title moves rev and names the document", () => { + test("an append that changes the title moves rev and names the document", async () => { const { id } = library.createDoc(); library.load(id, "w1"); library.takeIndexChanges(); @@ -59,7 +59,7 @@ describe("the index change feed", () => { expect(library.takeIndexChanges()).toBeNull(); }); - test("index writes report each changed row once", () => { + test("index writes report each changed row once", async () => { const folder = library.createFolder({ name: "A" }); const { id } = library.createDoc({ folderId: folder.id }); library.renameDoc(id, "x"); @@ -72,7 +72,7 @@ describe("the index change feed", () => { ]); }); - test("the idle Trash purge is announced", () => { + test("the idle Trash purge is announced", async () => { const clock = createClock(); library.shutdown(); library = openTestLibrary(temp.dir, createTestHost(clock)); @@ -80,7 +80,7 @@ describe("the index change feed", () => { library.trashDoc(id); library.takeIndexChanges(); clock.advance(TRASH_RETENTION_MS + IDLE_MS); - expect(library.runIdleJobs().purged).toBe(1); + expect((await library.runIdleJobs()).purged).toBe(1); expect(library.takeIndexChanges()?.changes).toEqual([{ kind: "doc", id }]); }); }); diff --git a/tests/unit/desktop/storage/library-index.test.ts b/tests/unit/desktop/storage/library-index.test.ts index 0947972..1daf981 100644 --- a/tests/unit/desktop/storage/library-index.test.ts +++ b/tests/unit/desktop/storage/library-index.test.ts @@ -44,7 +44,7 @@ function codeOf(run: () => unknown) { } describe("creating documents", () => { - test("mints a UUID, a lineage and a 32-hex link key at insert", () => { + test("mints a UUID, a lineage and a 32-hex link key at insert", async () => { const a = library.createDoc(); const b = library.createDoc(); expect(a.id).toMatch(/^[0-9a-f-]{36}$/); @@ -54,7 +54,7 @@ describe("creating documents", () => { expect(a.linkKey).not.toBe(b.linkKey); }); - test("keeps a lineage and link key only when given (backup restore)", () => { + test("keeps a lineage and link key only when given (backup restore)", async () => { const lineage = crypto.randomUUID(); const linkKey = "a".repeat(32); expect(library.createDoc({ lineage, linkKey })).toMatchObject({ @@ -66,7 +66,7 @@ describe("creating documents", () => { ); }); - test("refuses malformed ids, link keys and kinds", () => { + test("refuses malformed ids, link keys and kinds", async () => { expect(codeOf(() => library.createDoc({ id: "../x" }))).toBe( "invalid_argument", ); @@ -78,14 +78,14 @@ describe("creating documents", () => { ); }); - test("templates are stored with kind 'template'", () => { + test("templates are stored with kind 'template'", async () => { library.createDoc({ kind: "template", title: "Weekly" }); expect(library.sidebar().docs[0]?.kind).toBe("template"); }); }); describe("the sidebar snapshot", () => { - test("lists live folders, documents and tags; rev moves on every write", () => { + test("lists live folders, documents and tags; rev moves on every write", async () => { const start = library.sidebar().rev; const folder = library.createFolder({ name: "Work" }); const doc = library.createDoc({ title: "Plan", folderId: folder.id }); @@ -107,19 +107,19 @@ describe("the sidebar snapshot", () => { ]); }); - test("tag names are unique regardless of case", () => { + test("tag names are unique regardless of case", async () => { library.createTag("Urgent"); expect(codeOf(() => library.createTag("urgent"))).toBe("invalid_argument"); }); - test("opening a document records opened_at", () => { + test("opening a document records opened_at", async () => { const doc = library.createDoc(); clock.advance(1000); library.load(doc.id, "w1"); expect(library.sidebar().docs[0]?.openedAt).toBe(clock.now); }); - test("rename and move", () => { + test("rename and move", async () => { const folder = library.createFolder({ name: "A" }); const doc = library.createDoc({ title: "x" }); library.renameDoc(doc.id, " Renamed "); @@ -136,7 +136,7 @@ describe("the sidebar snapshot", () => { }); describe("Trash and purge", () => { - test("trash hides a document and restore brings it back to its folder", () => { + test("trash hides a document and restore brings it back to its folder", async () => { const folder = library.createFolder({ name: "A" }); const doc = library.createDoc({ folderId: folder.id }); library.trashDoc(doc.id); @@ -145,7 +145,7 @@ describe("Trash and purge", () => { expect(library.sidebar().docs[0]?.folderId).toBe(folder.id); }); - test("purge cascades to rows, snapshot, tags, view and import source", () => { + test("purge cascades to rows, snapshot, tags, view and import source", async () => { const doc = library.createDoc(); const tag = library.createTag("t"); library.setDocTags(doc.id, [tag.id]); @@ -188,7 +188,7 @@ describe("Trash and purge", () => { library = openTestLibrary(temp.dir, createTestHost(clock)); }); - test("idle jobs empty Trash after 30 days, skipping open documents", () => { + test("idle jobs empty Trash after 30 days, skipping open documents", async () => { const old = library.createDoc(); const open = library.createDoc(); const recent = library.createDoc(); @@ -198,18 +198,18 @@ describe("Trash and purge", () => { library.trashDoc(recent.id); library.load(open.id, "w1"); clock.advance(IDLE_MS + 1000); - expect(library.runIdleJobs().purged).toBe(1); + expect((await library.runIdleJobs()).purged).toBe(1); expect(library.stats().docs).toBe(2); }); }); describe("ids and schema at creation", () => { - test("accepts any safe id and stores the given schema", () => { + test("accepts any safe id and stores the given schema", async () => { const { id } = library.createDoc({ id: "doc-a_1", schema: 3 }); expect(library.load(id, "w1").schema).toBe(3); }); - test("refuses ids that aren't safe in SQL, paths or logs", () => { + test("refuses ids that aren't safe in SQL, paths or logs", async () => { for (const id of ["", "../x", "a b", "a/b", "-x", "x".repeat(65)]) expect(codeOf(() => library.createDoc({ id }))).toBe("invalid_argument"); expect(codeOf(() => library.createDoc({ schema: -1 }))).toBe( From a714704e7dc60133a1ab419881b5fefd6da4bde7 Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 14:15:53 +0330 Subject: [PATCH 3/7] feat(desktop): run the daily copy in its own thread and stop it on quit The storage worker starts copy-worker.ts for each daily copy, so appends and the quit flush never wait on it. Shutdown terminates the copy thread (it yields between batches, so this takes milliseconds), waits for it, deletes the partial file and keeps the previous copy. --- desktop/storage/client.ts | 8 ++- desktop/storage/copy-worker.ts | 36 +++++++++++++ desktop/storage/protocol.ts | 13 ++--- desktop/storage/worker.ts | 96 ++++++++++++++++++++++++++++++---- 4 files changed, 135 insertions(+), 18 deletions(-) create mode 100644 desktop/storage/copy-worker.ts diff --git a/desktop/storage/client.ts b/desktop/storage/client.ts index 42129d8..890f510 100644 --- a/desktop/storage/client.ts +++ b/desktop/storage/client.ts @@ -8,6 +8,7 @@ * pending call rejects with `worker_restarting`, which the renderer's * batcher retries (resending an append is harmless). */ +import { dirname, join } from "node:path"; import { Worker } from "node:worker_threads"; import type { StorageErrorEnvelope } from "./engine/errors"; @@ -86,7 +87,12 @@ export class StorageClient { events: StorageClientEvents = {}, ): Promise { const started = performance.now(); - const worker = new Worker(workerPath, { workerData: options }); + const worker = new Worker(workerPath, { + workerData: { + copyWorkerPath: join(dirname(workerPath), "copy-worker.cjs"), + ...options, + }, + }); return new Promise((resolve, reject) => { const onError = (error: Error) => reject(error); worker.once("error", onError); diff --git a/desktop/storage/copy-worker.ts b/desktop/storage/copy-worker.ts new file mode 100644 index 0000000..e5b5359 --- /dev/null +++ b/desktop/storage/copy-worker.ts @@ -0,0 +1,36 @@ +/** + * The daily-copy thread (P2.6 decision 5). The storage worker starts one per + * copy; it reads the library through its own read-only connection, so the + * writer never waits on it, and it yields between batches, so the storage + * worker can `terminate()` it on quit within milliseconds. + * + * Replies once: `{ ok: true, report, ms }` or `{ ok: false, code, message }`. + */ +import { parentPort, workerData } from "node:worker_threads"; + +import { openNodeSqlite } from "./drivers/node-sqlite"; +import { copyLibraryBatched } from "./engine/copy-library"; +import { toEnvelope } from "./engine/errors"; + +export type CopyWorkerData = { + source: string; + partial: string; + batchRows?: number; +}; + +const port = parentPort; +if (!port) throw new Error("the copy worker must run in a worker_thread"); +const { source, partial, batchRows } = workerData as CopyWorkerData; +const started = performance.now(); + +copyLibraryBatched(openNodeSqlite, source, partial, { + batchRows, + yieldNow: () => new Promise((resolve) => setImmediate(resolve)), +}).then( + (report) => + port.postMessage({ ok: true, report, ms: performance.now() - started }), + (error) => { + const { code, message } = toEnvelope(error); + port.postMessage({ ok: false, code, message }); + }, +); diff --git a/desktop/storage/protocol.ts b/desktop/storage/protocol.ts index 1d238cd..4f211be 100644 --- a/desktop/storage/protocol.ts +++ b/desktop/storage/protocol.ts @@ -58,10 +58,6 @@ export type StorageOps = { setDocTags: { args: { docId: string; tagIds: string[] }; result: number }; listDailyCopies: { args: Record; result: string[] }; readDailyCopy: { args: { date: string; docId: string }; result: Uint8Array }; - runIdleJobs: { - args: Record; - result: ReturnType; - }; stats: { args: Record; result: ReturnType }; /** The quit flush's last step: checkpoint, clear `unclean_exit`, close. */ shutdown: { args: Record; result: null }; @@ -95,6 +91,13 @@ export type StorageWorkerOptions = { trace?: boolean; /** How often the worker looks for idle work, in ms (default 5000). */ idlePollMs?: number; + /** Soak only: how long without a write counts as idle (default 30 s). */ + idleMs?: number; + /** + * The daily-copy thread's bundle (`copy-worker.cjs`). `StorageClient` + * defaults it to the worker's sibling. Without it, no daily copy runs. + */ + copyWorkerPath?: string; }; function run(library: Library, request: StorageRequest): unknown { @@ -155,8 +158,6 @@ function run(library: Library, request: StorageRequest): unknown { return library.listDailyCopies(); case "readDailyCopy": return library.readDailyCopy(request.args.date, request.args.docId); - case "runIdleJobs": - return library.runIdleJobs(); case "stats": return library.stats(); case "shutdown": diff --git a/desktop/storage/worker.ts b/desktop/storage/worker.ts index f402001..469ba71 100644 --- a/desktop/storage/worker.ts +++ b/desktop/storage/worker.ts @@ -6,17 +6,20 @@ * behind appends runs after they have committed. * * Messages: `ready` (or `open_failed`) once, then one `reply` per request, - * and `index_changed` after a metadata write. Idle jobs (daily copy, Trash) - * run between requests after 30 s without a write. + * and `index_changed` after a metadata write. Idle jobs run between + * requests after 30 s without a write: the Trash purge here, and the daily + * copy in its own thread (`copy-worker.ts`), so a long copy never holds an + * append or the quit flush. * * With `trace`, the engine's protocol lines (`BATCH_BEGIN`, `ACK `, * `COMPACT_BEGIN`, `COMPACT_ACK`) go straight to fd 1 with `writeSync`, so * the durability soak knows what committed before it kills the process. */ import { writeSync } from "node:fs"; -import { parentPort, workerData } from "node:worker_threads"; +import { parentPort, Worker, workerData } from "node:worker_threads"; import { openNodeSqlite } from "./drivers/node-sqlite"; +import type { DailyCopyJob } from "./engine/daily-copy"; import { toEnvelope } from "./engine/errors"; import { Library } from "./engine/library"; import { createNodeHost } from "./node-host"; @@ -28,8 +31,9 @@ import { transferablesOf, } from "./protocol"; +if (!parentPort) + throw new Error("the storage worker must run in a worker_thread"); const port = parentPort; -if (!port) throw new Error("the storage worker must run in a worker_thread"); const options = workerData as StorageWorkerOptions; @@ -56,6 +60,7 @@ try { fullfsync: options.fullfsync, synchronous: options.synchronous, trace: options.trace ? line : undefined, + idleMs: options.idleMs, }); post({ type: "ready", @@ -72,14 +77,71 @@ function announceChanges(open: Library) { if (changed) post({ type: "index_changed", ...changed }); } +/** The daily copy running in its own thread, if any (decision 5). */ +let copy: { job: DailyCopyJob; thread: Worker } | null = null; + +function startCopy(open: Library, job: DailyCopyJob, copyWorkerPath: string) { + const thread = new Worker(copyWorkerPath, { + workerData: { source: job.source, partial: job.partial }, + }); + copy = { job, thread }; + const settle = (reason: string | null, ms?: number) => { + if (copy?.thread !== thread) return; + copy = null; + if (!library) return; // shut down meanwhile; shutdown abandoned the job + try { + if (reason === null) { + open.completeDailyCopy(job, ms ?? null); + } else open.failDailyCopy(job, reason); + } catch (error) { + open.failDailyCopy(job, `daily_copy_failed: ${toEnvelope(error).code}`); + } + announceChanges(open); + }; + thread.once("message", (reply: { ok: boolean; code?: string; ms?: number }) => + settle(reply.ok ? null : `daily_copy_failed: ${reply.code}`, reply.ms), + ); + thread.once("error", (error) => + settle(`daily_copy_failed: ${String(error)}`), + ); + thread.once("exit", (code) => settle(`daily_copy_failed: exit ${code}`)); +} + +/** + * Stops a running copy for quit (decision 5): the thread yields between + * batches, so this takes milliseconds, and waiting for it closes the + * partial file before `shutdown` deletes it. Never longer than 500 ms. + */ +async function stopCopy() { + const running = copy; + if (!running) return; + copy = null; + const started = performance.now(); + let guard: ReturnType | undefined; + const outcome = await Promise.race([ + running.thread.terminate().then(() => "stopped"), + new Promise((resolve) => { + guard = setTimeout(resolve, 500, "timed_out"); + }), + ]); + // A pending timer would keep this thread alive after shutdown. + clearTimeout(guard); + if (options.trace) + line(`COPY_STOPPED ${outcome} ${Math.round(performance.now() - started)}`); +} + let idleTimer: ReturnType | null = null; if (library) { const open = library; idleTimer = setInterval(() => { try { - open.runIdleJobs(); + if (!copy && options.copyWorkerPath) { + const job = open.beginDailyCopy(); + if (job) startCopy(open, job, options.copyWorkerPath); + } + open.purgeExpiredTrash(); } catch { - // Only the Trash purge can throw here; it retries on the next pass. + // A failed purge or copy start retries on the next pass. } announceChanges(open); }, options.idlePollMs ?? 5000); @@ -103,13 +165,25 @@ port.on("message", (request: StorageRequest) => { return; } const open = library; - const result = dispatch(open, request); - post({ type: "reply", id: request.id, result }); if (request.op === "shutdown") { - if (idleTimer) clearInterval(idleTimer); - library = null; - port.close(); + void shutDown(open, request); return; } + const result = dispatch(open, request); + post({ type: "reply", id: request.id, result }); announceChanges(open); }); + +/** + * The quit flush's last step. Every request sent before it has committed: + * the writer never waited on the daily copy. Stop the copy, then + * checkpoint, clear `unclean_exit`, close and release the lock. + */ +async function shutDown(open: Library, request: StorageRequest) { + if (idleTimer) clearInterval(idleTimer); + await stopCopy(); + const result = dispatch(open, request); + library = null; + post({ type: "reply", id: request.id, result }); + port.close(); +} From 2a7782dff51a27608e106f8e0c230aa75987ea4d Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 14:15:53 +0330 Subject: [PATCH 4/7] test(desktop): soak a quit during a daily copy T4-copy quits the storage worker while a daily copy runs in its thread and checks that every typed edit is stored, the shutdown is clean, no partial copy is left and the previous copy is byte-identical. T8 times the batched copy on node:sqlite and compares its size with VACUUM INTO's on the year-end library. --- .github/workflows/desktop-storage-soak.yml | 4 +- desktop/storage-harness/build.ts | 1 + desktop/storage-harness/main.ts | 85 +++++++++++ tests/load/desktop-durability-soak.ts | 159 ++++++++++++++++++++- 4 files changed, 244 insertions(+), 5 deletions(-) diff --git a/.github/workflows/desktop-storage-soak.yml b/.github/workflows/desktop-storage-soak.yml index aca44f4..9dfc938 100644 --- a/.github/workflows/desktop-storage-soak.yml +++ b/.github/workflows/desktop-storage-soak.yml @@ -55,8 +55,8 @@ jobs: matrix: os: [macos-26, windows-latest, ubuntu-24.04] group: - - name: contract, T3–T6 - cases: contract,T3,T4,T5,T6 + - name: contract, T3–T6, quit during a copy + cases: contract,T3,T4,T4-copy,T5,T6 - name: T1 kill during append cases: T1,T1-dense - name: T2 kill during compaction diff --git a/desktop/storage-harness/build.ts b/desktop/storage-harness/build.ts index 194b77b..79d3f5c 100644 --- a/desktop/storage-harness/build.ts +++ b/desktop/storage-harness/build.ts @@ -10,6 +10,7 @@ const result = await Bun.build({ entrypoints: [ path.join(root, "main.ts"), path.join(root, "../storage/worker.ts"), + path.join(root, "../storage/copy-worker.ts"), ], outdir: path.join(root, "dist"), target: "node", diff --git a/desktop/storage-harness/main.ts b/desktop/storage-harness/main.ts index 2971574..e04544b 100644 --- a/desktop/storage-harness/main.ts +++ b/desktop/storage-harness/main.ts @@ -17,6 +17,8 @@ * - `readonly`: opens a newer library (T6) and tries to write. * - `commit-timing`: append round trips with `fullfsync` on or off (T3). * - `startup`: worker spawn → ready → sidebar rows (T8, the §10 budget). + * - `copy-quit`: quit while the daily copy runs in its thread (decision 5). + * - `copy-time`: how long the batched daily copy takes (decision 5). * * An uncaught throw in Electron main opens a modal dialog, so every failure * exits with `process.exit(1)` after printing `HARNESS_FAIL`. @@ -279,6 +281,87 @@ async function commitTiming() { await client.shutdown(); } +/** + * Decision 5: quit while the daily copy runs in its thread. Waits for the + * library to go idle (the copy starts), keeps typing through the copy (the + * writer must not wait on it), then runs the quit flush. + */ +async function copyQuit() { + const docId = args.doc ?? fail("--doc is required"); + const client = await StorageClient.start(workerPath(), { + dir: dir(), + trace: true, + idleMs: 300, + idlePollMs: 100, + }); + const owner = "copy-quit"; + const loaded = await client.call("load", { docId, owner }); + const doc = new Y.Doc(); + if (loaded.snapshot) Y.applyUpdate(doc, loaded.snapshot); + for (const update of loaded.updates) Y.applyUpdate(doc, update); + let k = + editCount(doc.getText(TEXT).toString()) ?? fail("stored text is broken"); + say(`READY ${k}`); + // Stay idle until the copy thread is running, then type through it. + const waitUntil = performance.now() + 10_000; + while (!(await client.call("stats", {})).dailyCopyRunning) { + if (performance.now() > waitUntil) fail("the daily copy never started"); + await new Promise((r) => setTimeout(r, 10)); + } + const appendMs: number[] = []; + const typeFor = Number(args["type-ms"] ?? 150); + const started = performance.now(); + while (performance.now() - started < typeFor) { + k += 1; + let update: Uint8Array = new Uint8Array(); + doc.once("update", (u: Uint8Array) => { + update = u; + }); + const text = doc.getText(TEXT); + text.insert(text.length, `${k};`); + const t0 = performance.now(); + await client.call("append", { + docId, + owner, + record: { origin: "local", update }, + }); + appendMs.push(performance.now() - t0); + await new Promise((r) => setTimeout(r, 20)); + } + const quitAt = performance.now(); + const outcome = await client.shutdown(); + say( + `QUIT ${outcome} ${k} ${Math.round(performance.now() - quitAt)} ${Math.round(Math.max(...appendMs))}`, + ); +} + +/** Decision 5: how long the batched daily copy takes on node:sqlite. */ +async function copyTime() { + const client = await StorageClient.start(workerPath(), { + dir: dir(), + idleMs: 0, + idlePollMs: 50, + }); + const today = (await client.call("stats", {})).today; + const deadline = performance.now() + 300_000; + for (;;) { + const stats = await client.call("stats", {}); + if (stats.lastDailyCopy === today || stats.lastDailyCopyError) { + say( + `COPYTIME ${JSON.stringify({ + ms: stats.lastDailyCopyMs, + error: stats.lastDailyCopyError, + libraryBytes: stats.bytes, + })}`, + ); + break; + } + if (performance.now() > deadline) fail("the daily copy never finished"); + await new Promise((r) => setTimeout(r, 100)); + } + await client.shutdown(); +} + async function startup() { // performance.now() counts from process start in Node and Electron main. const spawnAt = performance.now(); @@ -310,6 +393,8 @@ const MODES: Record Promise> = { readonly, "commit-timing": commitTiming, startup, + "copy-quit": copyQuit, + "copy-time": copyTime, }; app.whenReady().then(async () => { diff --git a/tests/load/desktop-durability-soak.ts b/tests/load/desktop-durability-soak.ts index 5fa349e..c0c68f3 100644 --- a/tests/load/desktop-durability-soak.ts +++ b/tests/load/desktop-durability-soak.ts @@ -8,7 +8,7 @@ * the two drivers read each other's files. The shape follows spike S2's * kill harness. * - * bun tests/load/desktop-durability-soak.ts --case=all|contract|T1|T2|T3|T4|T5|T6|T8 + * bun tests/load/desktop-durability-soak.ts --case=all|contract|T1|T2|T3|T4|T4-copy|T5|T6|T8 * [--kills=200] [--out=soak-out] [--verbose] * * T7 (slow-hydrate overwrite) is a web-library case: it tests the session @@ -28,6 +28,7 @@ import { mkdirSync, mkdtempSync, readFileSync, + renameSync, rmSync, statSync, writeFileSync, @@ -695,7 +696,7 @@ async function caseT8(): Promise { } // Idle: the daily copy and the Trash purge. clock.now += IDLE_MS; - const idle = library.runIdleJobs(); + const idle = await library.runIdleJobs(); if (idle.dailyCopyMs !== null) copyMs.push(idle.dailyCopyMs); if (idle.dailyCopy === null && idle.dailyCopyMs !== null) failures.push( @@ -791,11 +792,15 @@ async function caseT8(): Promise { const sortedCopies = [...copyMs].sort((a, b) => a - b); const copyMedian = sortedCopies[Math.floor(sortedCopies.length / 2)] ?? 0; const copyMax = sortedCopies.at(-1) ?? 0; + // Decision 5: the batched copy in its thread on node:sqlite, against + // VACUUM INTO on the same year-end library. + const comparison = args["no-electron"] ? null : await compareCopies(dir); + if (comparison?.error) failures.push(`batched copy: ${comparison.error}`); removeDir(dir); return { name: "T8", pass: failures.length === 0, - summary: `${days} days, ${appends} appends, ${compactions} compactions, ${trashed} trashed; ${(seededBytes / 1048576).toFixed(0)} → ${(stats.bytes / 1048576).toFixed(0)} MB; ${dailies.length} daily copies (VACUUM INTO median ${copyMedian.toFixed(0)} ms, max ${copyMax.toFixed(0)} ms); ${liveDocs} documents: worker ready ${toWorkerReady.toFixed(0)} ms and sidebar rows ${toSidebar.toFixed(0)} ms after process start (median of 5; §10 ${budget.workerReadyMs} / ${budget.sidebarMs} ms; excludes renderer IPC and paint); unclean-exit open with quick_check ${unclean ? unclean.openMs.toFixed(0) : "n/a"} ms`, + summary: `${days} days, ${appends} appends, ${compactions} compactions, ${trashed} trashed; ${(seededBytes / 1048576).toFixed(0)} → ${(stats.bytes / 1048576).toFixed(0)} MB; ${dailies.length} daily copies (bun inline median ${copyMedian.toFixed(0)} ms, max ${copyMax.toFixed(0)} ms)${comparison ? `; batched copy on node:sqlite ${comparison.batchedMs?.toFixed(0)} ms, ${(comparison.batchedBytes / 1048576).toFixed(1)} MB vs VACUUM INTO ${comparison.vacuumMs.toFixed(0)} ms, ${(comparison.vacuumBytes / 1048576).toFixed(1)} MB` : ""}; ${liveDocs} documents: worker ready ${toWorkerReady.toFixed(0)} ms and sidebar rows ${toSidebar.toFixed(0)} ms after process start (median of 5; §10 ${budget.workerReadyMs} / ${budget.sidebarMs} ms; excludes renderer IPC and paint); unclean-exit open with quick_check ${unclean ? unclean.openMs.toFixed(0) : "n/a"} ms`, details: { days, docs: liveDocs, @@ -809,12 +814,159 @@ async function caseT8(): Promise { unclean, budget, dailyCopyMs: { median: copyMedian, max: copyMax, count: copyMs.length }, + comparison, simMs, }, failures, }; } +/** Batched copy (Electron, its thread) vs VACUUM INTO (bun), same library. */ +async function compareCopies(dir: string) { + const daily = join(dir, "daily"); + const before = new Set(createNodeHost().list(daily)); + const run = await runHarness(["--mode=copy-time", `--dir=${dir}`]); + const line = run.lines.find((l) => l.startsWith("COPYTIME ")); + if (!line) + return { + error: run.stderr.slice(0, 500), + batchedMs: null, + batchedBytes: 0, + vacuumMs: 0, + vacuumBytes: 0, + }; + const timing = JSON.parse(line.slice(9)) as { + ms: number | null; + error: string | null; + }; + const made = createNodeHost() + .list(daily) + .find((file) => !before.has(file) && file.endsWith(".sqlite")); + const batchedBytes = made ? statSync(join(daily, made)).size : 0; + const vacuumPath = join(dir, "vacuum-into.sqlite"); + // Read-write: a read-only open of a WAL file with no -shm can't start. + const db = new Database(join(dir, LIBRARY_FILE)); + const t0 = performance.now(); + db.prepare("VACUUM INTO ?").run(vacuumPath); + const vacuumMs = performance.now() - t0; + db.close(); + return { + error: timing.error ?? (made ? null : "no copy file"), + batchedMs: timing.ms, + batchedBytes, + vacuumMs, + vacuumBytes: statSync(vacuumPath).size, + }; +} + +// ── T4-copy: quit while the daily copy runs (decision 5) ────────────── + +function yesterday(): string { + const date = new Date(Date.now() - 24 * 60 * 60 * 1000); + const pad = (n: number) => String(n).padStart(2, "0"); + return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`; +} + +async function caseQuitDuringCopy(runs: number): Promise { + const dir = mkdtempSync(join(tmpdir(), "soak-t4c-")); + const docs = Number(args["copy-docs"] ?? 1000); + const failures: string[] = []; + // A library big enough that the copy outlasts the typing: ~200 MB. + const library = openEngine(dir); + const pad = new Y.Doc(); + pad.getText(PAD).insert(0, "p".repeat(200 * 1024)); + const padState = Y.encodeStateAsUpdate(pad); + for (let i = 0; i < docs; i += 1) { + const { id } = library.createDoc({ title: `Doc ${i}` }); + library.load(id, "seed"); + library.compact(id, "seed", padState, 0); + library.close(id, "seed"); + } + const { id: docId } = library.createDoc({ title: "Typed" }); + // The previous good copy, dated yesterday: it must survive every quit. + const previous = await library.takeDailyCopyNow(); + library.shutdown(); + const daily = join(dir, "daily"); + const previousPath = join(daily, `library-${yesterday()}.sqlite`); + renameSync(join(daily, previous ?? ""), previousPath); + const resetDay = () => { + const db = new Database(join(dir, LIBRARY_FILE)); + db.prepare("UPDATE meta SET value = ? WHERE key = 'last_daily_copy'").run( + yesterday(), + ); + db.close(); + }; + resetDay(); + const hashOf = (path: string) => + createHash("sha256").update(readFileSync(path)).digest("hex"); + const previousHash = hashOf(previousPath); + let midCopy = 0; + const quitMs: number[] = []; + const exitLagMs: number[] = []; + const appendMaxMs: number[] = []; + for (let i = 1; i <= runs; i += 1) { + let quitSeenAt = 0; + let began = false; + let done = false; + const run = await runHarness( + ["--mode=copy-quit", `--dir=${dir}`, `--doc=${docId}`], + (line) => { + if (line.startsWith("COPY_BEGIN")) began = true; + if (line.startsWith("COPY_DONE")) done = true; + if (line.startsWith("QUIT ")) quitSeenAt = performance.now(); + }, + ); + const exitedAt = performance.now(); + note(run.lines.filter((l) => !/^(BATCH_BEGIN|ACK)/.test(l)).join(" | ")); + const quit = run.lines.find((l) => l.startsWith("QUIT ")); + if (run.exitCode !== 0 || !quit) { + failures.push( + `run ${i}: exit ${run.exitCode} ${run.stderr.slice(0, 500)}`, + ); + continue; + } + const [, outcome, kText, quitText, appendText] = quit.split(" "); + if (began && !done) midCopy += 1; + quitMs.push(Number(quitText)); + appendMaxMs.push(Number(appendText)); + exitLagMs.push(exitedAt - quitSeenAt); + if (outcome !== "clean") failures.push(`run ${i}: shutdown ${outcome}`); + if (Number(quitText) > 2000) + failures.push(`run ${i}: shutdown took ${quitText} ms (budget 2 s)`); + const partials = createNodeHost() + .list(daily) + .filter((file) => file.endsWith(".partial")); + if (partials.length > 0) + failures.push( + `run ${i}: partial copy left behind: ${partials.join(", ")}`, + ); + if (!existsSync(previousPath) || hashOf(previousPath) !== previousHash) + failures.push(`run ${i}: the previous good copy changed`); + const reopened = openEngine(dir); + if (reopened.status.uncleanExit) + failures.push(`run ${i}: unclean_exit set`); + const k = editCount(textOf(reopened, docId).text); + reopened.shutdown(); + if (k !== Number(kText)) + failures.push(`run ${i}: ${k} edits stored, ${kText} typed before quit`); + // A run whose copy finished leaves today's copy: undo it for the next run. + for (const file of createNodeHost().list(daily)) + if (file.endsWith(".sqlite") && join(daily, file) !== previousPath) + removeDir(join(daily, file)); + resetDay(); + } + removeDir(dir); + const max = (values: number[]) => Math.round(Math.max(0, ...values)); + if (midCopy === 0) failures.push("no quit landed during a copy"); + return { + name: "T4-copy", + pass: failures.length === 0, + summary: `${runs} quits, ${midCopy} during a daily copy: every typed edit stored, clean shutdowns (max ${max(quitMs)} ms), process gone ${max(exitLagMs)} ms after the quit at worst, no partial copy left, the previous copy byte-identical; slowest append during a copy ${max(appendMaxMs)} ms`, + details: { runs, midCopy, quitMs, exitLagMs, appendMaxMs }, + failures, + }; +} + // ── Main ──────────────────────────────────────────────────────────────── const CASES: Record Promise> = { @@ -825,6 +977,7 @@ const CASES: Record Promise> = { T2: () => killLoop("T2", "compaction", KILLS), T3: caseT3, T4: () => caseT4(Number(args.quits ?? 20)), + "T4-copy": () => caseQuitDuringCopy(Number(args["copy-quits"] ?? 10)), T5: () => caseT5(Number(args["migration-kills"] ?? 40)), T6: caseT6, T8: caseT8, From e7463ded4663bee28891a3ef4abf42f49a11d2a3 Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 14:15:54 +0330 Subject: [PATCH 5/7] docs(desktop): record the quit budget and the batched daily copy --- CHANGELOG.md | 28 +++++++++++++++++++ .../specs/2026-11-15-desktop-architecture.md | 10 ++++++- 2 files changed, 37 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e2e957c..cbc5f5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,34 @@ since the previous tag. ## [Unreleased] +### Desktop P2.6 decision 5: daily copies never hold up a save or a quit + +A daily `VACUUM INTO` on the storage worker blocked every save behind it, for +up to 9 s on a Windows runner. That included the quit flush, so a quit during +a copy could drop the final batch of edits. The copy now runs in its own +thread and can be stopped. + +#### Changed + +- **Desktop:** the daily copy (`desktop/storage/engine/copy-library.ts`) + - copies the library in batches of at most 256 rows or about 4 MiB, from + one read-only snapshot, inside one transaction; + - creates the schema verbatim and copies `sqlite_sequence`, + `user_version` and `application_id`; + - checks row counts and `integrity_check` before the copy is renamed into + place, then runs a passive WAL checkpoint; + - on failure, keeps the previous copies and records `daily_copy_failed`. +- **Desktop:** the storage worker runs that copy in a second thread + (`desktop/storage/copy-worker.ts`), so saves never wait on it. On quit it + stops the copy in milliseconds and deletes the partial file; the previous + copy stays. Opening the library sweeps any partial copy left behind. +- **Tests:** soak case `T4-copy` quits during a copy on all three OSes. T8 + compares the copy's time and size with `VACUUM INTO`'s on the year-end + library. A unit test restores a copy and runs the `DocStore` contract on it. +- **Docs:** the architecture spec records the quit budget (3 s flush, then + 2 s worker shutdown), the batched copy, and `utilityProcess` as the + upgrade path once P2.5 exists. + ### Desktop P2.6: the library storage worker The SQLite engine behind the desktop library, where documents, folders and diff --git a/docs/superpowers/specs/2026-11-15-desktop-architecture.md b/docs/superpowers/specs/2026-11-15-desktop-architecture.md index 5fd5f1b..9ddd631 100644 --- a/docs/superpowers/specs/2026-11-15-desktop-architecture.md +++ b/docs/superpowers/specs/2026-11-15-desktop-architecture.md @@ -297,12 +297,19 @@ INDEX import_sources(source_hash) | `migrations.ts` | `format_version` runner:
- **newer than the app:** read-only, with `library_too_new`
- **older:** take `VACUUM INTO library.pre-v-.sqlite` (keep 2), then run each step in its own transaction; a failure means read-only with `migration_failed` | | `content.ts` | `load`, `append` (an INSERT plus the `docs` title and `modified_at`, in one transaction), `compact`, `stateOf` (`Y.mergeUpdates`, for `.tinydoc` and restore) | | `index.ts` | CRUD on `docs` and `folders`, `rev` and the change feed, metadata search (D-A8: titles, tags, folder names and properties, matched on normalised strings; never body text), purge of Trash after 30 days | -| `daily-copy.ts` | Once a day on the first idle: `VACUUM INTO daily/library-.sqlite`, keeping 7. "Restore from a daily snapshot" opens a copy read-only and returns one document's state, which the renderer applies to the open document as a forward edit through `updateYFragment` (the `schema-repair.ts` approach). | +| `daily-copy.ts` | Once a day on the first idle: a copy to `daily/library-.sqlite`, keeping 7. Since decision 5 (P2.6), the copy is `copy-library.ts`, run in its own thread; see "Daily copies off the writer" below. "Restore from a daily snapshot" opens a copy read-only and returns one document's state, which the renderer applies to the open document as a forward edit through `updateYFragment` (the `schema-repair.ts` approach). | | `mirror.ts` | For each document where `modified_at > mirrored_at` and the renderer has sent Markdown: write `-.tinydoc` (`TINYDOC2`: magic, JSON header with `v`, `minReader`, `enc: null` and `lineage`, then `stateOf`; v8, S8) and `.md`, writing each to a temp file and then renaming it. It keeps one current copy per document. When a document is permanently deleted, its copies move to `Deleted/` and are removed after 30 days. Image files go once to `assets/.` beside them, and backup `.tinydoc` files reference them as external assets (image store §5.4). No build writes or reads `TINYDOC1` (D40). | | `filenames.ts` | Cleans names: path separators, `..`, Windows reserved names, trailing dots and spaces, and paths over 240 characters | | `assets.ts` (P2.6b) | Owns `/assets/`: the temp-fsync-rename write order, dedupe, the startup `tmp/` sweep, `recount` and GC with a 30-day grace (image store §3.2–§3.4). | | `jobs.ts` | One queue. Foreground calls run first; daily copy and mirror run on idle, meaning 30 s without an append. Each job yields between documents. | +**Daily copies off the writer (P2.6 decision 5).** `VACUUM INTO` on the storage worker blocked every append and the quit flush behind it: up to 9 s for a 240 MB library on `windows-latest`. Moved to a second `worker_thread`, it still couldn't be stopped: `terminate()` doesn't interrupt a native SQLite call, Node joins worker threads on exit, and Electron 45's `node:sqlite` has no `interrupt()`. (`backup()` restarts on every write from the writer, so it never finishes while someone types.) So: +- `copy-library.ts` copies in batches from one snapshot. The library is attached read-only (`mode=ro`) to the copy's own connection, and the whole copy is one transaction there. Tables are created from `sqlite_master` verbatim, then the rows, then indexes and triggers. `sqlite_sequence`, `user_version` and `application_id` are copied too. Batches hold at most 256 rows or about 4 MiB, sized by `LENGTH()` from the record headers, and the copy yields between them. +- The copy is verified before it is kept: row counts per table against the snapshot, and `integrity_check`, all inside the transaction. On a mismatch it rolls back. Only a verified copy is renamed over the day's file, then a passive WAL checkpoint runs. A failure keeps the previous copies, records `daily_copy_failed` and retries on the next idle pass. +- `copy-worker.ts` runs the copy in a thread that the storage worker starts. The writer never waits on it. +- On quit, the worker terminates the copy thread (6–9 ms measured), waits for it, deletes the partial file and keeps the previous copy. The next open sweeps any partial left behind. Soak `T4-copy` checks this on all three OSes: every typed edit stored, a clean shutdown, no partial, and the previous copy byte-identical. +- **Upgrade path (option B):** once P2.5's main process exists, `VACUUM INTO` can run in an Electron `utilityProcess`, which `kill()` stops at once. That would give back `VACUUM INTO`'s compaction of free pages. The batched copy keeps every row and skips the library's free pages, so its size is close; T8 reports both. + **As built in P2.6.** - **Modules.** `index.ts` is `library-index.ts` (not a barrel). `library.ts` holds the open order, the registry and the lifecycle; `lock.ts`, `cloud-folder.ts`, `sql.ts`, `driver.ts` (the `SqliteDriver` seam and SQLite error mapping) and `host.ts` (an injected `StorageHost` for files, the pid check and the clock) keep the engine free of `node:*`. `jobs.ts` is two lines in the worker: idle jobs run between requests after 30 s without a write. `mirror.ts`, `filenames.ts` and `assets.ts` come with P2.10 and P2.6b. - **Open order.** Cloud-folder refusal, lock, pragmas, schema creation for a new file, then `format_version` before any other write (a newer library is reopened read-only and never written), then `quick_check` if `unclean_exit` was set (a failure is read-only, `corrupt`), then migrations, then `unclean_exit` is set. @@ -411,6 +418,7 @@ stateDiagram-v2 Ready --> Degraded: "3 crashes in 1 min → read-only, Library unavailable" ``` +- **Quit budget:** `Flushing` gets 3 s (every window flushes its batches), then `WorkerShutdown` gets 2 s (checkpoint, clear `unclean_exit`, close, release the lock), and then the worker is terminated. The WAL keeps every committed row. A daily copy never counts against this budget: it runs in its own thread, which shutdown stops in milliseconds (§5.4, decision 5). The worker's appends commit throughout, so the final batch is durable. - **Re-entrancy:** `quitAndInstall` fires `before-quit` again, and the coordinator ignores it once it is past `Flushing`. - **No `beforeunload` on desktop.** It would silently block quitting (`editor-app.tsx:58-62`). From 9e8d8d9778a3e194a0ea01082aa0fc1a8ea2251d Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 14:44:53 +0330 Subject: [PATCH 6/7] fix(desktop): let the bun:sqlite driver open URI filenames MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The daily copy attaches the library as file:…?mode=ro. Bun's bundled SQLite on Linux and Windows treats that as a plain file name unless the connection is opened with SQLITE_OPEN_URI, so every copy failed with io_error there; macOS's system SQLite and node:sqlite accept URIs. --- desktop/dev/bun-sqlite-driver.ts | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/desktop/dev/bun-sqlite-driver.ts b/desktop/dev/bun-sqlite-driver.ts index 4e91520..08473de 100644 --- a/desktop/dev/bun-sqlite-driver.ts +++ b/desktop/dev/bun-sqlite-driver.ts @@ -8,7 +8,7 @@ * connection with live statements open as a zombie that keeps the file * handles, which on Windows blocks deleting or replacing the library. */ -import { Database } from "bun:sqlite"; +import { constants, Database } from "bun:sqlite"; import type { OpenSqlite, @@ -19,9 +19,16 @@ import type { } from "@/desktop/storage/engine/driver"; export const openBunSqlite: OpenSqlite = (path, { readOnly }) => { - const db = readOnly - ? new Database(path, { readonly: true }) - : new Database(path, { create: true }); + // SQLITE_OPEN_URI: the daily copy ATTACHes the library as + // `file:…?mode=ro`. node:sqlite accepts URIs by default; Bun's bundled + // SQLite (Linux, Windows) only with this flag. + const db = new Database( + path, + constants.SQLITE_OPEN_URI | + (readOnly + ? constants.SQLITE_OPEN_READONLY + : constants.SQLITE_OPEN_READWRITE | constants.SQLITE_OPEN_CREATE), + ); const cache = new Map(); const statements: { finalize(): void }[] = []; const driver: SqliteDriver = { From 056daf311e359fdbce3c8fb7235dd969b3f774d2 Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 14:44:53 +0330 Subject: [PATCH 7/7] test(desktop): compare copies in an empty folder and fail T4-copy setup clearly The year's daily copies are dated ahead of the runner's clock, so the real-clock copy sorted oldest and was pruned before T8 could measure it. T4-copy now stops with the copy error if its setup copy fails. --- tests/load/desktop-durability-soak.ts | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/load/desktop-durability-soak.ts b/tests/load/desktop-durability-soak.ts index c0c68f3..b72c746 100644 --- a/tests/load/desktop-durability-soak.ts +++ b/tests/load/desktop-durability-soak.ts @@ -824,7 +824,10 @@ async function caseT8(): Promise { /** Batched copy (Electron, its thread) vs VACUUM INTO (bun), same library. */ async function compareCopies(dir: string) { const daily = join(dir, "daily"); - const before = new Set(createNodeHost().list(daily)); + // The year's copies are dated ahead of today's real clock: today's copy + // would sort oldest and be pruned at once, so start from an empty folder. + removeDir(daily); + const before = new Set(); const run = await runHarness(["--mode=copy-time", `--dir=${dir}`]); const line = run.lines.find((l) => l.startsWith("COPYTIME ")); if (!line) @@ -885,10 +888,12 @@ async function caseQuitDuringCopy(runs: number): Promise { const { id: docId } = library.createDoc({ title: "Typed" }); // The previous good copy, dated yesterday: it must survive every quit. const previous = await library.takeDailyCopyNow(); + const copyError = library.stats().lastDailyCopyError; library.shutdown(); + if (!previous) throw new Error(`setup copy failed: ${copyError}`); const daily = join(dir, "daily"); const previousPath = join(daily, `library-${yesterday()}.sqlite`); - renameSync(join(daily, previous ?? ""), previousPath); + renameSync(join(daily, previous), previousPath); const resetDay = () => { const db = new Database(join(dir, LIBRARY_FILE)); db.prepare("UPDATE meta SET value = ? WHERE key = 'last_daily_copy'").run(