From c49d549c4b9b1f0995c4e03e4b3f1af23cd40f9a Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:25:02 +0330 Subject: [PATCH 01/19] build(desktop): typecheck the storage worker in a Node-only project MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tsconfig.desktop-node.json covers desktop/{storage,main,build} with Node types and no Bun types, so a Bun-only API fails the typecheck in code that runs inside Electron (desktop architecture §3.2). The root project excludes those folders. --- package.json | 2 +- tsconfig.desktop-node.json | 19 +++++++++++++++++++ tsconfig.json | 10 +++++++++- 3 files changed, 29 insertions(+), 2 deletions(-) create mode 100644 tsconfig.desktop-node.json diff --git a/package.json b/package.json index 6b4393af..630944f6 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "build": "next build", "start": "next start", "lint": "eslint .", - "typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.sw.json", + "typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.sw.json && tsc --noEmit -p tsconfig.desktop-node.json", "format": "biome format --write .", "format:check": "biome format .", "check": "biome check .", diff --git a/tsconfig.desktop-node.json b/tsconfig.desktop-node.json new file mode 100644 index 00000000..d77f129c --- /dev/null +++ b/tsconfig.desktop-node.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2023", + "lib": ["es2023"], + "module": "esnext", + "moduleResolution": "bundler", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "isolatedModules": true, + "resolveJsonModule": true, + "types": ["node"] + }, + "include": [ + "desktop/storage/**/*.ts", + "desktop/main/**/*.ts", + "desktop/build/**/*.ts" + ] +} diff --git a/tsconfig.json b/tsconfig.json index 99c6bb37..9ea694ab 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -31,5 +31,13 @@ ".next/dev/types/**/*.ts", "**/*.mts" ], - "exclude": ["node_modules", "lib/pwa/service-worker.ts", "desktop/smoke"] + "exclude": [ + "node_modules", + "lib/pwa/service-worker.ts", + "desktop/smoke", + "desktop/storage-harness", + "desktop/storage", + "desktop/main", + "desktop/build" + ] } From f45c5361f95fedf7e6e504d80afaacd7f567686a Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:25:03 +0330 Subject: [PATCH 02/19] feat(desktop): add the library storage engine The SQLite engine behind the desktop library (plan P2.6), behind a SqliteDriver seam so the same code runs on node:sqlite in the Electron storage worker and on bun:sqlite in tests: - format-1 schema: docs (lineage, link_key minted at insert, kind and the list columns), folders, tags, doc_tags, doc_view, import_sources, snapshots and updates with an AUTOINCREMENT seq - open order: cloud-folder refusal, library.lock, WAL and synchronous=FULL (fullfsync on macOS), then the format_version guard before any write; quick_check after an unclean exit - format migrations with a pre-migration VACUUM INTO (keeps 2), one transaction per step, read-only on failure - append and renderer-driven compaction in one transaction each, with stale and future upToSeq guards - the one-window registry: already_open, forbidden, document_open - documents, folders, tags, Trash and the sidebar snapshot, with rev - daily VACUUM INTO copies after 30 s idle, keeping 7 - shutdown: checkpoint, clear unclean_exit, release the lock --- desktop/dev/bun-sqlite-driver.ts | 45 ++ desktop/storage/drivers/node-sqlite.ts | 64 +++ desktop/storage/engine/cloud-folder.ts | 81 +++ desktop/storage/engine/content.ts | 197 +++++++ desktop/storage/engine/daily-copy.ts | 69 +++ desktop/storage/engine/driver.ts | 96 ++++ desktop/storage/engine/errors.ts | 66 +++ desktop/storage/engine/host.ts | 32 ++ desktop/storage/engine/library-index.ts | 387 ++++++++++++++ desktop/storage/engine/library.ts | 491 ++++++++++++++++++ desktop/storage/engine/lock.ts | 34 ++ desktop/storage/engine/migrations.ts | 100 ++++ desktop/storage/engine/schema.ts | 120 +++++ desktop/storage/engine/sql.ts | 54 ++ desktop/storage/node-host.ts | 88 ++++ tests/helpers/storage-library.ts | 68 +++ .../storage/cloud-folder-and-codes.test.ts | 91 ++++ tests/unit/desktop/storage/content.test.ts | 394 ++++++++++++++ tests/unit/desktop/storage/daily-copy.test.ts | 97 ++++ .../desktop/storage/library-index.test.ts | 203 ++++++++ .../unit/desktop/storage/library-open.test.ts | 343 ++++++++++++ 21 files changed, 3120 insertions(+) create mode 100644 desktop/dev/bun-sqlite-driver.ts create mode 100644 desktop/storage/drivers/node-sqlite.ts create mode 100644 desktop/storage/engine/cloud-folder.ts create mode 100644 desktop/storage/engine/content.ts create mode 100644 desktop/storage/engine/daily-copy.ts create mode 100644 desktop/storage/engine/driver.ts create mode 100644 desktop/storage/engine/errors.ts create mode 100644 desktop/storage/engine/host.ts create mode 100644 desktop/storage/engine/library-index.ts create mode 100644 desktop/storage/engine/library.ts create mode 100644 desktop/storage/engine/lock.ts create mode 100644 desktop/storage/engine/migrations.ts create mode 100644 desktop/storage/engine/schema.ts create mode 100644 desktop/storage/engine/sql.ts create mode 100644 desktop/storage/node-host.ts create mode 100644 tests/helpers/storage-library.ts create mode 100644 tests/unit/desktop/storage/cloud-folder-and-codes.test.ts create mode 100644 tests/unit/desktop/storage/content.test.ts create mode 100644 tests/unit/desktop/storage/daily-copy.test.ts create mode 100644 tests/unit/desktop/storage/library-index.test.ts create mode 100644 tests/unit/desktop/storage/library-open.test.ts diff --git a/desktop/dev/bun-sqlite-driver.ts b/desktop/dev/bun-sqlite-driver.ts new file mode 100644 index 00000000..200801a8 --- /dev/null +++ b/desktop/dev/bun-sqlite-driver.ts @@ -0,0 +1,45 @@ +/** + * `SqliteDriver` over `bun:sqlite`, for unit tests and the dev storage host + * (architecture §11). Normalizes Bun's differences from `node:sqlite`: a + * missing row is `null` in Bun and `undefined` here, and integers stay + * numbers (`safeIntegers` off). + */ +import { Database } from "bun:sqlite"; + +import type { + OpenSqlite, + SqliteDriver, + SqliteStatement, + SqlRow, + SqlValue, +} from "@/desktop/storage/engine/driver"; + +export const openBunSqlite: OpenSqlite = (path, { readOnly }) => { + const db = readOnly + ? new Database(path, { readonly: true }) + : new Database(path, { create: true }); + const cache = new Map(); + const driver: SqliteDriver = { + exec: (sql) => db.exec(sql), + prepare(sql) { + const cached = cache.get(sql); + if (cached) return cached; + const statement = db.prepare(sql); + const wrapped: SqliteStatement = { + run: (...params) => { + const result = statement.run(...params); + return { + changes: result.changes, + lastInsertRowid: Number(result.lastInsertRowid), + }; + }, + get: (...params) => statement.get(...params) ?? undefined, + all: (...params) => statement.all(...params), + }; + cache.set(sql, wrapped); + return wrapped; + }, + close: () => db.close(), + }; + return driver; +}; diff --git a/desktop/storage/drivers/node-sqlite.ts b/desktop/storage/drivers/node-sqlite.ts new file mode 100644 index 00000000..a3375a6c --- /dev/null +++ b/desktop/storage/drivers/node-sqlite.ts @@ -0,0 +1,64 @@ +/** + * `SqliteDriver` over `node:sqlite`, for the Electron storage worker + * (D-A1). `node:sqlite` returns `undefined` for a missing row and plain + * numbers by default, which is what the driver contract wants; BLOBs come + * back as fresh `Uint8Array`s. Loaded with `process.getBuiltinModule`, so a + * build of Electron without it fails here with a clear error (D-A12). + */ +import type { DatabaseSync } from "node:sqlite"; + +import type { + OpenSqlite, + SqliteDriver, + SqliteStatement, + SqlRow, + SqlValue, +} from "../engine/driver"; + +type NodeSqlite = typeof import("node:sqlite"); + +function loadNodeSqlite(): NodeSqlite { + const sqlite = process.getBuiltinModule("node:sqlite") as + | NodeSqlite + | undefined; + if (!sqlite) throw new Error("node:sqlite is not built into this runtime"); + return sqlite; +} + +function wrap(db: DatabaseSync): SqliteDriver { + const cache = new Map(); + return { + exec: (sql) => db.exec(sql), + prepare(sql) { + const cached = cache.get(sql); + if (cached) return cached; + const statement = db.prepare(sql); + const wrapped: SqliteStatement = { + run: (...params: SqlValue[]) => { + const result = statement.run(...params); + return { + changes: Number(result.changes), + lastInsertRowid: Number(result.lastInsertRowid), + }; + }, + get: (...params: SqlValue[]) => + statement.get(...params) as SqlRow | undefined, + all: (...params: SqlValue[]) => statement.all(...params) as SqlRow[], + }; + cache.set(sql, wrapped); + return wrapped; + }, + close: () => db.close(), + }; +} + +export const openNodeSqlite: OpenSqlite = (path, { readOnly }) => { + const { DatabaseSync } = loadNodeSqlite(); + return wrap( + new DatabaseSync(path, { + readOnly, + // The engine turns them on itself, the same on both drivers. + enableForeignKeyConstraints: false, + }), + ); +}; diff --git a/desktop/storage/engine/cloud-folder.ts b/desktop/storage/engine/cloud-folder.ts new file mode 100644 index 00000000..32401886 --- /dev/null +++ b/desktop/storage/engine/cloud-folder.ts @@ -0,0 +1,81 @@ +/** + * Refuses a library inside a cloud-synced folder (plan Appendix A, P2.6; + * architecture §5.4 `open.ts`). A sync client that uploads `library.sqlite` + * without its `-wal` file, or restores an older copy under a live + * connection, corrupts the library. `userData` never lives in one by + * default; this catches a moved home folder, OneDrive Known Folder Move and + * a `TINYDOCY_USER_DATA` override. + */ +import type { StorageHost } from "./host"; + +export type CloudProvider = + | "icloud" + | "onedrive" + | "dropbox" + | "google_drive" + | "box" + | "other"; + +// Folder names sync clients create. Compared case-insensitively against +// every path segment. +const SEGMENTS: ReadonlyArray<[RegExp, CloudProvider]> = [ + [/^mobile documents$/i, "icloud"], // ~/Library/Mobile Documents (iCloud Drive) + [/^icloud ?drive/i, "icloud"], // Windows iCloud for Windows + [/^onedrive( |$|-)/i, "onedrive"], // OneDrive, OneDrive - Contoso + [/^dropbox( |$|\()/i, "dropbox"], // Dropbox, Dropbox (Personal) + [/^google ?drive/i, "google_drive"], + [/^my drive$/i, "google_drive"], + [/^box( |$)/i, "box"], +]; + +// macOS File Provider puts every modern sync client under +// ~/Library/CloudStorage/-. +const CLOUD_STORAGE: ReadonlyArray<[RegExp, CloudProvider]> = [ + [/^onedrive/i, "onedrive"], + [/^dropbox/i, "dropbox"], + [/^googledrive/i, "google_drive"], + [/^box/i, "box"], + [/^icloud/i, "icloud"], +]; + +const ONEDRIVE_ENV = ["OneDrive", "OneDriveConsumer", "OneDriveCommercial"]; + +function segmentsOf(path: string): string[] { + return path.split(/[\\/]+/).filter(Boolean); +} + +function isUnder(path: string, root: string, caseInsensitive: boolean) { + const a = segmentsOf(path); + const b = segmentsOf(root); + if (b.length === 0 || b.length > a.length) return false; + return b.every((part, i) => + caseInsensitive + ? part.toLowerCase() === a[i]?.toLowerCase() + : part === a[i], + ); +} + +/** The cloud provider syncing `dir`, or null. Symlinks are resolved first. */ +export function cloudFolderOf( + dir: string, + host: Pick, +): CloudProvider | null { + const resolved = host.realpath(dir); + const caseInsensitive = host.platform !== "linux"; + for (const name of ONEDRIVE_ENV) { + const root = host.env[name]; + if (root && isUnder(resolved, root, caseInsensitive)) return "onedrive"; + } + const parts = segmentsOf(resolved); + for (const [i, part] of parts.entries()) { + if (part.toLowerCase() === "cloudstorage") { + const next = parts[i + 1] ?? ""; + for (const [pattern, provider] of CLOUD_STORAGE) + if (pattern.test(next)) return provider; + return "other"; + } + for (const [pattern, provider] of SEGMENTS) + if (pattern.test(part)) return provider; + } + return null; +} diff --git a/desktop/storage/engine/content.ts b/desktop/storage/engine/content.ts new file mode 100644 index 00000000..d9e117db --- /dev/null +++ b/desktop/storage/engine/content.ts @@ -0,0 +1,197 @@ +/** + * Document content: `load`, `append`, `compact` and `stateOf` (architecture + * §5.2, §5.4 `content.ts`). The renderer owns the Y.Doc and compacts it + * (D-A3); this module only stores bytes. It never parses an update, except + * in {@link stateOf}, which merges stored rows for export and restore. + * + * Rule one: a Yjs update is never dropped once acknowledged. `append` + * resolves only after COMMIT, and `compact` refuses an `upToSeq` that could + * cover a row the renderer has not seen. + */ +import * as Y from "yjs"; + +import type { SqliteDriver } from "./driver"; +import { StorageError } from "./errors"; +import { scalar, transaction } from "./sql"; + +/** The stored `updates.origin` values: `UPDATE_ORIGINS` in `lib/platform/doc-store.ts`. */ +export const STORED_ORIGINS = [ + "local", + "migration", + "remote", + "restore", +] as const; +export type StoredOrigin = (typeof STORED_ORIGINS)[number]; + +export type LoadedContent = { + snapshot: Uint8Array | null; + updates: Uint8Array[]; + maxSeq: number; + schema: number; +}; + +export type AppendInput = { + origin: StoredOrigin; + update: Uint8Array; + title?: string; + snippet?: string; + schemaTo?: number; +}; + +/** Title and snippet caps (architecture §7 `str(200)`; details spec §6.3). */ +export const MAX_TITLE_LENGTH = 200; +export const MAX_SNIPPET_LENGTH = 300; + +function docExists(db: SqliteDriver, docId: string): boolean { + return scalar(db, "SELECT 1 FROM docs WHERE id = ?", docId) !== undefined; +} + +function snapshotUpto(db: SqliteDriver, docId: string): number { + const value = scalar( + db, + "SELECT upto_seq FROM snapshots WHERE doc_id = ?", + docId, + ); + return typeof value === "number" ? value : 0; +} + +/** The highest seq ever assigned in this library (0 before the first append). */ +function lastAssignedSeq(db: SqliteDriver): number { + const value = scalar( + db, + "SELECT seq FROM sqlite_sequence WHERE name = 'updates'", + ); + return typeof value === "number" ? value : 0; +} + +export function load(db: SqliteDriver, docId: string): LoadedContent { + const doc = db.prepare("SELECT schema FROM docs WHERE id = ?").get(docId); + if (!doc) throw new StorageError("not_found", "no such document"); + const snap = db + .prepare("SELECT data, upto_seq FROM snapshots WHERE doc_id = ?") + .get(docId); + const upto = typeof snap?.upto_seq === "number" ? snap.upto_seq : 0; + const rows = db + .prepare( + "SELECT seq, data FROM updates WHERE doc_id = ? AND seq > ? ORDER BY seq", + ) + .all(docId, upto); + const last = rows.at(-1)?.seq; + return { + snapshot: snap ? (snap.data as Uint8Array) : null, + updates: rows.map((row) => row.data as Uint8Array), + maxSeq: typeof last === "number" ? last : upto, + schema: doc.schema as number, + }; +} + +function validateAppend(input: AppendInput) { + if (!STORED_ORIGINS.includes(input.origin)) + throw new StorageError("invalid_argument", "unknown origin"); + if (!(input.update instanceof Uint8Array) || input.update.byteLength === 0) + throw new StorageError("invalid_argument", "empty update"); + if ( + input.schemaTo !== undefined && + (!Number.isInteger(input.schemaTo) || input.schemaTo < 0) + ) + throw new StorageError("invalid_argument", "bad schemaTo"); +} + +/** + * One row, plus the document's list metadata, in one transaction. Returns + * the row's seq once committed. + */ +export function append( + db: SqliteDriver, + docId: string, + input: AppendInput, + now: number, +): number { + validateAppend(input); + return transaction(db, () => { + const updated = db + .prepare( + `UPDATE docs SET + modified_at = ?, + size_bytes = size_bytes + ?, + title = COALESCE(?, title), + snippet = COALESCE(?, snippet), + schema = COALESCE(?, schema) + WHERE id = ?`, + ) + .run( + now, + input.update.byteLength, + input.title === undefined + ? null + : input.title.slice(0, MAX_TITLE_LENGTH), + input.snippet === undefined + ? null + : input.snippet.slice(0, MAX_SNIPPET_LENGTH), + input.schemaTo ?? null, + docId, + ); + if (updated.changes === 0) + throw new StorageError("not_found", "no such document"); + return db + .prepare( + "INSERT INTO updates (doc_id, data, origin, created_at) VALUES (?, ?, ?, ?)", + ) + .run(docId, input.update, input.origin, now).lastInsertRowid; + }); +} + +/** + * Stores `state` as the snapshot and deletes rows `seq ≤ upToSeq`, in one + * transaction (D-A3). A stale compaction (`upToSeq` below the stored + * snapshot's) changes nothing. An `upToSeq` past the last seq the library + * has assigned is refused: it would hide every later append. + */ +export function compact( + db: SqliteDriver, + docId: string, + state: Uint8Array, + upToSeq: number, +): void { + if (!Number.isInteger(upToSeq) || upToSeq < 0) + throw new StorageError("invalid_argument", "bad upToSeq"); + if (!(state instanceof Uint8Array) || state.byteLength === 0) + throw new StorageError("invalid_argument", "empty state"); + transaction(db, () => { + if (!docExists(db, docId)) + throw new StorageError("not_found", "no such document"); + if (upToSeq > lastAssignedSeq(db)) + throw new StorageError("invalid_argument", "upToSeq is in the future"); + if (upToSeq < snapshotUpto(db, docId)) return; + db.prepare( + `INSERT INTO snapshots (doc_id, data, upto_seq) VALUES (?, ?, ?) + ON CONFLICT (doc_id) DO UPDATE SET data = excluded.data, upto_seq = excluded.upto_seq`, + ).run(docId, state, upToSeq); + db.prepare("DELETE FROM updates WHERE doc_id = ? AND seq <= ?").run( + docId, + upToSeq, + ); + db.prepare( + `UPDATE docs SET size_bytes = ? + COALESCE( + (SELECT SUM(LENGTH(data)) FROM updates WHERE doc_id = ?), 0) + WHERE id = ?`, + ).run(state.byteLength, docId, docId); + }); +} + +/** + * The document's whole state as one update (for `.tinydoc`, restore and + * the daily-copy reader). Merges without building a Y.Doc. + */ +export function stateOf(content: LoadedContent): Uint8Array { + const parts = content.snapshot + ? [content.snapshot, ...content.updates] + : content.updates; + if (parts.length === 0) return Y.encodeStateAsUpdate(new Y.Doc()); + return Y.mergeUpdates(parts); +} + +/** Rows waiting for compaction, for diagnostics and T8. */ +export function pendingRows(db: SqliteDriver): number { + return Number(scalar(db, "SELECT COUNT(*) FROM updates") ?? 0); +} diff --git a/desktop/storage/engine/daily-copy.ts b/desktop/storage/engine/daily-copy.ts new file mode 100644 index 00000000..884a4a70 --- /dev/null +++ b/desktop/storage/engine/daily-copy.ts @@ -0,0 +1,69 @@ +/** + * Daily copies (D-A6; architecture §5.4 `daily-copy.ts`): once a local day, + * on the first idle moment, `VACUUM INTO daily/library-.sqlite`, + * keeping the newest 7. They are the Local release's restore points + * ("Restore from a daily snapshot"), and its protection against a corrupt + * library. + * + * Each copy is written to `.partial` and renamed, so a kill mid-copy + * never leaves a half file under a real name. Retention is by count, so a + * clock jump can't delete good copies early. + */ +import type { SqliteDriver } from "./driver"; +import type { StorageHost } from "./host"; +import { readMeta, writeMeta } from "./sql"; + +export const DAILY_DIR = "daily"; +export const DAILY_KEEP = 7; +const DAILY = /^library-\d{4}-\d{2}-\d{2}\.sqlite$/; + +/** `YYYY-MM-DD` in local time: "today" is the person's day, not UTC's. */ +export function localDate(ms: number): string { + const date = new Date(ms); + const pad = (n: number) => String(n).padStart(2, "0"); + return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`; +} + +export function dailyCopyName(date: string): string { + return `library-${date}.sqlite`; +} + +export function listDailyCopies(host: StorageHost, dir: string): string[] { + return host + .list(host.join(dir, DAILY_DIR)) + .filter((file) => DAILY.test(file)) + .sort(); +} + +export function isDailyCopyDue(db: SqliteDriver, now: number): boolean { + return readMeta(db, "last_daily_copy") !== localDate(now); +} + +/** + * Writes today's copy if there isn't one yet. Returns the copy's file name, + * or null when it was already taken today. + */ +export function takeDailyCopy( + db: SqliteDriver, + host: StorageHost, + dir: string, + now: number, +): string | null { + if (!isDailyCopyDue(db, now)) return null; + const date = localDate(now); + const folder = host.join(dir, DAILY_DIR); + host.mkdirp(folder); + for (const file of host.list(folder)) + if (file.endsWith(".partial")) host.remove(host.join(folder, file)); + const name = dailyCopyName(date); + const target = host.join(folder, name); + const partial = `${target}.partial`; + db.prepare("VACUUM INTO ?").run(partial); + if (host.exists(target)) host.remove(target); + host.rename(partial, target); + writeMeta(db, "last_daily_copy", date); + const copies = listDailyCopies(host, dir); + for (const old of copies.slice(0, -DAILY_KEEP)) + host.remove(host.join(folder, old)); + return name; +} diff --git a/desktop/storage/engine/driver.ts b/desktop/storage/engine/driver.ts new file mode 100644 index 00000000..7f28044d --- /dev/null +++ b/desktop/storage/engine/driver.ts @@ -0,0 +1,96 @@ +/** + * The engine's only view of SQLite (desktop architecture §3, §12 "Mobile"). + * Two drivers implement it: `node:sqlite` in the Electron storage worker + * (`desktop/storage/drivers/node-sqlite.ts`) and `bun:sqlite` for unit tests + * and the dev storage host (`desktop/dev/bun-sqlite-driver.ts`). + * + * Drivers normalize what differs between the two runtimes, so the engine + * behaves the same on both: + * - a missing row is `undefined` (Bun returns `null`); + * - integers are numbers (no `bigint`); + * - a BLOB is a `Uint8Array` the caller owns; + * - parameters are positional `?` only; + * - a failure carries the SQLite extended result code (see + * {@link sqliteResultCode}). + */ +import { StorageError, type StorageErrorCode } from "./errors"; + +export type SqlValue = string | number | null | Uint8Array; +export type SqlRow = Record; + +export interface SqliteStatement { + run(...params: SqlValue[]): { changes: number; lastInsertRowid: number }; + get(...params: SqlValue[]): SqlRow | undefined; + all(...params: SqlValue[]): SqlRow[]; +} + +export interface SqliteDriver { + exec(sql: string): void; + prepare(sql: string): SqliteStatement; + close(): void; +} + +export type OpenSqlite = ( + path: string, + options: { readOnly: boolean }, +) => SqliteDriver; + +/** + * The SQLite extended result code of a driver error, or null when the error + * did not come from SQLite. Bun puts it in `errno`, Node in `errcode`. + */ +export function sqliteResultCode(error: unknown): number | null { + if (typeof error !== "object" || error === null) return null; + const { errno, errcode } = error as { errno?: unknown; errcode?: unknown }; + if (typeof errcode === "number") return errcode; + // Bun's SQLiteError: its constructor reports as Error, its name doesn't. + if (typeof errno === "number" && (error as Error).name === "SQLiteError") + return errno; + return null; +} + +// Primary result codes (https://sqlite.org/rescode.html). +const SQLITE_PERM = 3; +const SQLITE_BUSY = 5; +const SQLITE_LOCKED = 6; +const SQLITE_READONLY = 8; +const SQLITE_IOERR = 10; +const SQLITE_CORRUPT = 11; +const SQLITE_FULL = 13; +const SQLITE_CANTOPEN = 14; +const SQLITE_CONSTRAINT = 19; +const SQLITE_NOTADB = 26; + +/** Maps a driver failure to the storage error a caller sees. */ +export function storageErrorFrom(error: unknown): StorageError { + if (error instanceof StorageError) return error; + const code = sqliteResultCode(error); + const message = error instanceof Error ? error.message : String(error); + let mapped: StorageErrorCode = "io_error"; + switch (code === null ? null : code & 0xff) { + case SQLITE_FULL: + mapped = "disk_full"; + break; + case SQLITE_CORRUPT: + case SQLITE_NOTADB: + mapped = "corrupt"; + break; + case SQLITE_BUSY: + case SQLITE_LOCKED: + mapped = "locked_elsewhere"; + break; + case SQLITE_READONLY: + case SQLITE_PERM: + mapped = "library_read_only"; + break; + case SQLITE_CONSTRAINT: + // A duplicate tag name or link key: the caller's input, not the disk. + mapped = "invalid_argument"; + break; + case SQLITE_CANTOPEN: + case SQLITE_IOERR: + mapped = "io_error"; + break; + } + return new StorageError(mapped, message); +} diff --git a/desktop/storage/engine/errors.ts b/desktop/storage/engine/errors.ts new file mode 100644 index 00000000..3304884a --- /dev/null +++ b/desktop/storage/engine/errors.ts @@ -0,0 +1,66 @@ +/** + * Storage error codes (desktop architecture §9). The engine rejects with a + * {@link StorageError}; the worker sends `{ code, retryable }` across, never + * a thrown object. This list must stay a superset of `DOC_STORE_ERROR_CODES` + * in `lib/platform/doc-store.ts` minus the web library's `quota_exceeded` + * (a unit test compares them): the engine cannot import `lib/`. + */ +export const STORAGE_ERROR_CODES = [ + "invalid_argument", + "forbidden", + "not_found", + "already_open", + "document_open", + "library_read_only", + "library_too_new", + "migration_failed", + "disk_full", + "io_error", + "corrupt", + "locked_elsewhere", + "cloud_folder", + "worker_restarting", + "library_unavailable", + "import_too_large", + "unknown_schema", +] as const; + +export type StorageErrorCode = (typeof STORAGE_ERROR_CODES)[number]; + +const RETRYABLE: ReadonlySet = new Set([ + "disk_full", + "io_error", + "worker_restarting", + "library_unavailable", +]); + +export class StorageError extends Error { + readonly code: StorageErrorCode; + readonly retryable: boolean; + + constructor(code: StorageErrorCode, message: string = code) { + super(message); + this.name = "StorageError"; + this.code = code; + this.retryable = RETRYABLE.has(code); + } +} + +/** What crosses the worker boundary in place of a thrown error. */ +export type StorageErrorEnvelope = { + code: StorageErrorCode; + retryable: boolean; + message: string; +}; + +export function toEnvelope(error: unknown): StorageErrorEnvelope { + const storage = + error instanceof StorageError + ? error + : new StorageError("io_error", String(error)); + return { + code: storage.code, + retryable: storage.retryable, + message: storage.message, + }; +} diff --git a/desktop/storage/engine/host.ts b/desktop/storage/engine/host.ts new file mode 100644 index 00000000..4269ad49 --- /dev/null +++ b/desktop/storage/engine/host.ts @@ -0,0 +1,32 @@ +/** + * What the engine needs from the operating system, injected so the engine + * imports no `node:*` or `bun:*` module (desktop architecture §3.3). The + * worker passes `createNodeHost()` (`desktop/storage/node-host.ts`); tests + * pass the same host with a fake clock, or a fake pid check. + */ +export interface StorageHost { + /** `process.platform`. */ + readonly platform: string; + /** This process's pid, written to `library.lock`. */ + readonly pid: number; + /** The home folder, used only to recognise cloud-synced folders. */ + readonly homeDir: string; + /** Environment variables (OneDrive sets `OneDrive*` on Windows). */ + readonly env: Readonly>; + /** Milliseconds since the epoch. */ + now(): number; + join(...parts: string[]): string; + /** Resolves symlinks; returns the path unchanged when it doesn't exist. */ + realpath(path: string): string; + mkdirp(path: string): void; + exists(path: string): boolean; + /** File names in `dir`, or `[]` when it doesn't exist. */ + list(dir: string): string[]; + remove(path: string): void; + rename(from: string, to: string): void; + readText(path: string): string | null; + /** Creates `path` with `text`; false when it already exists. */ + createExclusive(path: string, text: string): boolean; + /** Whether a process with this pid is running. */ + isPidAlive(pid: number): boolean; +} diff --git a/desktop/storage/engine/library-index.ts b/desktop/storage/engine/library-index.ts new file mode 100644 index 00000000..7ce23e65 --- /dev/null +++ b/desktop/storage/engine/library-index.ts @@ -0,0 +1,387 @@ +/** + * The library's metadata: documents, folders and tags (architecture §5.4 + * `index.ts`; named `library-index.ts` so it isn't taken for a barrel). + * `IpcDocumentIndex` (P2.3's `DocumentIndex` port) sits on top of these + * operations; the row shapes here are the storage side and stay neutral. + * + * Every metadata write goes through this module (seam S10) and bumps + * `meta.rev` in the same transaction, so the change feed can say "changed + * since rev N". Inserting a document mints its `lineage` and `link_key` + * (seam S16): never lazily, so two synced devices can't mint different keys. + */ +import type { SqliteDriver, SqlRow } from "./driver"; +import { StorageError } from "./errors"; +import { randomHex32, readMeta, scalar, transaction, writeMeta } from "./sql"; + +export type DocKind = "doc" | "template"; + +export type CreateDocInput = { + /** A UUID; minted when omitted. */ + id?: string; + title?: string; + folderId?: string | null; + sortKey?: string; + kind?: DocKind; + /** Kept only by a backup restore (S8c); everything else gets a new one. */ + lineage?: string; + /** Kept only by a backup zip or `library.json` restore (S8c). */ + linkKey?: string; +}; + +export type CreatedDoc = { + id: string; + lineage: string; + linkKey: string; + rev: number; +}; + +export type SidebarFolder = { + id: string; + parentId: string | null; + name: string; + sortKey: string; + sort: string | null; + view: string | null; + group: string | null; +}; + +export type SidebarDoc = { + id: string; + title: string; + folderId: string | null; + sortKey: string; + pinKey: string | null; + pinned: boolean; + createdAt: number; + modifiedAt: number; + openedAt: number | null; + sizeBytes: number; + snippet: string; + kind: DocKind; + tagIds: string[]; +}; + +export type SidebarTag = { id: string; name: string; color: string | null }; + +/** Everything the files sidebar and lists paint from (P2.7a/b). */ +export type SidebarSnapshot = { + rev: number; + folders: SidebarFolder[]; + docs: SidebarDoc[]; + tags: SidebarTag[]; +}; + +/** Trash is emptied after this long (details spec §9). */ +export const TRASH_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +const HEX32 = /^[0-9a-f]{32}$/; +export const MAX_NAME_LENGTH = 200; + +function requireId(id: string, what: string) { + if (!UUID.test(id)) throw new StorageError("invalid_argument", `bad ${what}`); +} + +function cleanName(name: string): string { + return name.trim().slice(0, MAX_NAME_LENGTH); +} + +export function readRev(db: SqliteDriver): number { + return Number.parseInt(readMeta(db, "rev") ?? "0", 10); +} + +/** Bumps `meta.rev`; call inside the write's transaction. */ +function bumpRev(db: SqliteDriver): number { + const rev = readRev(db) + 1; + writeMeta(db, "rev", String(rev)); + return rev; +} + +/** Wraps a metadata write: one transaction, `rev` bumped, new rev returned. */ +function indexWrite(db: SqliteDriver, body: () => void): number { + return transaction(db, () => { + body(); + return bumpRev(db); + }); +} + +function requireChanged(changes: number, what: string) { + if (changes === 0) throw new StorageError("not_found", `no such ${what}`); +} + +export function createDoc( + db: SqliteDriver, + input: CreateDocInput, + now: number, +): CreatedDoc { + const id = input.id ?? crypto.randomUUID(); + requireId(id, "document id"); + const lineage = input.lineage ?? crypto.randomUUID(); + requireId(lineage, "lineage"); + const linkKey = input.linkKey ?? randomHex32(); + if (!HEX32.test(linkKey)) + throw new StorageError("invalid_argument", "bad link key"); + if (input.folderId) requireId(input.folderId, "folder id"); + const kind = input.kind ?? "doc"; + if (kind !== "doc" && kind !== "template") + throw new StorageError("invalid_argument", "bad kind"); + const rev = indexWrite(db, () => { + db.prepare( + `INSERT INTO docs (id, title, folder_id, sort_key, created_at, modified_at, + lineage, link_key, kind) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).run( + id, + cleanName(input.title ?? ""), + input.folderId ?? null, + input.sortKey ?? "", + now, + now, + lineage, + linkKey, + kind, + ); + }); + return { id, lineage, linkKey, rev }; +} + +export function createFolder( + db: SqliteDriver, + input: { + id?: string; + name: string; + parentId?: string | null; + sortKey?: string; + }, + now: number, +): { id: string; rev: number } { + const id = input.id ?? crypto.randomUUID(); + requireId(id, "folder id"); + if (input.parentId) requireId(input.parentId, "parent id"); + const rev = indexWrite(db, () => { + db.prepare( + `INSERT INTO folders (id, parent_id, name, sort_key, created_at, modified_at) + VALUES (?, ?, ?, ?, ?, ?)`, + ).run( + id, + input.parentId ?? null, + cleanName(input.name), + input.sortKey ?? "", + now, + now, + ); + }); + return { id, rev }; +} + +export function renameDoc(db: SqliteDriver, id: string, title: string): number { + return indexWrite(db, () => { + requireChanged( + db + .prepare("UPDATE docs SET title = ? WHERE id = ?") + .run(cleanName(title), id).changes, + "document", + ); + }); +} + +export function moveDoc( + db: SqliteDriver, + id: string, + folderId: string | null, + sortKey: string, +): number { + if (folderId) requireId(folderId, "folder id"); + return indexWrite(db, () => { + requireChanged( + db + .prepare("UPDATE docs SET folder_id = ?, sort_key = ? WHERE id = ?") + .run(folderId, sortKey, id).changes, + "document", + ); + }); +} + +export function markOpened(db: SqliteDriver, id: string, now: number): number { + return indexWrite(db, () => { + requireChanged( + db.prepare("UPDATE docs SET opened_at = ? WHERE id = ?").run(now, id) + .changes, + "document", + ); + }); +} + +export function trashDoc( + db: SqliteDriver, + id: string, + now: number, + group: string | null = null, +): number { + return indexWrite(db, () => { + requireChanged( + db + .prepare( + `UPDATE docs SET trashed_at = ?, trash_group = ?, trashed_from = folder_id + WHERE id = ? AND trashed_at IS NULL`, + ) + .run(now, group, id).changes, + "document", + ); + }); +} + +export function restoreDoc(db: SqliteDriver, id: string): number { + return indexWrite(db, () => { + requireChanged( + db + .prepare( + `UPDATE docs SET trashed_at = NULL, trash_group = NULL, + folder_id = CASE WHEN trashed_from IS NULL + OR EXISTS (SELECT 1 FROM folders WHERE id = docs.trashed_from AND trashed_at IS NULL) + THEN trashed_from ELSE NULL END, + trashed_from = NULL + WHERE id = ? AND trashed_at IS NOT NULL`, + ) + .run(id).changes, + "trashed document", + ); + }); +} + +/** + * Deletes a document and everything that cascades from it: rows, snapshot, + * tags, view and import source. The caller (`Library`) refuses a document + * that is open (`document_open`). + */ +export function purgeDoc(db: SqliteDriver, id: string): number { + return indexWrite(db, () => { + requireChanged( + db.prepare("DELETE FROM docs WHERE id = ?").run(id).changes, + "document", + ); + }); +} + +/** Ids of documents in Trash for longer than {@link TRASH_RETENTION_MS}. */ +export function expiredTrash(db: SqliteDriver, now: number): string[] { + return db + .prepare( + "SELECT id FROM docs WHERE trashed_at IS NOT NULL AND trashed_at <= ?", + ) + .all(now - TRASH_RETENTION_MS) + .map((row) => row.id as string); +} + +export function createTag( + db: SqliteDriver, + name: string, + color: string | null = null, +): { id: string; rev: number } { + const clean = cleanName(name); + if (!clean) throw new StorageError("invalid_argument", "empty tag name"); + const id = crypto.randomUUID(); + const rev = indexWrite(db, () => { + db.prepare("INSERT INTO tags (id, name, color) VALUES (?, ?, ?)").run( + id, + clean, + color, + ); + }); + return { id, rev }; +} + +export function setDocTags( + db: SqliteDriver, + docId: string, + tagIds: readonly string[], +): number { + for (const tagId of tagIds) requireId(tagId, "tag id"); + return indexWrite(db, () => { + if (scalar(db, "SELECT 1 FROM docs WHERE id = ?", docId) === undefined) + throw new StorageError("not_found", "no such document"); + db.prepare("DELETE FROM doc_tags WHERE doc_id = ?").run(docId); + const insert = db.prepare( + "INSERT INTO doc_tags (doc_id, tag_id) VALUES (?, ?)", + ); + for (const tagId of new Set(tagIds)) insert.run(docId, tagId); + }); +} + +const nullableString = (value: SqlRow[string]) => + typeof value === "string" ? value : null; +const nullableNumber = (value: SqlRow[string]) => + typeof value === "number" ? value : null; + +/** + * The files sidebar's whole picture in one read: every live (untrashed) + * folder and document, and the tags. Reads no BLOB column. P2.6's exit + * budget (architecture §10): populated within 600 ms on an M1 and 1.0 s on + * Windows with 1,000 documents. + */ +export function sidebarSnapshot(db: SqliteDriver): SidebarSnapshot { + const tagsByDoc = new Map(); + for (const row of db + .prepare( + `SELECT doc_tags.doc_id, doc_tags.tag_id FROM doc_tags + JOIN docs ON docs.id = doc_tags.doc_id WHERE docs.trashed_at IS NULL`, + ) + .all()) { + const docId = row.doc_id as string; + const list = tagsByDoc.get(docId); + if (list) list.push(row.tag_id as string); + else tagsByDoc.set(docId, [row.tag_id as string]); + } + const docs = db + .prepare( + `SELECT id, title, folder_id, sort_key, pin_key, pinned, created_at, + modified_at, opened_at, size_bytes, snippet, kind + FROM docs WHERE trashed_at IS NULL`, + ) + .all() + .map( + (row): SidebarDoc => ({ + id: row.id as string, + title: row.title as string, + folderId: nullableString(row.folder_id), + sortKey: row.sort_key as string, + pinKey: nullableString(row.pin_key), + pinned: row.pinned === 1, + createdAt: row.created_at as number, + modifiedAt: row.modified_at as number, + openedAt: nullableNumber(row.opened_at), + sizeBytes: row.size_bytes as number, + snippet: row.snippet as string, + kind: row.kind as DocKind, + tagIds: tagsByDoc.get(row.id as string) ?? [], + }), + ); + const folders = db + .prepare( + `SELECT id, parent_id, name, sort_key, sort, view, "group" + FROM folders WHERE trashed_at IS NULL`, + ) + .all() + .map( + (row): SidebarFolder => ({ + id: row.id as string, + parentId: nullableString(row.parent_id), + name: row.name as string, + sortKey: row.sort_key as string, + sort: nullableString(row.sort), + view: nullableString(row.view), + group: nullableString(row.group), + }), + ); + const tags = db + .prepare("SELECT id, name, color FROM tags") + .all() + .map( + (row): SidebarTag => ({ + id: row.id as string, + name: row.name as string, + color: nullableString(row.color), + }), + ); + return { rev: readRev(db), folders, docs, tags }; +} diff --git a/desktop/storage/engine/library.ts b/desktop/storage/engine/library.ts new file mode 100644 index 00000000..d291e1fb --- /dev/null +++ b/desktop/storage/engine/library.ts @@ -0,0 +1,491 @@ +/** + * One open library: the storage worker's whole state (architecture §5.4). + * The worker owns exactly one `Library`; tests and the dev storage host + * build one directly over `bun:sqlite`. + * + * **Open order** (architecture §5.4 `open.ts`, D-A6). Nothing that writes + * runs before the `format_version` check, so a library made by a newer app + * is never touched (soak T6): + * 1. refuse a cloud-synced folder (`cloud_folder`); + * 2. take `library.lock` (`locked_elsewhere`); + * 3. pragmas: WAL, `synchronous=FULL`, foreign keys, and on macOS + * `fullfsync` and `checkpoint_fullfsync`; + * 4. create the schema if the file is new (lineage and `library_id` from day + * one); + * 5. `format_version` newer than the app → reopen read-only, + * `library_too_new`; + * 6. `unclean_exit` still set → `quick_check`; a failure is read-only, + * `corrupt`; + * 7. older → pre-migration copy, then the format steps (`migration_failed` + * is read-only); + * 8. set `unclean_exit`; {@link Library.shutdown} clears it. + * + * **Registry** (architecture §5.2, §6.2): a document is open in at most one + * window (D-A3). `load` claims it for an owner (a window id) and rejects + * another owner with `already_open`; appends and compactions from another + * owner are `forbidden`; main's own content writes (purge, and P2.8's + * import-into and restore) refuse an open document with `document_open`. + */ +import { cloudFolderOf } from "./cloud-folder"; +import { + type AppendInput, + append, + compact, + type LoadedContent, + load, + pendingRows, + stateOf, +} from "./content"; +import { + DAILY_DIR, + dailyCopyName, + isDailyCopyDue, + listDailyCopies, + localDate, + takeDailyCopy, +} from "./daily-copy"; +import type { OpenSqlite, SqliteDriver } from "./driver"; +import { storageErrorFrom } from "./driver"; +import { StorageError } from "./errors"; +import type { StorageHost } from "./host"; +import * as index from "./library-index"; +import { acquireLock, releaseLock } from "./lock"; +import { + FORMAT_STEPS, + type FormatStep, + readFormatVersion, + runFormatSteps, +} from "./migrations"; +import { FORMAT_VERSION, SCHEMA_V1, YJS_ENCODING } from "./schema"; +import { readMeta, scalar, transaction, writeMeta } from "./sql"; + +export const LIBRARY_FILE = "library.sqlite"; +/** Idle jobs (daily copy, Trash purge) wait for this long without a write. */ +export const IDLE_MS = 30_000; + +export type ReadOnlyReason = "library_too_new" | "migration_failed" | "corrupt"; + +export type LibraryStatus = { + readOnly: ReadOnlyReason | null; + formatVersion: number; + /** The previous session did not shut down cleanly. */ + uncleanExit: boolean; + /** `quick_check`'s verdict after an unclean exit; null when it didn't run. */ + quickCheck: "ok" | string | null; + libraryId: string; +}; + +/** Protocol lines for the durability soak (written to fd 1 by the worker). */ +export type Trace = (line: string) => void; + +export type OpenLibraryOptions = { + dir: string; + host: StorageHost; + openSqlite: OpenSqlite; + /** Default on; the T3 control turns it off. macOS only. */ + fullfsync?: boolean; + /** Default `FULL`; the soak's atomicity control uses `OFF`. */ + synchronous?: "FULL" | "OFF"; + /** For tests: the format this "app" writes, and the steps that reach it. */ + formatVersion?: number; + formatSteps?: readonly FormatStep[]; + trace?: Trace; +}; + +function applyPragmas( + db: SqliteDriver, + host: StorageHost, + options: OpenLibraryOptions, +) { + db.exec("PRAGMA journal_mode = WAL"); + db.exec(`PRAGMA synchronous = ${options.synchronous ?? "FULL"}`); + db.exec("PRAGMA foreign_keys = ON"); + if (host.platform === "darwin" && options.fullfsync !== false) { + db.exec("PRAGMA fullfsync = 1"); + db.exec("PRAGMA checkpoint_fullfsync = 1"); + } +} + +function hasTable(db: SqliteDriver, name: string): boolean { + return ( + scalar( + db, + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", + name, + ) !== undefined + ); +} + +function createLibrary(db: SqliteDriver, now: number) { + const tables = Number( + scalar(db, "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table'"), + ); + if (tables > 0) + throw new StorageError("corrupt", "not a TinyDocy library (no meta)"); + transaction(db, () => { + db.exec(SCHEMA_V1); + writeMeta(db, "format_version", String(FORMAT_VERSION)); + writeMeta(db, "yjs_encoding", YJS_ENCODING); + writeMeta(db, "library_id", crypto.randomUUID()); + writeMeta(db, "created_at", String(now)); + writeMeta(db, "rev", "0"); + writeMeta(db, "unclean_exit", "0"); + }); +} + +function quickCheck(db: SqliteDriver): string { + const rows = db.prepare("PRAGMA quick_check").all(); + const messages = rows.map((row) => String(Object.values(row)[0])); + return messages.length === 1 && messages[0] === "ok" + ? "ok" + : messages.slice(0, 5).join("; "); +} + +export class Library { + readonly dir: string; + readonly path: string; + readonly status: LibraryStatus; + private db: SqliteDriver | null; + private readonly host: StorageHost; + private readonly openSqlite: OpenSqlite; + private readonly lockPath: string; + private readonly holders = new Map(); + private readonly trace: Trace | undefined; + private lastWriteAt: number; + + private constructor(fields: { + dir: string; + path: string; + status: LibraryStatus; + db: SqliteDriver; + host: StorageHost; + openSqlite: OpenSqlite; + lockPath: string; + trace: Trace | undefined; + }) { + this.dir = fields.dir; + this.path = fields.path; + this.status = fields.status; + this.db = fields.db; + this.host = fields.host; + this.openSqlite = fields.openSqlite; + this.lockPath = fields.lockPath; + this.trace = fields.trace; + this.lastWriteAt = fields.host.now(); + } + + static open(options: OpenLibraryOptions): Library { + const { dir, host, openSqlite } = options; + const target = options.formatVersion ?? FORMAT_VERSION; + host.mkdirp(dir); + const cloud = cloudFolderOf(dir, host); + if (cloud) throw new StorageError("cloud_folder", cloud); + const lockPath = acquireLock(host, dir); + const path = host.join(dir, LIBRARY_FILE); + let db: SqliteDriver | null = null; + try { + db = openSqlite(path, { readOnly: false }); + applyPragmas(db, host, options); + if (!hasTable(db, "meta")) createLibrary(db, host.now()); + const version = readFormatVersion(db); + const libraryId = readMeta(db, "library_id") ?? ""; + const status: LibraryStatus = { + readOnly: null, + formatVersion: version, + uncleanExit: false, + quickCheck: null, + libraryId, + }; + if (version > target) { + db.close(); + db = openSqlite(path, { readOnly: true }); + status.readOnly = "library_too_new"; + } else { + status.uncleanExit = readMeta(db, "unclean_exit") !== "0"; + if (status.uncleanExit) { + status.quickCheck = quickCheck(db); + if (status.quickCheck !== "ok") status.readOnly = "corrupt"; + } + if (!status.readOnly && version < target) { + try { + status.formatVersion = runFormatSteps( + db, + host, + dir, + target, + options.formatSteps ?? FORMAT_STEPS, + ); + } catch (error) { + if ( + !(error instanceof StorageError) || + error.code !== "migration_failed" + ) + throw error; + status.readOnly = "migration_failed"; + status.formatVersion = readFormatVersion(db); + } + } + if (!status.readOnly) writeMeta(db, "unclean_exit", "1"); + } + return new Library({ + dir, + path, + status, + db, + host, + openSqlite, + lockPath, + trace: options.trace, + }); + } catch (error) { + try { + db?.close(); + } catch { + // Already failing; the original error matters. + } + releaseLock(host, lockPath); + throw storageErrorFrom(error); + } + } + + private get conn(): SqliteDriver { + if (!this.db) + throw new StorageError("library_unavailable", "library is closed"); + return this.db; + } + + private assertWritable() { + if (this.status.readOnly) + throw new StorageError( + this.status.readOnly === "library_too_new" + ? "library_too_new" + : "library_read_only", + this.status.readOnly, + ); + } + + /** Content writes from the renderer: refused while another owner holds the doc. */ + private assertWriter(docId: string, owner: string) { + const holder = this.holders.get(docId); + if (holder !== undefined && holder !== owner) + throw new StorageError("forbidden", "document is open in another window"); + } + + /** Main's own content writes: refused while any window holds the doc. */ + private assertClosed(docId: string) { + if (this.holders.has(docId)) + throw new StorageError("document_open", "document is open"); + } + + private wrote() { + this.lastWriteAt = this.host.now(); + } + + // ── Content (DocStore) ──────────────────────────────────────────────── + + load(docId: string, owner: string): LoadedContent { + const holder = this.holders.get(docId); + if (holder !== undefined && holder !== owner) + throw new StorageError("already_open", "document is open elsewhere"); + const content = load(this.conn, docId); + this.holders.set(docId, owner); + if (!this.status.readOnly) { + try { + index.markOpened(this.conn, docId, this.host.now()); + } catch { + // "Opened" sorting is a nicety; opening must not fail on it. + } + } + return content; + } + + append(docId: string, owner: string, input: AppendInput): number { + this.assertWritable(); + this.assertWriter(docId, owner); + this.trace?.(`BATCH_BEGIN ${docId}`); + const seq = append(this.conn, docId, input, this.host.now()); + this.trace?.(`ACK ${seq}`); + this.wrote(); + return seq; + } + + compact( + docId: string, + owner: string, + state: Uint8Array, + upToSeq: number, + ): { foreign: null } { + this.assertWritable(); + this.assertWriter(docId, owner); + this.trace?.(`COMPACT_BEGIN ${upToSeq} ${state.byteLength}`); + compact(this.conn, docId, state, upToSeq); + this.trace?.(`COMPACT_ACK ${upToSeq}`); + this.wrote(); + return { foreign: null }; + } + + close(docId: string, owner: string): void { + if (this.holders.get(docId) === owner) this.holders.delete(docId); + } + + /** A window closed or its renderer died: every hold it had is released. */ + releaseOwner(owner: string): void { + for (const [docId, holder] of this.holders) + if (holder === owner) this.holders.delete(docId); + } + + isOpen(docId: string): boolean { + return this.holders.has(docId); + } + + /** The whole state of a closed document, for export (P2.8). */ + stateOf(docId: string): Uint8Array { + return stateOf(load(this.conn, docId)); + } + + // ── Index ───────────────────────────────────────────────────────────── + + sidebar(): index.SidebarSnapshot { + return index.sidebarSnapshot(this.conn); + } + + createDoc(input: index.CreateDocInput = {}): index.CreatedDoc { + this.assertWritable(); + const created = index.createDoc(this.conn, input, this.host.now()); + this.wrote(); + return created; + } + + createFolder(input: { name: string; parentId?: string | null }) { + this.assertWritable(); + const created = index.createFolder(this.conn, input, this.host.now()); + this.wrote(); + return created; + } + + renameDoc(id: string, title: string): number { + this.assertWritable(); + return index.renameDoc(this.conn, id, title); + } + + moveDoc(id: string, folderId: string | null, sortKey: string): number { + this.assertWritable(); + return index.moveDoc(this.conn, id, folderId, sortKey); + } + + trashDoc(id: string, group: string | null = null): number { + this.assertWritable(); + return index.trashDoc(this.conn, id, this.host.now(), group); + } + + restoreDoc(id: string): number { + this.assertWritable(); + return index.restoreDoc(this.conn, id); + } + + purgeDoc(id: string): number { + this.assertWritable(); + this.assertClosed(id); + const rev = index.purgeDoc(this.conn, id); + this.wrote(); + return rev; + } + + createTag(name: string, color: string | null = null) { + this.assertWritable(); + return index.createTag(this.conn, name, color); + } + + setDocTags(docId: string, tagIds: readonly string[]): number { + this.assertWritable(); + return index.setDocTags(this.conn, docId, tagIds); + } + + // ── Restore points ──────────────────────────────────────────────────── + + listDailyCopies(): string[] { + return listDailyCopies(this.host, this.dir).map((file) => + file.slice("library-".length, -".sqlite".length), + ); + } + + /** One document's state in the daily copy of `date` (`YYYY-MM-DD`). */ + readDailyCopy(date: string, docId: string): Uint8Array { + if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) + throw new StorageError("invalid_argument", "bad date"); + const file = this.host.join(this.dir, DAILY_DIR, dailyCopyName(date)); + if (!this.host.exists(file)) + throw new StorageError("not_found", "no copy for that day"); + const copy = this.openSqlite(file, { readOnly: true }); + try { + return stateOf(load(copy, docId)); + } finally { + copy.close(); + } + } + + // ── Jobs and lifecycle ──────────────────────────────────────────────── + + /** + * Runs the idle jobs when nothing was written for {@link IDLE_MS}: today's + * daily copy, then Trash older than 30 days (skipping open documents). + * Returns what ran, for the worker's diagnostics. + */ + runIdleJobs(): { dailyCopy: string | null; purged: number } { + const now = this.host.now(); + const result = { dailyCopy: null as string | null, purged: 0 }; + if (this.status.readOnly || now - this.lastWriteAt < IDLE_MS) return result; + if (isDailyCopyDue(this.conn, now)) + result.dailyCopy = takeDailyCopy(this.conn, this.host, this.dir, now); + for (const id of index.expiredTrash(this.conn, now)) { + if (this.holders.has(id)) continue; + index.purgeDoc(this.conn, id); + result.purged += 1; + } + return result; + } + + /** Forces today's daily copy now (diagnostics, tests). */ + takeDailyCopyNow(): string | null { + this.assertWritable(); + return takeDailyCopy(this.conn, this.host, this.dir, this.host.now()); + } + + stats() { + const db = this.conn; + return { + formatVersion: this.status.formatVersion, + docs: Number(scalar(db, "SELECT COUNT(*) FROM docs")), + pendingRows: pendingRows(db), + bytes: Number( + scalar( + db, + "SELECT page_count * page_size FROM pragma_page_count(), pragma_page_size()", + ), + ), + lastDailyCopy: readMeta(db, "last_daily_copy"), + today: localDate(this.host.now()), + }; + } + + /** + * The quit flush's last step (architecture §8.2): every request queued + * before this one has already committed (the worker handles one message + * at a time), so checkpoint the WAL into the main file, clear + * `unclean_exit`, close and release the lock. Safe to call twice. + */ + shutdown(): void { + const db = this.db; + if (!db) return; + try { + if (!this.status.readOnly) { + writeMeta(db, "unclean_exit", "0"); + db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); + } + } finally { + this.db = null; + db.close(); + releaseLock(this.host, this.lockPath); + } + } +} diff --git a/desktop/storage/engine/lock.ts b/desktop/storage/engine/lock.ts new file mode 100644 index 00000000..b67d704e --- /dev/null +++ b/desktop/storage/engine/lock.ts @@ -0,0 +1,34 @@ +/** + * `library.lock` (architecture §5.4 `open.ts`): one process opens a library + * at a time. The file holds the owner's pid; a lock whose pid is no longer + * running is stale (a crash or a SIGKILL) and is taken over. Electron's + * single-instance lock covers the usual case; this one also covers a second + * build (internal and release side by side) pointed at the same folder. + */ +import { StorageError } from "./errors"; +import type { StorageHost } from "./host"; + +export const LOCK_FILE = "library.lock"; + +export function acquireLock(host: StorageHost, dir: string): string { + const path = host.join(dir, LOCK_FILE); + for (let attempt = 0; attempt < 3; attempt += 1) { + if (host.createExclusive(path, String(host.pid))) return path; + const holder = Number.parseInt(host.readText(path) ?? "", 10); + if ( + Number.isInteger(holder) && + holder > 0 && + holder !== host.pid && + host.isPidAlive(holder) + ) { + throw new StorageError("locked_elsewhere", `library held by ${holder}`); + } + // Stale (dead pid, unreadable, or our own from a restarted worker). + host.remove(path); + } + throw new StorageError("locked_elsewhere", "could not take library.lock"); +} + +export function releaseLock(host: StorageHost, path: string) { + if (host.readText(path) === String(host.pid)) host.remove(path); +} diff --git a/desktop/storage/engine/migrations.ts b/desktop/storage/engine/migrations.ts new file mode 100644 index 00000000..efbac6a6 --- /dev/null +++ b/desktop/storage/engine/migrations.ts @@ -0,0 +1,100 @@ +/** + * The `format_version` runner (architecture §5.4 `migrations.ts`, D-A6). + * + * - **Newer than the app:** the library opens read-only with + * `library_too_new`, and nothing is written (checked before any other + * write; see `library.ts`). + * - **Older:** `VACUUM INTO library.pre-v-.sqlite` first (the last 2 + * are kept), then each step in its own transaction, which also bumps + * `format_version`. A failed step leaves the library read-only with + * `migration_failed`; a kill between steps resumes at the next one. + * + * Format 1 is the first, so {@link FORMAT_STEPS} is empty. Tests pass their + * own steps to exercise the runner (soak T5). + */ +import type { SqliteDriver } from "./driver"; +import { StorageError } from "./errors"; +import type { StorageHost } from "./host"; +import { readMeta, transaction, writeMeta } from "./sql"; + +export type FormatStep = { + /** The `format_version` this step produces. */ + to: number; + up(db: SqliteDriver): void; +}; + +/** Steps from format 1 to the current `FORMAT_VERSION`, in order. */ +export const FORMAT_STEPS: readonly FormatStep[] = []; + +export const PRE_MIGRATION_KEEP = 2; +const PRE_MIGRATION = /^library\.pre-v\d+-[0-9TZ]+\.sqlite$/; + +/** `20261117T031700Z`: sortable, and valid in a file name on every OS. */ +export function utcStamp(ms: number): string { + return new Date(ms).toISOString().replace(/[-:]/g, "").replace(/\.\d+/, ""); +} + +export function readFormatVersion(db: SqliteDriver): number { + const value = Number.parseInt(readMeta(db, "format_version") ?? "", 10); + if (!Number.isInteger(value) || value < 1) + throw new StorageError("corrupt", "meta.format_version is missing"); + return value; +} + +/** Takes the pre-migration copy and prunes older ones. */ +export function backUpBeforeMigration( + db: SqliteDriver, + host: StorageHost, + dir: string, + from: number, +): string { + const name = `library.pre-v${from}-${utcStamp(host.now())}.sqlite`; + const target = host.join(dir, name); + const partial = `${target}.partial`; + if (host.exists(partial)) host.remove(partial); + db.prepare("VACUUM INTO ?").run(partial); + host.rename(partial, target); + const copies = host + .list(dir) + .filter((file) => PRE_MIGRATION.test(file)) + .sort(); + for (const old of copies.slice(0, -PRE_MIGRATION_KEEP)) + host.remove(host.join(dir, old)); + return target; +} + +/** + * Migrates `db` from its `format_version` to `target`. Returns the version + * reached; throws `migration_failed` when a step fails (that step rolled + * back, earlier ones stay). + */ +export function runFormatSteps( + db: SqliteDriver, + host: StorageHost, + dir: string, + target: number, + steps: readonly FormatStep[], +): number { + let version = readFormatVersion(db); + if (version >= target) return version; + backUpBeforeMigration(db, host, dir, version); + for (const step of steps) { + if (step.to <= version) continue; + if (step.to > target) break; + try { + transaction(db, () => { + step.up(db); + writeMeta(db, "format_version", String(step.to)); + }); + } catch (error) { + throw new StorageError( + "migration_failed", + `step to v${step.to}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + version = step.to; + } + if (version !== target) + throw new StorageError("migration_failed", `no step reaches v${target}`); + return version; +} diff --git a/desktop/storage/engine/schema.ts b/desktop/storage/engine/schema.ts new file mode 100644 index 00000000..ce6a6636 --- /dev/null +++ b/desktop/storage/engine/schema.ts @@ -0,0 +1,120 @@ +/** + * The library schema, `format_version` 1 (desktop architecture §5.3). + * + * Everything the Local release needs is created here, so no later migration + * is needed for it: `tags`, `doc_tags` and `doc_view` (D7, D10), the details + * spec §6.3 list columns, and the v7/v8 seams `docs.link_key` (NOT NULL, + * minted at insert), `docs.kind` and `import_sources`. `blobs` and + * `doc_blobs` arrive with P2.6b, and the Live release adds `remotes` and + * `versions`; each comes with a `format_version` step. + * + * `updates.seq` is `AUTOINCREMENT`, so a seq is never reused after + * compaction deletes the newest rows: a reused seq at or below a snapshot's + * `upto_seq` would be skipped on load. + */ +export const FORMAT_VERSION = 1; + +/** The Yjs update encoding of every stored BLOB (`meta.yjs_encoding`). */ +export const YJS_ENCODING = "v1"; + +export const SCHEMA_V1 = ` +CREATE TABLE meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +) STRICT; + +CREATE TABLE folders ( + id TEXT PRIMARY KEY, + parent_id TEXT, + name TEXT NOT NULL, + sort_key TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + modified_at INTEGER NOT NULL, + trashed_at INTEGER, + sort TEXT, + view TEXT, + "group" TEXT +) STRICT; + +CREATE TABLE docs ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + folder_id TEXT, + sort_key TEXT NOT NULL DEFAULT '', + pin_key TEXT, + pinned INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + modified_at INTEGER NOT NULL, + opened_at INTEGER, + trashed_at INTEGER, + trash_group TEXT, + trashed_from TEXT, + schema INTEGER NOT NULL DEFAULT 0, + lineage TEXT NOT NULL, + size_bytes INTEGER NOT NULL DEFAULT 0, + snippet TEXT NOT NULL DEFAULT '', + mirrored_at INTEGER, + link_key TEXT NOT NULL UNIQUE, + kind TEXT NOT NULL DEFAULT 'doc' CHECK (kind IN ('doc', 'template')) +) STRICT; + +CREATE TABLE import_sources ( + doc_id TEXT PRIMARY KEY REFERENCES docs(id) ON DELETE CASCADE, + relative_path TEXT, + source_app TEXT, + source_hash TEXT, + imported_at INTEGER +) STRICT; + +CREATE TABLE snapshots ( + doc_id TEXT PRIMARY KEY REFERENCES docs(id) ON DELETE CASCADE, + data BLOB NOT NULL, + upto_seq INTEGER NOT NULL +) STRICT; + +CREATE TABLE updates ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + doc_id TEXT NOT NULL REFERENCES docs(id) ON DELETE CASCADE, + data BLOB NOT NULL, + origin TEXT NOT NULL, + created_at INTEGER NOT NULL +) STRICT; + +CREATE TABLE tags ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE COLLATE NOCASE, + color TEXT +) STRICT; + +CREATE TABLE doc_tags ( + doc_id TEXT NOT NULL REFERENCES docs(id) ON DELETE CASCADE, + tag_id TEXT NOT NULL REFERENCES tags(id) ON DELETE CASCADE, + PRIMARY KEY (doc_id, tag_id) +) STRICT, WITHOUT ROWID; + +CREATE TABLE doc_view ( + doc_id TEXT PRIMARY KEY REFERENCES docs(id) ON DELETE CASCADE, + width TEXT CHECK (width IS NULL OR width IN ('narrow', 'standard', 'wide', 'full')), + place TEXT, + remote_images INTEGER NOT NULL DEFAULT 0 +) STRICT; + +CREATE INDEX updates_doc_seq ON updates(doc_id, seq); +CREATE INDEX docs_folder_list ON docs(folder_id, trashed_at, opened_at); +CREATE INDEX folders_parent ON folders(parent_id); +CREATE INDEX doc_tags_tag ON doc_tags(tag_id); +CREATE INDEX import_sources_hash ON import_sources(source_hash); +`; + +/** Every table format 1 creates; the schema test checks the file against it. */ +export const TABLES_V1 = [ + "doc_tags", + "doc_view", + "docs", + "folders", + "import_sources", + "meta", + "snapshots", + "tags", + "updates", +] as const; diff --git a/desktop/storage/engine/sql.ts b/desktop/storage/engine/sql.ts new file mode 100644 index 00000000..74b6bf14 --- /dev/null +++ b/desktop/storage/engine/sql.ts @@ -0,0 +1,54 @@ +import type { SqliteDriver, SqlRow } from "./driver"; +import { storageErrorFrom } from "./driver"; + +/** + * Runs `body` in one `BEGIN IMMEDIATE … COMMIT`: one transaction, one fsync. + * Any throw rolls back, and a driver failure becomes a `StorageError`. + */ +export function transaction(db: SqliteDriver, body: () => T): T { + try { + db.exec("BEGIN IMMEDIATE"); + } catch (error) { + throw storageErrorFrom(error); + } + try { + const result = body(); + db.exec("COMMIT"); + return result; + } catch (error) { + try { + db.exec("ROLLBACK"); + } catch { + // The failed statement may already have rolled the transaction back. + } + throw storageErrorFrom(error); + } +} + +/** Reads one column of one row; undefined when there is no row. */ +export function scalar( + db: SqliteDriver, + sql: string, + ...params: (string | number | null)[] +): SqlRow[string] | undefined { + const row = db.prepare(sql).get(...params); + return row ? Object.values(row)[0] : undefined; +} + +export function readMeta(db: SqliteDriver, key: string): string | null { + const value = scalar(db, "SELECT value FROM meta WHERE key = ?", key); + return typeof value === "string" ? value : null; +} + +export function writeMeta(db: SqliteDriver, key: string, value: string) { + db.prepare( + "INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value", + ).run(key, value); +} + +/** 32 lowercase hex characters from the platform CSPRNG (`link_key`). */ +export function randomHex32(): string { + const bytes = new Uint8Array(16); + crypto.getRandomValues(bytes); + return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join(""); +} diff --git a/desktop/storage/node-host.ts b/desktop/storage/node-host.ts new file mode 100644 index 00000000..dd38fb3c --- /dev/null +++ b/desktop/storage/node-host.ts @@ -0,0 +1,88 @@ +/** + * The `StorageHost` the worker runs with: `node:fs`, `node:os` and + * `node:path`. Bun implements these modules too, so unit tests use the same + * host (with `now` or `isPidAlive` overridden where a test needs to). + */ +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + realpathSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; + +import type { StorageHost } from "./engine/host"; + +function isErrno(error: unknown, code: string): boolean { + return ( + typeof error === "object" && + error !== null && + (error as { code?: unknown }).code === code + ); +} + +export function createNodeHost( + overrides: Partial = {}, +): StorageHost { + return { + platform: process.platform, + pid: process.pid, + homeDir: homedir(), + env: process.env, + now: () => Date.now(), + join: (...parts) => join(...parts), + realpath(path) { + try { + return realpathSync.native(path); + } catch { + return path; + } + }, + mkdirp: (path) => { + mkdirSync(path, { recursive: true }); + }, + exists: (path) => existsSync(path), + list(dir) { + try { + return readdirSync(dir); + } catch (error) { + if (isErrno(error, "ENOENT")) return []; + throw error; + } + }, + remove: (path) => rmSync(path, { force: true }), + rename: (from, to) => renameSync(from, to), + readText(path) { + try { + return readFileSync(path, "utf8"); + } catch (error) { + if (isErrno(error, "ENOENT")) return null; + throw error; + } + }, + createExclusive(path, text) { + try { + writeFileSync(path, text, { flag: "wx" }); + return true; + } catch (error) { + if (isErrno(error, "EEXIST")) return false; + throw error; + } + }, + isPidAlive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (error) { + // EPERM: it exists but belongs to someone else. + return isErrno(error, "EPERM"); + } + }, + ...overrides, + }; +} diff --git a/tests/helpers/storage-library.ts b/tests/helpers/storage-library.ts new file mode 100644 index 00000000..dc5d8f0a --- /dev/null +++ b/tests/helpers/storage-library.ts @@ -0,0 +1,68 @@ +/** + * Test helpers for the desktop storage engine on `bun:sqlite`: a temp + * library folder, a controllable clock, and an opener with test defaults. + */ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { openBunSqlite } from "@/desktop/dev/bun-sqlite-driver"; +import type { StorageHost } from "@/desktop/storage/engine/host"; +import { + Library, + type OpenLibraryOptions, +} from "@/desktop/storage/engine/library"; +import { createNodeHost } from "@/desktop/storage/node-host"; + +export type TestClock = { now: number; advance(ms: number): void }; + +export function createClock( + start = Date.UTC(2026, 10, 17, 9, 0, 0), +): TestClock { + return { + now: start, + advance(ms) { + this.now += ms; + }, + }; +} + +export function createTempDir(prefix = "tinydocy-lib-"): { + dir: string; + cleanUp(): void; +} { + const dir = mkdtempSync(join(tmpdir(), prefix)); + return { dir, cleanUp: () => rmSync(dir, { recursive: true, force: true }) }; +} + +export function createTestHost( + clock: TestClock, + overrides: Partial = {}, +): StorageHost { + return createNodeHost({ now: () => clock.now, ...overrides }); +} + +export function openTestLibrary( + dir: string, + host: StorageHost, + options: Partial = {}, +): Library { + return Library.open({ + dir, + host, + openSqlite: openBunSqlite, + // Tests don't need F_FULLFSYNC's 4 ms per commit. + fullfsync: false, + ...options, + }); +} + +/** + * Simulates a crash: the library is dropped without `shutdown`, so + * `unclean_exit` stays set and the lock file stays behind. The lock's pid is + * this test process, which the next open treats as stale. + */ +export function crash(library: Library) { + (library as unknown as { db: { close(): void } | null }).db?.close(); + (library as unknown as { db: unknown }).db = null; +} diff --git a/tests/unit/desktop/storage/cloud-folder-and-codes.test.ts b/tests/unit/desktop/storage/cloud-folder-and-codes.test.ts new file mode 100644 index 00000000..c1caed2e --- /dev/null +++ b/tests/unit/desktop/storage/cloud-folder-and-codes.test.ts @@ -0,0 +1,91 @@ +import { describe, expect, test } from "bun:test"; + +import { cloudFolderOf } from "@/desktop/storage/engine/cloud-folder"; +import { STORED_ORIGINS } from "@/desktop/storage/engine/content"; +import { storageErrorFrom } from "@/desktop/storage/engine/driver"; +import { STORAGE_ERROR_CODES } from "@/desktop/storage/engine/errors"; +import { + DOC_STORE_ERROR_CODES, + UPDATE_ORIGINS, +} from "@/lib/platform/doc-store"; + +const host = (platform: string, env: Record = {}) => ({ + platform, + env, + realpath: (path: string) => path, +}); + +describe("cloudFolderOf", () => { + const cases: [string, string, Record, string | null][] = [ + ["darwin", "/Users/a/Library/Application Support/TinyDocy", {}, null], + [ + "darwin", + "/Users/a/Library/Mobile Documents/com~apple~CloudDocs/T", + {}, + "icloud", + ], + [ + "darwin", + "/Users/a/Library/CloudStorage/OneDrive-Contoso/T", + {}, + "onedrive", + ], + [ + "darwin", + "/Users/a/Library/CloudStorage/GoogleDrive-a@b.c/My Drive/T", + {}, + "google_drive", + ], + ["darwin", "/Users/a/Library/CloudStorage/SomethingNew/T", {}, "other"], + ["darwin", "/Users/a/Dropbox/T", {}, "dropbox"], + ["linux", "/home/a/.config/TinyDocy", {}, null], + ["linux", "/home/a/Dropbox (Personal)/T", {}, "dropbox"], + ["win32", "C:\\Users\\a\\AppData\\Local\\TinyDocy", {}, null], + [ + "win32", + "C:\\Users\\a\\Work Files\\TinyDocy", + { OneDriveCommercial: "c:\\users\\a\\work files" }, + "onedrive", + ], + ["win32", "C:\\Users\\a\\OneDrive - Contoso\\T", {}, "onedrive"], + ["win32", "C:\\Users\\a\\iCloudDrive\\T", {}, "icloud"], + ["win32", "D:\\OneDriveBackup\\T", {}, null], + ]; + for (const [platform, path, env, expected] of cases) + test(`${platform} ${path} → ${expected}`, () => { + expect(cloudFolderOf(path, host(platform, env))).toBe(expected as never); + }); +}); + +describe("codes and origins stay in step with the DocStore port", () => { + test("every DocStore code but the web library's is a storage code", () => { + const storage = new Set(STORAGE_ERROR_CODES); + const missing = DOC_STORE_ERROR_CODES.filter( + (code) => + !storage.has(code) && + code !== "quota_exceeded" && + code !== "build_mismatch", + ); + expect(missing).toEqual([]); + }); + + test("stored origins match UPDATE_ORIGINS", () => { + expect([...STORED_ORIGINS]).toEqual([...UPDATE_ORIGINS]); + }); +}); + +describe("storageErrorFrom", () => { + const nodeError = (errcode: number) => + Object.assign(new Error("x"), { errcode }); + test.each([ + [13, "disk_full"], + [11, "corrupt"], + [26, "corrupt"], + [19 | (8 << 8), "invalid_argument"], + [5, "locked_elsewhere"], + [8, "library_read_only"], + [10 | (12 << 8), "io_error"], + ])("SQLite %i → %s", (code, expected) => { + expect(storageErrorFrom(nodeError(code)).code).toBe(expected as never); + }); +}); diff --git a/tests/unit/desktop/storage/content.test.ts b/tests/unit/desktop/storage/content.test.ts new file mode 100644 index 00000000..d533c9d4 --- /dev/null +++ b/tests/unit/desktop/storage/content.test.ts @@ -0,0 +1,394 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; + +import * as Y from "yjs"; + +import { StorageError } from "@/desktop/storage/engine/errors"; +import type { Library } from "@/desktop/storage/engine/library"; +import { + crash, + createClock, + createTempDir, + createTestHost, + openTestLibrary, + type TestClock, +} from "@/tests/helpers/storage-library"; + +let temp: ReturnType; +let clock: TestClock; +let library: Library; +let docId: string; + +beforeEach(() => { + temp = createTempDir(); + clock = createClock(); + library = openTestLibrary(temp.dir, createTestHost(clock)); + docId = library.createDoc({ title: "Untitled" }).id; +}); +afterEach(() => { + library.shutdown(); + temp.cleanUp(); +}); + +function edit(doc: Y.Doc, text: string): Uint8Array { + let update: Uint8Array = new Uint8Array(); + const capture = (u: Uint8Array) => { + update = u; + }; + doc.once("update", capture); + doc.getText("t").insert(doc.getText("t").length, text); + return update; +} + +function textOf(loaded: { + snapshot: Uint8Array | null; + updates: Uint8Array[]; +}) { + const doc = new Y.Doc(); + if (loaded.snapshot) Y.applyUpdate(doc, loaded.snapshot); + for (const update of loaded.updates) Y.applyUpdate(doc, update); + return doc.getText("t").toString(); +} + +function codeOf(run: () => unknown): string | undefined { + try { + run(); + } catch (error) { + if (error instanceof StorageError) return error.code; + throw error; + } + return undefined; +} + +describe("load and append", () => { + test("a new document loads empty at schema 0", () => { + expect(library.load(docId, "w1")).toEqual({ + snapshot: null, + updates: [], + maxSeq: 0, + schema: 0, + }); + }); + + test("appends come back in order with increasing seqs", () => { + const doc = new Y.Doc(); + library.load(docId, "w1"); + const a = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "a"), + }); + const b = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "b"), + }); + expect(b).toBeGreaterThan(a); + const loaded = library.load(docId, "w1"); + expect(loaded.updates).toHaveLength(2); + expect(loaded.maxSeq).toBe(b); + expect(textOf(loaded)).toBe("ab"); + }); + + test("seqs are global across documents", () => { + const other = library.createDoc().id; + const doc = new Y.Doc(); + const a = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "a"), + }); + const b = library.append(other, "w2", { + origin: "local", + update: edit(doc, "b"), + }); + expect(b).toBe(a + 1); + }); + + test("an append updates the title, snippet, schema, size and modified time", () => { + clock.advance(5000); + const update = edit(new Y.Doc(), "hello"); + library.append(docId, "w1", { + origin: "migration", + update, + title: "Hello", + snippet: "hello world", + schemaTo: 2, + }); + const row = library.sidebar().docs[0]; + expect(row?.title).toBe("Hello"); + expect(row?.snippet).toBe("hello world"); + expect(row?.modifiedAt).toBe(clock.now); + expect(row?.sizeBytes).toBe(update.byteLength); + expect(library.load(docId, "w1").schema).toBe(2); + }); + + test("an append without a title keeps the old one; titles are capped", () => { + library.append(docId, "w1", { + origin: "local", + update: edit(new Y.Doc(), "x"), + title: "T".repeat(500), + }); + library.append(docId, "w1", { + origin: "local", + update: edit(new Y.Doc(), "y"), + }); + expect(library.sidebar().docs[0]?.title).toHaveLength(200); + }); + + test("appending the same bytes twice (a resend) changes nothing", () => { + const doc = new Y.Doc(); + const update = edit(doc, "once"); + library.append(docId, "w1", { origin: "local", update }); + library.append(docId, "w1", { origin: "local", update }); + expect(textOf(library.load(docId, "w1"))).toBe("once"); + }); + + test("refuses bad input and unknown documents", () => { + const update = edit(new Y.Doc(), "x"); + expect( + codeOf(() => + library.append(docId, "w1", { origin: "bogus" as "local", update }), + ), + ).toBe("invalid_argument"); + expect( + codeOf(() => + library.append(docId, "w1", { + origin: "local", + update: new Uint8Array(), + }), + ), + ).toBe("invalid_argument"); + expect( + codeOf(() => + library.append(crypto.randomUUID(), "w1", { origin: "local", update }), + ), + ).toBe("not_found"); + expect(codeOf(() => library.load(crypto.randomUUID(), "w1"))).toBe( + "not_found", + ); + }); + + test("a failed append writes nothing", () => { + const update = edit(new Y.Doc(), "x"); + codeOf(() => + library.append(crypto.randomUUID(), "w1", { origin: "local", update }), + ); + expect(library.stats().pendingRows).toBe(0); + }); +}); + +describe("compaction (D-A3)", () => { + test("stores the snapshot and deletes the rows it covers", () => { + const doc = new Y.Doc(); + library.load(docId, "w1"); + library.append(docId, "w1", { origin: "local", update: edit(doc, "a") }); + const covered = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "b"), + }); + const state = Y.encodeStateAsUpdate(doc); + const late = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "c"), + }); + expect(library.compact(docId, "w1", state, covered)).toEqual({ + foreign: null, + }); + const loaded = library.load(docId, "w1"); + expect(loaded.updates).toHaveLength(1); + expect(loaded.maxSeq).toBe(late); + expect(textOf(loaded)).toBe("abc"); + expect(library.sidebar().docs[0]?.sizeBytes).toBe( + state.byteLength + (loaded.updates[0]?.byteLength ?? 0), + ); + }); + + test("maxSeq is the snapshot's when every row is folded in", () => { + const doc = new Y.Doc(); + const seq = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "a"), + }); + library.compact(docId, "w1", Y.encodeStateAsUpdate(doc), seq); + expect(library.load(docId, "w1")).toMatchObject({ + updates: [], + maxSeq: seq, + }); + }); + + test("a seq is never reused after compaction deletes the newest rows", () => { + const doc = new Y.Doc(); + const seq = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "a"), + }); + library.compact(docId, "w1", Y.encodeStateAsUpdate(doc), seq); + library.shutdown(); + library = openTestLibrary(temp.dir, createTestHost(clock)); + const next = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "b"), + }); + expect(next).toBeGreaterThan(seq); + expect(textOf(library.load(docId, "w1"))).toBe("ab"); + }); + + test("a stale compaction never moves the snapshot back", () => { + const doc = new Y.Doc(); + const first = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "a"), + }); + const oldState = Y.encodeStateAsUpdate(doc); + const second = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "b"), + }); + library.compact(docId, "w1", Y.encodeStateAsUpdate(doc), second); + library.compact(docId, "w1", oldState, first); + expect(textOf(library.load(docId, "w1"))).toBe("ab"); + }); + + test("refuses an upToSeq past the last assigned seq", () => { + const doc = new Y.Doc(); + const seq = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "a"), + }); + expect( + codeOf(() => + library.compact(docId, "w1", Y.encodeStateAsUpdate(doc), seq + 1), + ), + ).toBe("invalid_argument"); + }); + + test("stateOf merges the snapshot and rows", () => { + const doc = new Y.Doc(); + const seq = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "a"), + }); + library.compact(docId, "w1", Y.encodeStateAsUpdate(doc), seq); + library.append(docId, "w1", { origin: "local", update: edit(doc, "b") }); + const copy = new Y.Doc(); + Y.applyUpdate(copy, library.stateOf(docId)); + expect(copy.getText("t").toString()).toBe("ab"); + }); +}); + +describe("one window per document (registry)", () => { + test("a second window gets already_open; the first keeps working", () => { + library.load(docId, "w1"); + expect(codeOf(() => library.load(docId, "w2"))).toBe("already_open"); + expect(library.load(docId, "w1").schema).toBe(0); + }); + + test("another window can't append or compact an open document", () => { + library.load(docId, "w1"); + const update = edit(new Y.Doc(), "x"); + expect( + codeOf(() => library.append(docId, "w2", { origin: "local", update })), + ).toBe("forbidden"); + expect(codeOf(() => library.compact(docId, "w2", update, 0))).toBe( + "forbidden", + ); + }); + + test("close and a closed window release the document", () => { + library.load(docId, "w1"); + library.close(docId, "w1"); + expect(library.load(docId, "w2").schema).toBe(0); + library.releaseOwner("w2"); + expect(library.isOpen(docId)).toBe(false); + }); + + test("closing from another window does nothing", () => { + library.load(docId, "w1"); + library.close(docId, "w2"); + expect(library.isOpen(docId)).toBe(true); + }); + + test("main's content writes refuse an open document", () => { + library.load(docId, "w1"); + expect(codeOf(() => library.purgeDoc(docId))).toBe("document_open"); + library.close(docId, "w1"); + library.purgeDoc(docId); + expect(library.sidebar().docs).toEqual([]); + }); +}); + +describe("compaction property (architecture §11)", () => { + // A seeded PRNG keeps failures reproducible. + function mulberry32(seed: number) { + let a = seed; + return () => { + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + } + + for (const seed of [1, 2, 3, 4, 5]) { + test(`random appends, compactions, crashes and reopens (seed ${seed})`, () => { + const random = mulberry32(seed); + const reference = new Y.Doc(); + // The renderer's copy: rebuilt from storage on every reopen. + let session = new Y.Doc(); + let ackedSeq = 0; + const hydrate = () => { + const loaded = library.load(docId, "w1"); + session = new Y.Doc(); + if (loaded.snapshot) Y.applyUpdate(session, loaded.snapshot); + for (const u of loaded.updates) Y.applyUpdate(session, u); + ackedSeq = loaded.maxSeq; + }; + hydrate(); + for (let step = 0; step < 300; step += 1) { + const roll = random(); + if (roll < 0.7) { + const text = session.getText("t"); + const pos = Math.floor(random() * (text.length + 1)); + let update: Uint8Array = new Uint8Array(); + session.once("update", (u: Uint8Array) => { + update = u; + }); + if (text.length > 0 && random() < 0.25) + text.delete(Math.min(pos, text.length - 1), 1); + else + text.insert( + pos, + String.fromCharCode(97 + Math.floor(random() * 26)), + ); + Y.applyUpdate(reference, update); + ackedSeq = library.append(docId, "w1", { origin: "local", update }); + } else if (roll < 0.85) { + library.compact( + docId, + "w1", + Y.encodeStateAsUpdate(session), + ackedSeq, + ); + } else if (roll < 0.95) { + library.close(docId, "w1"); + library.shutdown(); + library = openTestLibrary(temp.dir, createTestHost(clock)); + hydrate(); + } else { + crash(library); + library = openTestLibrary(temp.dir, createTestHost(clock)); + hydrate(); + } + if (step % 25 === 0 || roll >= 0.85) { + const stored = library.load(docId, "w1"); + const rebuilt = new Y.Doc(); + if (stored.snapshot) Y.applyUpdate(rebuilt, stored.snapshot); + for (const u of stored.updates) Y.applyUpdate(rebuilt, u); + expect(rebuilt.getText("t").toString()).toBe( + reference.getText("t").toString(), + ); + expect(Y.encodeStateVector(rebuilt)).toEqual( + Y.encodeStateVector(reference), + ); + } + } + }); + } +}); diff --git a/tests/unit/desktop/storage/daily-copy.test.ts b/tests/unit/desktop/storage/daily-copy.test.ts new file mode 100644 index 00000000..766f2d5b --- /dev/null +++ b/tests/unit/desktop/storage/daily-copy.test.ts @@ -0,0 +1,97 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; + +import * as Y from "yjs"; + +import { DAILY_KEEP, localDate } from "@/desktop/storage/engine/daily-copy"; +import { IDLE_MS, type Library } from "@/desktop/storage/engine/library"; +import { + createClock, + createTempDir, + createTestHost, + openTestLibrary, + type TestClock, +} from "@/tests/helpers/storage-library"; + +const DAY = 24 * 60 * 60 * 1000; + +let temp: ReturnType; +let clock: TestClock; +let library: Library; + +beforeEach(() => { + temp = createTempDir(); + clock = createClock(); + library = openTestLibrary(temp.dir, createTestHost(clock)); +}); +afterEach(() => { + library.shutdown(); + temp.cleanUp(); +}); + +function write(docId: string, text: string) { + const doc = new Y.Doc(); + doc.getText("t").insert(0, text); + library.append(docId, "w1", { + origin: "local", + update: Y.encodeStateAsUpdate(doc), + }); +} + +describe("daily copies (D-A6)", () => { + test("waits for 30 s without a write", () => { + write(library.createDoc().id, "x"); + clock.advance(IDLE_MS - 1); + expect(library.runIdleJobs().dailyCopy).toBeNull(); + clock.advance(1); + expect(library.runIdleJobs().dailyCopy).toBe( + `library-${localDate(clock.now)}.sqlite`, + ); + }); + + test("takes one copy a day and keeps the newest 7", () => { + const docId = library.createDoc().id; + const taken: string[] = []; + for (let day = 0; day < 10; day += 1) { + write(docId, `day ${day}`); + clock.advance(IDLE_MS); + const first = library.runIdleJobs().dailyCopy; + expect(first).not.toBeNull(); + taken.push(localDate(clock.now)); + expect(library.runIdleJobs().dailyCopy).toBeNull(); + clock.advance(DAY); + } + expect(library.listDailyCopies()).toEqual(taken.slice(-DAILY_KEEP)); + }); + + test("a copy restores one document's state as of that day", () => { + const docId = library.createDoc().id; + write(docId, "before"); + clock.advance(IDLE_MS); + library.runIdleJobs(); + const day = localDate(clock.now); + write(docId, "after"); + const restored = new Y.Doc(); + Y.applyUpdate(restored, library.readDailyCopy(day, docId)); + expect(restored.getText("t").toString()).toBe("before"); + }); + + test("survives a leftover partial copy", () => { + const host = createTestHost(clock); + host.mkdirp(host.join(temp.dir, "daily")); + const partial = host.join( + temp.dir, + "daily", + `library-${localDate(clock.now)}.sqlite.partial`, + ); + host.createExclusive(partial, "junk"); + clock.advance(IDLE_MS); + expect(library.runIdleJobs().dailyCopy).not.toBeNull(); + expect(host.exists(partial)).toBe(false); + }); + + test("refuses a malformed date", () => { + expect(() => + library.readDailyCopy("../../etc", crypto.randomUUID()), + ).toThrow(); + }); +}); diff --git a/tests/unit/desktop/storage/library-index.test.ts b/tests/unit/desktop/storage/library-index.test.ts new file mode 100644 index 00000000..72774bbf --- /dev/null +++ b/tests/unit/desktop/storage/library-index.test.ts @@ -0,0 +1,203 @@ +import { Database } from "bun:sqlite"; +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { join } from "node:path"; + +import * as Y from "yjs"; + +import { StorageError } from "@/desktop/storage/engine/errors"; +import { + IDLE_MS, + LIBRARY_FILE, + type Library, +} from "@/desktop/storage/engine/library"; +import { TRASH_RETENTION_MS } from "@/desktop/storage/engine/library-index"; +import { + createClock, + createTempDir, + createTestHost, + openTestLibrary, + type TestClock, +} from "@/tests/helpers/storage-library"; + +let temp: ReturnType; +let clock: TestClock; +let library: Library; + +beforeEach(() => { + temp = createTempDir(); + clock = createClock(); + library = openTestLibrary(temp.dir, createTestHost(clock)); +}); +afterEach(() => { + library.shutdown(); + temp.cleanUp(); +}); + +function codeOf(run: () => unknown) { + try { + run(); + } catch (error) { + if (error instanceof StorageError) return error.code; + throw error; + } + return undefined; +} + +describe("creating documents", () => { + test("mints a UUID, a lineage and a 32-hex link key at insert", () => { + const a = library.createDoc(); + const b = library.createDoc(); + expect(a.id).toMatch(/^[0-9a-f-]{36}$/); + expect(a.lineage).toMatch(/^[0-9a-f-]{36}$/); + expect(a.linkKey).toMatch(/^[0-9a-f]{32}$/); + expect(a.lineage).not.toBe(b.lineage); + expect(a.linkKey).not.toBe(b.linkKey); + }); + + test("keeps a lineage and link key only when given (backup restore)", () => { + const lineage = crypto.randomUUID(); + const linkKey = "a".repeat(32); + expect(library.createDoc({ lineage, linkKey })).toMatchObject({ + lineage, + linkKey, + }); + expect(codeOf(() => library.createDoc({ linkKey }))).toBe( + "invalid_argument", + ); + }); + + test("refuses malformed ids, link keys and kinds", () => { + expect(codeOf(() => library.createDoc({ id: "../x" }))).toBe( + "invalid_argument", + ); + expect(codeOf(() => library.createDoc({ linkKey: "XYZ" }))).toBe( + "invalid_argument", + ); + expect(codeOf(() => library.createDoc({ kind: "page" as "doc" }))).toBe( + "invalid_argument", + ); + }); + + test("templates are stored with kind 'template'", () => { + library.createDoc({ kind: "template", title: "Weekly" }); + expect(library.sidebar().docs[0]?.kind).toBe("template"); + }); +}); + +describe("the sidebar snapshot", () => { + test("lists live folders, documents and tags; rev moves on every write", () => { + const start = library.sidebar().rev; + const folder = library.createFolder({ name: "Work" }); + const doc = library.createDoc({ title: "Plan", folderId: folder.id }); + const tag = library.createTag("urgent", "red"); + library.setDocTags(doc.id, [tag.id]); + const snapshot = library.sidebar(); + expect(snapshot.rev).toBe(start + 4); + expect(snapshot.folders.map((f) => f.name)).toEqual(["Work"]); + expect(snapshot.docs[0]).toMatchObject({ + id: doc.id, + title: "Plan", + folderId: folder.id, + tagIds: [tag.id], + pinned: false, + kind: "doc", + }); + expect(snapshot.tags).toEqual([ + { id: tag.id, name: "urgent", color: "red" }, + ]); + }); + + test("tag names are unique regardless of case", () => { + library.createTag("Urgent"); + expect(codeOf(() => library.createTag("urgent"))).toBe("invalid_argument"); + }); + + test("opening a document records opened_at", () => { + const doc = library.createDoc(); + clock.advance(1000); + library.load(doc.id, "w1"); + expect(library.sidebar().docs[0]?.openedAt).toBe(clock.now); + }); + + test("rename and move", () => { + const folder = library.createFolder({ name: "A" }); + const doc = library.createDoc({ title: "x" }); + library.renameDoc(doc.id, " Renamed "); + library.moveDoc(doc.id, folder.id, "m"); + expect(library.sidebar().docs[0]).toMatchObject({ + title: "Renamed", + folderId: folder.id, + sortKey: "m", + }); + expect(codeOf(() => library.renameDoc(crypto.randomUUID(), "y"))).toBe( + "not_found", + ); + }); +}); + +describe("Trash and purge", () => { + test("trash hides a document and restore brings it back to its folder", () => { + const folder = library.createFolder({ name: "A" }); + const doc = library.createDoc({ folderId: folder.id }); + library.trashDoc(doc.id); + expect(library.sidebar().docs).toEqual([]); + library.restoreDoc(doc.id); + expect(library.sidebar().docs[0]?.folderId).toBe(folder.id); + }); + + test("purge cascades to rows, snapshot, tags, view and import source", () => { + const doc = library.createDoc(); + const tag = library.createTag("t"); + library.setDocTags(doc.id, [tag.id]); + const ydoc = new Y.Doc(); + ydoc.getText("t").insert(0, "x"); + const update = Y.encodeStateAsUpdate(ydoc); + const seq = library.append(doc.id, "w1", { origin: "local", update }); + library.compact(doc.id, "w1", update, seq); + library.append(doc.id, "w1", { origin: "local", update }); + library.shutdown(); + const raw = new Database(join(temp.dir, LIBRARY_FILE)); + raw.exec("PRAGMA foreign_keys = ON"); + raw + .prepare("INSERT INTO doc_view (doc_id, width) VALUES (?, 'wide')") + .run(doc.id); + raw + .prepare( + "INSERT INTO import_sources (doc_id, source_hash) VALUES (?, 'h')", + ) + .run(doc.id); + raw.close(); + library = openTestLibrary(temp.dir, createTestHost(clock)); + library.purgeDoc(doc.id); + library.shutdown(); + const check = new Database(join(temp.dir, LIBRARY_FILE)); + for (const table of [ + "updates", + "snapshots", + "doc_tags", + "doc_view", + "import_sources", + ]) { + const row = check.query(`SELECT COUNT(*) AS n FROM ${table}`).get() as { + n: number; + }; + expect(`${table}=${row.n}`).toBe(`${table}=0`); + } + check.close(); + library = openTestLibrary(temp.dir, createTestHost(clock)); + }); + + test("idle jobs empty Trash after 30 days, skipping open documents", () => { + const old = library.createDoc(); + const open = library.createDoc(); + const recent = library.createDoc(); + library.trashDoc(old.id); + library.trashDoc(open.id); + clock.advance(TRASH_RETENTION_MS - 1000); + library.trashDoc(recent.id); + library.load(open.id, "w1"); + clock.advance(IDLE_MS + 1000); + expect(library.runIdleJobs().purged).toBe(1); + expect(library.stats().docs).toBe(2); + }); +}); diff --git a/tests/unit/desktop/storage/library-open.test.ts b/tests/unit/desktop/storage/library-open.test.ts new file mode 100644 index 00000000..2be72f48 --- /dev/null +++ b/tests/unit/desktop/storage/library-open.test.ts @@ -0,0 +1,343 @@ +import { Database } from "bun:sqlite"; +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +import { StorageError } from "@/desktop/storage/engine/errors"; +import { LIBRARY_FILE, Library } from "@/desktop/storage/engine/library"; +import { LOCK_FILE } from "@/desktop/storage/engine/lock"; +import type { FormatStep } from "@/desktop/storage/engine/migrations"; +import { TABLES_V1 } from "@/desktop/storage/engine/schema"; +import { + crash, + createClock, + createTempDir, + createTestHost, + openTestLibrary, + type TestClock, +} from "@/tests/helpers/storage-library"; + +let temp: ReturnType; +let clock: TestClock; + +beforeEach(() => { + temp = createTempDir(); + clock = createClock(); +}); +afterEach(() => temp.cleanUp()); + +function raw(dir = temp.dir) { + return new Database(join(dir, LIBRARY_FILE)); +} + +function rejectsWith(code: string, run: () => unknown) { + let caught: unknown; + try { + run(); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(StorageError); + expect((caught as StorageError).code).toBe(code as StorageError["code"]); +} + +describe("creating a library", () => { + test("creates every format-1 table and the meta rows", () => { + const library = openTestLibrary(temp.dir, createTestHost(clock)); + library.shutdown(); + const db = raw(); + const tables = db + .query( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", + ) + .all() + .map((row) => (row as { name: string }).name); + expect(tables).toEqual([...TABLES_V1]); + const meta = Object.fromEntries( + db + .query("SELECT key, value FROM meta") + .all() + .map((row) => { + const { key, value } = row as { key: string; value: string }; + return [key, value]; + }), + ); + expect(meta.format_version).toBe("1"); + expect(meta.yjs_encoding).toBe("v1"); + expect(meta.library_id).toMatch(/^[0-9a-f-]{36}$/); + expect(meta.unclean_exit).toBe("0"); + db.close(); + }); + + test("the docs columns include the v7/v8 seams and the §6.3 list columns", () => { + openTestLibrary(temp.dir, createTestHost(clock)).shutdown(); + const db = raw(); + const columns = db + .query("SELECT name, \"notnull\" FROM pragma_table_info('docs')") + .all() as { name: string; notnull: number }[]; + const byName = new Map(columns.map((c) => [c.name, c.notnull])); + for (const name of [ + "lineage", + "link_key", + "kind", + "pin_key", + "size_bytes", + "snippet", + "trash_group", + "trashed_from", + "mirrored_at", + "schema", + ]) + expect(byName.has(name)).toBe(true); + expect(byName.get("link_key")).toBe(1); + expect(byName.get("lineage")).toBe(1); + const view = db + .query("SELECT name FROM pragma_table_info('doc_view')") + .all() + .map((row) => (row as { name: string }).name); + expect(view).toEqual(["doc_id", "width", "place", "remote_images"]); + db.close(); + }); + + test("sets WAL, synchronous=FULL and foreign keys", () => { + const library = openTestLibrary(temp.dir, createTestHost(clock)); + const db = ( + library as unknown as { + db: { + prepare(s: string): { get(): Record | undefined }; + }; + } + ).db; + expect(db.prepare("PRAGMA journal_mode").get()).toEqual({ + journal_mode: "wal", + }); + expect(db.prepare("PRAGMA synchronous").get()).toEqual({ synchronous: 2 }); + expect(db.prepare("PRAGMA foreign_keys").get()).toEqual({ + foreign_keys: 1, + }); + library.shutdown(); + }); + + test("turns on fullfsync on macOS", () => { + const library = openTestLibrary( + temp.dir, + createTestHost(clock, { platform: "darwin" }), + { fullfsync: true }, + ); + const db = ( + library as unknown as { + db: { + prepare(s: string): { get(): Record | undefined }; + }; + } + ).db; + expect(db.prepare("PRAGMA fullfsync").get()).toEqual({ fullfsync: 1 }); + expect(db.prepare("PRAGMA checkpoint_fullfsync").get()).toEqual({ + checkpoint_fullfsync: 1, + }); + library.shutdown(); + }); + + test("refuses a file that isn't a library", () => { + const db = raw(); + db.exec("CREATE TABLE other (x)"); + db.close(); + rejectsWith("corrupt", () => + openTestLibrary(temp.dir, createTestHost(clock)), + ); + // The lock is released on failure. + expect(() => readFileSync(join(temp.dir, LOCK_FILE))).toThrow(); + }); +}); + +describe("cloud folders", () => { + test("refuses a library inside a synced folder", () => { + const dir = join(temp.dir, "Dropbox", "TinyDocy"); + rejectsWith("cloud_folder", () => + openTestLibrary(dir, createTestHost(clock)), + ); + }); +}); + +describe("library.lock", () => { + test("refuses a library another running process holds", () => { + writeFileSync(join(temp.dir, LOCK_FILE), "424242"); + const host = createTestHost(clock, { isPidAlive: (pid) => pid === 424242 }); + rejectsWith("locked_elsewhere", () => openTestLibrary(temp.dir, host)); + }); + + test("takes over a stale lock", () => { + writeFileSync(join(temp.dir, LOCK_FILE), "424242"); + const host = createTestHost(clock, { isPidAlive: () => false }); + const library = openTestLibrary(temp.dir, host); + expect(readFileSync(join(temp.dir, LOCK_FILE), "utf8")).toBe( + String(process.pid), + ); + library.shutdown(); + expect(() => readFileSync(join(temp.dir, LOCK_FILE))).toThrow(); + }); +}); + +describe("unclean exit", () => { + test("a clean shutdown needs no quick_check", () => { + openTestLibrary(temp.dir, createTestHost(clock)).shutdown(); + const library = openTestLibrary(temp.dir, createTestHost(clock)); + expect(library.status.uncleanExit).toBe(false); + expect(library.status.quickCheck).toBeNull(); + library.shutdown(); + }); + + test("a crash runs quick_check on the next open", () => { + const first = openTestLibrary(temp.dir, createTestHost(clock)); + first.createDoc(); + crash(first); + const library = openTestLibrary(temp.dir, createTestHost(clock)); + expect(library.status.uncleanExit).toBe(true); + expect(library.status.quickCheck).toBe("ok"); + expect(library.status.readOnly).toBeNull(); + library.shutdown(); + }); + + test("shutdown checkpoints the WAL into the main file", () => { + const library = openTestLibrary(temp.dir, createTestHost(clock)); + library.createDoc({ title: "kept" }); + library.shutdown(); + const wal = join(temp.dir, `${LIBRARY_FILE}-wal`); + let walBytes = 0; + try { + walBytes = readFileSync(wal).byteLength; + } catch { + walBytes = 0; + } + expect(walBytes).toBe(0); + }); + + test("shutdown is idempotent", () => { + const library = openTestLibrary(temp.dir, createTestHost(clock)); + library.shutdown(); + expect(() => library.shutdown()).not.toThrow(); + }); +}); + +describe("format_version guard (T6)", () => { + function makeNewerLibrary() { + const library = openTestLibrary(temp.dir, createTestHost(clock)); + library.createDoc({ title: "from the future" }); + library.shutdown(); + const db = raw(); + db.exec("UPDATE meta SET value = '7' WHERE key = 'format_version'"); + db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); + db.close(); + } + + test("a newer library opens read-only and is never written", () => { + makeNewerLibrary(); + const before = readFileSync(join(temp.dir, LIBRARY_FILE)); + const library = openTestLibrary(temp.dir, createTestHost(clock)); + expect(library.status.readOnly).toBe("library_too_new"); + expect(library.status.formatVersion).toBe(7); + expect(library.sidebar().docs.map((d) => d.title)).toEqual([ + "from the future", + ]); + rejectsWith("library_too_new", () => library.createDoc()); + const id = library.sidebar().docs[0]?.id ?? ""; + expect(library.load(id, "w1").updates).toEqual([]); + rejectsWith("library_too_new", () => + library.append(id, "w1", { + origin: "local", + update: new Uint8Array([1]), + }), + ); + library.shutdown(); + expect(readFileSync(join(temp.dir, LIBRARY_FILE)).equals(before)).toBe( + true, + ); + }); +}); + +describe("format migrations (T5)", () => { + const addColumn: FormatStep = { + to: 2, + up: (db) => db.exec("ALTER TABLE docs ADD COLUMN extra TEXT"), + }; + const addTable: FormatStep = { + to: 3, + up: (db) => db.exec("CREATE TABLE later (x INTEGER) STRICT"), + }; + + test("takes a pre-migration copy, then runs each step", () => { + openTestLibrary(temp.dir, createTestHost(clock)).shutdown(); + const library = openTestLibrary(temp.dir, createTestHost(clock), { + formatVersion: 3, + formatSteps: [addColumn, addTable], + }); + expect(library.status.formatVersion).toBe(3); + expect(library.status.readOnly).toBeNull(); + library.shutdown(); + const copies = createTestHost(clock) + .list(temp.dir) + .filter((f) => f.startsWith("library.pre-v1-")); + expect(copies).toHaveLength(1); + const copy = new Database(join(temp.dir, copies[0] ?? "")); + expect( + ( + copy + .query("SELECT value FROM meta WHERE key = 'format_version'") + .get() as { value: string } + ).value, + ).toBe("1"); + copy.close(); + }); + + test("a failed step leaves the library read-only at the last good version", () => { + openTestLibrary(temp.dir, createTestHost(clock)).shutdown(); + const library = openTestLibrary(temp.dir, createTestHost(clock), { + formatVersion: 3, + formatSteps: [ + addColumn, + { + to: 3, + up: (db) => { + db.exec("CREATE TABLE half (x)"); + throw new Error("boom"); + }, + }, + ], + }); + expect(library.status.readOnly).toBe("migration_failed"); + expect(library.status.formatVersion).toBe(2); + rejectsWith("library_read_only", () => library.createDoc()); + library.shutdown(); + const db = raw(); + expect( + db.query("SELECT 1 FROM sqlite_master WHERE name = 'half'").get(), + ).toBeNull(); + db.close(); + }); + + test("keeps the newest two pre-migration copies", () => { + openTestLibrary(temp.dir, createTestHost(clock)).shutdown(); + for (const to of [2, 3, 4]) { + clock.advance(60_000); + openTestLibrary(temp.dir, createTestHost(clock), { + formatVersion: to, + formatSteps: [ + { to: 2, up: () => {} }, + { to: 3, up: () => {} }, + { to: 4, up: () => {} }, + ], + }).shutdown(); + } + const copies = createTestHost(clock) + .list(temp.dir) + .filter((f) => f.startsWith("library.pre-v")) + .sort(); + expect(copies).toHaveLength(2); + expect(copies[0]).toStartWith("library.pre-v2-"); + expect(copies[1]).toStartWith("library.pre-v3-"); + }); +}); + +test("Library.open is the only constructor", () => { + expect(typeof Library.open).toBe("function"); +}); From 4b944d842435f257107b81cbef975064c80faf1c Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:26:24 +0330 Subject: [PATCH 03/19] feat(platform): add the document migration hook and the import guard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit lib/doc-migrations/ holds the pure Yjs forward migrations that local-session runs after hydrate and before the editor binds, and that import runs on .tinydoc and web downloads (desktop architecture §5.1, D-A5). The list starts empty, so DOC_SCHEMA is 0. checkImport refuses a state that is too large, isn't Yjs, or carries a node, mark or root this build doesn't know, before it reaches the library. The legacy link and code marks pass: the import migrations rename them. --- lib/doc-migrations/import-guard.ts | 96 +++++++++++++++++++ lib/doc-migrations/index.ts | 70 ++++++++++++++ .../lib/doc-migrations/doc-migrations.test.ts | 49 ++++++++++ .../lib/doc-migrations/import-guard.test.ts | 92 ++++++++++++++++++ 4 files changed, 307 insertions(+) create mode 100644 lib/doc-migrations/import-guard.ts create mode 100644 lib/doc-migrations/index.ts create mode 100644 tests/unit/lib/doc-migrations/doc-migrations.test.ts create mode 100644 tests/unit/lib/doc-migrations/import-guard.test.ts diff --git a/lib/doc-migrations/import-guard.ts b/lib/doc-migrations/import-guard.ts new file mode 100644 index 00000000..7547a6f4 --- /dev/null +++ b/lib/doc-migrations/import-guard.ts @@ -0,0 +1,96 @@ +/** + * The import guard (plan P2.6; architecture §3.3): checks a document's + * state before it enters the library from a `.tinydoc`, a web download or a + * backup. It rejects and never repairs; the import migrations + * ({@link import("./index")}) run only on a state that passed. + * + * Refused: + * - **size:** over {@link IMPORT_MAX_BYTES} (`import_too_large`); + * - **not Yjs:** bytes that don't decode (`corrupt`); + * - **unknown names:** a node or mark this build's schema doesn't know + * (`unknown_schema`). y-prosemirror would delete such content the moment + * the editor binds. The legacy `link` and `code` marks are allowed: the + * import migrations rename them; + * - **unknown roots:** a top-level type other than the editor fragment + * (`unknown_schema`). Comments add their `comments` root here, with their + * validator, in P4.11. + * + * Pure: Yjs and the schema name lists only, so main imports it. + */ +import * as Y from "yjs"; + +import { + isAllowedMarkKey, + isAllowedNodeName, + markNameFromKey, +} from "@/lib/schema-names"; + +/** The editor fragment, `EDITOR_FRAGMENT_NAME` in `lib/yjs-mark-migration.ts`. */ +const EDITOR_ROOT = "default"; + +/** + * 64 MiB: far above any document the editor stays responsive with, and + * low enough that a hostile file can't exhaust the worker's memory while it + * is decoded. + */ +export const IMPORT_MAX_BYTES = 64 * 1024 * 1024; + +export const IMPORT_ROOTS: readonly string[] = [EDITOR_ROOT]; + +export type ImportVerdict = + | { ok: true } + | { + ok: false; + code: "import_too_large" | "corrupt" | "unknown_schema"; + /** Up to five offending names, for the dialog and the log. */ + names: string[]; + }; + +export type ImportGuardOptions = { + maxBytes?: number; + roots?: readonly string[]; +}; + +const MAX_NAMES = 5; + +export function checkImport( + state: Uint8Array, + { + maxBytes = IMPORT_MAX_BYTES, + roots = IMPORT_ROOTS, + }: ImportGuardOptions = {}, +): ImportVerdict { + if (state.byteLength > maxBytes) + return { ok: false, code: "import_too_large", names: [] }; + const names = new Set(); + const note = (name: string) => { + if (names.size < MAX_NAMES) names.add(name.slice(0, 64)); + }; + const doc = new Y.Doc(); + try { + for (const struct of Y.decodeUpdate(state).structs) { + if (!(struct instanceof Y.Item)) continue; + const { content } = struct; + if ( + content instanceof Y.ContentType && + content.type instanceof Y.XmlElement && + !isAllowedNodeName(content.type.nodeName) + ) + note(content.type.nodeName); + else if ( + content instanceof Y.ContentFormat && + !isAllowedMarkKey(content.key) + ) + note(markNameFromKey(content.key)); + } + Y.applyUpdate(doc, state); + for (const root of doc.share.keys()) if (!roots.includes(root)) note(root); + } catch { + return { ok: false, code: "corrupt", names: [] }; + } finally { + doc.destroy(); + } + return names.size === 0 + ? { ok: true } + : { ok: false, code: "unknown_schema", names: [...names] }; +} diff --git a/lib/doc-migrations/index.ts b/lib/doc-migrations/index.ts new file mode 100644 index 00000000..d45ad5f7 --- /dev/null +++ b/lib/doc-migrations/index.ts @@ -0,0 +1,70 @@ +/** + * Document schema migrations (desktop architecture §5.1, D-A5): pure Yjs + * forward edits that bring a stored document up to {@link DOC_SCHEMA}. + * + * - **On open**, `local-session` runs them after hydrate and before the + * editor binds (y-prosemirror deletes runs whose marks the schema doesn't + * know, `lib/yjs-mark-migration.ts`), inside one `doc.transact` with + * `MIGRATION_ORIGIN`, outside undo. Their update is appended as origin + * `migration` with `schemaTo`, which the storage engine writes to + * `docs.schema` in the same transaction. + * - **At import** (P2.8), the same functions run on `.tinydoc` and web + * downloads. + * + * The list starts empty: schema 0 is today's documents. Each entry raises + * `DOC_SCHEMA` by one. A document whose schema is above `DOC_SCHEMA` was + * written by a newer app and opens read-only (`doc_too_new`). + */ +import type * as Y from "yjs"; + +export type DocMigration = { + /** The schema this step produces: the previous step's `to` plus one. */ + to: number; + /** Edits `doc` in place. Runs inside the caller's transaction. */ + run(doc: Y.Doc): void; +}; + +export const DOC_MIGRATIONS: readonly DocMigration[] = []; + +/** The schema documents written by this build are at. */ +export const DOC_SCHEMA: number = DOC_MIGRATIONS.at(-1)?.to ?? 0; + +export type MigrationPlan = + | { kind: "current" } + | { kind: "too_new" } + | { kind: "migrate"; steps: readonly DocMigration[]; schemaTo: number }; + +/** What opening a document stored at `schema` needs. */ +export function planDocMigrations( + schema: number, + migrations: readonly DocMigration[] = DOC_MIGRATIONS, +): MigrationPlan { + const current = migrations.at(-1)?.to ?? 0; + if (!Number.isInteger(schema) || schema < 0) + throw new RangeError(`bad document schema ${schema}`); + if (schema > current) return { kind: "too_new" }; + if (schema === current) return { kind: "current" }; + const steps = migrations.filter((step) => step.to > schema); + steps.forEach((step, i) => { + if (step.to !== schema + i + 1) + throw new RangeError(`migration to ${step.to} is out of order`); + }); + return { kind: "migrate", steps, schemaTo: current }; +} + +/** + * Runs the steps a plan lists in one transaction under `origin`, so they + * produce one update. Returns the schema reached, or null when there was + * nothing to do. + */ +export function runDocMigrations( + doc: Y.Doc, + plan: MigrationPlan, + origin: unknown, +): number | null { + if (plan.kind !== "migrate") return null; + doc.transact(() => { + for (const step of plan.steps) step.run(doc); + }, origin); + return plan.schemaTo; +} diff --git a/tests/unit/lib/doc-migrations/doc-migrations.test.ts b/tests/unit/lib/doc-migrations/doc-migrations.test.ts new file mode 100644 index 00000000..f67f15a2 --- /dev/null +++ b/tests/unit/lib/doc-migrations/doc-migrations.test.ts @@ -0,0 +1,49 @@ +import { describe, expect, test } from "bun:test"; + +import * as Y from "yjs"; + +import { + DOC_MIGRATIONS, + DOC_SCHEMA, + type DocMigration, + planDocMigrations, + runDocMigrations, +} from "@/lib/doc-migrations"; + +const steps: DocMigration[] = [ + { to: 1, run: (doc) => doc.getMap("m").set("one", true) }, + { to: 2, run: (doc) => doc.getMap("m").set("two", true) }, +]; + +describe("document migrations", () => { + test("the list starts empty, so today's documents are current", () => { + expect(DOC_MIGRATIONS).toEqual([]); + expect(DOC_SCHEMA).toBe(0); + expect(planDocMigrations(0)).toEqual({ kind: "current" }); + }); + + test("a newer document is too new", () => { + expect(planDocMigrations(1)).toEqual({ kind: "too_new" }); + }); + + test("plans only the missing steps", () => { + const plan = planDocMigrations(1, steps); + expect(plan).toMatchObject({ kind: "migrate", schemaTo: 2 }); + expect(plan.kind === "migrate" && plan.steps.map((s) => s.to)).toEqual([2]); + }); + + test("refuses a gap in the list", () => { + expect(() => planDocMigrations(0, [{ to: 2, run: () => {} }])).toThrow(); + }); + + test("runs every step in one transaction under the given origin", () => { + const doc = new Y.Doc(); + const origin = Symbol("migration"); + const seen: unknown[] = []; + doc.on("update", (_u: Uint8Array, o: unknown) => seen.push(o)); + expect(runDocMigrations(doc, planDocMigrations(0, steps), origin)).toBe(2); + expect(seen).toEqual([origin]); + expect(doc.getMap("m").toJSON()).toEqual({ one: true, two: true }); + expect(runDocMigrations(doc, { kind: "current" }, origin)).toBeNull(); + }); +}); diff --git a/tests/unit/lib/doc-migrations/import-guard.test.ts b/tests/unit/lib/doc-migrations/import-guard.test.ts new file mode 100644 index 00000000..82678a37 --- /dev/null +++ b/tests/unit/lib/doc-migrations/import-guard.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, test } from "bun:test"; + +import * as Y from "yjs"; + +import { checkImport } from "@/lib/doc-migrations/import-guard"; + +function docWith(build: (doc: Y.Doc) => void): Uint8Array { + const doc = new Y.Doc(); + build(doc); + return Y.encodeStateAsUpdate(doc); +} + +function paragraph(text: string, attrs?: Record) { + const p = new Y.XmlElement("paragraph"); + const t = new Y.XmlText(); + t.insert(0, text, attrs); + p.insert(0, [t]); + return p; +} + +describe("checkImport", () => { + test("accepts a document the editor can open", () => { + const state = docWith((doc) => { + const heading = new Y.XmlElement("heading"); + heading.setAttribute("level", "1"); + doc + .getXmlFragment("default") + .insert(0, [heading, paragraph("hi", { bold: true })]); + }); + expect(checkImport(state)).toEqual({ ok: true }); + }); + + test("accepts the legacy link and code marks (the import migrations rename them)", () => { + const state = docWith((doc) => { + doc + .getXmlFragment("default") + .insert(0, [ + paragraph("x", { link: { href: "https://a.b" } }), + paragraph("y", { code: true }), + ]); + }); + expect(checkImport(state)).toEqual({ ok: true }); + }); + + test("refuses unknown nodes and marks, naming them", () => { + const state = docWith((doc) => { + doc + .getXmlFragment("default") + .insert(0, [ + new Y.XmlElement("marquee"), + paragraph("x", { blink: true }), + ]); + }); + expect(checkImport(state)).toEqual({ + ok: false, + code: "unknown_schema", + names: ["marquee", "blink"], + }); + }); + + test("refuses unknown roots", () => { + const state = docWith((doc) => { + doc.getXmlFragment("default").insert(0, [paragraph("x")]); + doc.getMap("payload").set("a", 1); + }); + expect(checkImport(state)).toMatchObject({ + ok: false, + code: "unknown_schema", + names: ["payload"], + }); + }); + + test("refuses oversized states before decoding them", () => { + const state = docWith((doc) => { + doc.getXmlFragment("default").insert(0, [paragraph("x".repeat(100))]); + }); + expect(checkImport(state, { maxBytes: 10 })).toEqual({ + ok: false, + code: "import_too_large", + names: [], + }); + }); + + test("refuses bytes that aren't a Yjs update", () => { + expect( + checkImport(new Uint8Array([255, 255, 255, 255, 1, 2, 3])), + ).toMatchObject({ + ok: false, + code: "corrupt", + }); + }); +}); From 6897139b81e7fcbc2b74944de040f99539fcbebf Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:29:25 +0330 Subject: [PATCH 04/19] fix(desktop): only the window holding a document may write it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The worker accepted appends and compactions from any window when no window held the document. It now requires the writer's own hold, as desktop architecture §7 requires of main, and adds claim() so main can replay its registry after a worker restart. Compaction is bounded by the document's own last row instead of the library-wide sequence. --- desktop/storage/engine/content.ts | 26 ++++++++++----- desktop/storage/engine/library.ts | 33 ++++++++++++++++--- tests/unit/desktop/storage/content.test.ts | 26 +++++++++++++-- tests/unit/desktop/storage/daily-copy.test.ts | 1 + .../desktop/storage/library-index.test.ts | 1 + 5 files changed, 72 insertions(+), 15 deletions(-) diff --git a/desktop/storage/engine/content.ts b/desktop/storage/engine/content.ts index d9e117db..73b16b59 100644 --- a/desktop/storage/engine/content.ts +++ b/desktop/storage/engine/content.ts @@ -55,13 +55,20 @@ function snapshotUpto(db: SqliteDriver, docId: string): number { return typeof value === "number" ? value : 0; } -/** The highest seq ever assigned in this library (0 before the first append). */ -function lastAssignedSeq(db: SqliteDriver): number { +/** + * The highest seq this document has: of its rows, or of the rows its + * snapshot replaced. A compaction can't claim to cover more than this. + */ +function lastDocSeq(db: SqliteDriver, docId: string): number { const value = scalar( db, - "SELECT seq FROM sqlite_sequence WHERE name = 'updates'", + "SELECT MAX(seq) FROM updates WHERE doc_id = ?", + docId, + ); + return Math.max( + typeof value === "number" ? value : 0, + snapshotUpto(db, docId), ); - return typeof value === "number" ? value : 0; } export function load(db: SqliteDriver, docId: string): LoadedContent { @@ -144,8 +151,8 @@ export function append( /** * Stores `state` as the snapshot and deletes rows `seq ≤ upToSeq`, in one * transaction (D-A3). A stale compaction (`upToSeq` below the stored - * snapshot's) changes nothing. An `upToSeq` past the last seq the library - * has assigned is refused: it would hide every later append. + * snapshot's) changes nothing. An `upToSeq` past the document's last row + * is refused: it could claim rows the renderer has never seen. */ export function compact( db: SqliteDriver, @@ -160,8 +167,11 @@ export function compact( transaction(db, () => { if (!docExists(db, docId)) throw new StorageError("not_found", "no such document"); - if (upToSeq > lastAssignedSeq(db)) - throw new StorageError("invalid_argument", "upToSeq is in the future"); + if (upToSeq > lastDocSeq(db, docId)) + throw new StorageError( + "invalid_argument", + "upToSeq is past the document's rows", + ); if (upToSeq < snapshotUpto(db, docId)) return; db.prepare( `INSERT INTO snapshots (doc_id, data, upto_seq) VALUES (?, ?, ?) diff --git a/desktop/storage/engine/library.ts b/desktop/storage/engine/library.ts index d291e1fb..3ed6f0a8 100644 --- a/desktop/storage/engine/library.ts +++ b/desktop/storage/engine/library.ts @@ -22,8 +22,9 @@ * * **Registry** (architecture §5.2, §6.2): a document is open in at most one * window (D-A3). `load` claims it for an owner (a window id) and rejects - * another owner with `already_open`; appends and compactions from another - * owner are `forbidden`; main's own content writes (purge, and P2.8's + * another owner with `already_open`; appends and compactions from any + * other owner are `forbidden` (and {@link Library.claim} re-registers holds + * after a worker restart); main's own content writes (purge, and P2.8's * import-into and restore) refuse an open document with `document_open`. */ import { cloudFolderOf } from "./cloud-folder"; @@ -265,10 +266,17 @@ export class Library { } /** Content writes from the renderer: refused while another owner holds the doc. */ + /** + * Content writes from the renderer: only the window holding the document + * may write it (architecture §7: "the registry must hold docId open for + * this window"). Main checks this first; the worker checks again. + */ private assertWriter(docId: string, owner: string) { - const holder = this.holders.get(docId); - if (holder !== undefined && holder !== owner) - throw new StorageError("forbidden", "document is open in another window"); + if (this.holders.get(docId) !== owner) + throw new StorageError( + "forbidden", + "this window has not opened the document", + ); } /** Main's own content writes: refused while any window holds the doc. */ @@ -324,6 +332,21 @@ export class Library { return { foreign: null }; } + /** + * Re-registers a hold without loading: after a worker restart, main + * replays its registry so the windows' resent batches are accepted. + */ + claim(docId: string, owner: string): void { + const holder = this.holders.get(docId); + if (holder !== undefined && holder !== owner) + throw new StorageError("already_open", "document is open elsewhere"); + if ( + scalar(this.conn, "SELECT 1 FROM docs WHERE id = ?", docId) === undefined + ) + throw new StorageError("not_found", "no such document"); + this.holders.set(docId, owner); + } + close(docId: string, owner: string): void { if (this.holders.get(docId) === owner) this.holders.delete(docId); } diff --git a/tests/unit/desktop/storage/content.test.ts b/tests/unit/desktop/storage/content.test.ts index d533c9d4..8295eac7 100644 --- a/tests/unit/desktop/storage/content.test.ts +++ b/tests/unit/desktop/storage/content.test.ts @@ -23,6 +23,7 @@ beforeEach(() => { clock = createClock(); library = openTestLibrary(temp.dir, createTestHost(clock)); docId = library.createDoc({ title: "Untitled" }).id; + library.load(docId, "w1"); }); afterEach(() => { library.shutdown(); @@ -89,6 +90,7 @@ describe("load and append", () => { test("seqs are global across documents", () => { const other = library.createDoc().id; + library.load(other, "w2"); const doc = new Y.Doc(); const a = library.append(docId, "w1", { origin: "local", @@ -159,7 +161,7 @@ describe("load and append", () => { codeOf(() => library.append(crypto.randomUUID(), "w1", { origin: "local", update }), ), - ).toBe("not_found"); + ).toBe("forbidden"); expect(codeOf(() => library.load(crypto.randomUUID(), "w1"))).toBe( "not_found", ); @@ -222,6 +224,7 @@ describe("compaction (D-A3)", () => { library.compact(docId, "w1", Y.encodeStateAsUpdate(doc), seq); library.shutdown(); library = openTestLibrary(temp.dir, createTestHost(clock)); + library.load(docId, "w1"); const next = library.append(docId, "w1", { origin: "local", update: edit(doc, "b"), @@ -246,7 +249,7 @@ describe("compaction (D-A3)", () => { expect(textOf(library.load(docId, "w1"))).toBe("ab"); }); - test("refuses an upToSeq past the last assigned seq", () => { + test("refuses an upToSeq past the document's last row", () => { const doc = new Y.Doc(); const seq = library.append(docId, "w1", { origin: "local", @@ -392,3 +395,22 @@ describe("compaction property (architecture §11)", () => { }); } }); + +describe("claim (registry replay after a worker restart)", () => { + test("lets the holding window write again without a load", () => { + library.shutdown(); + library = openTestLibrary(temp.dir, createTestHost(clock)); + const update = edit(new Y.Doc(), "x"); + expect( + codeOf(() => library.append(docId, "w1", { origin: "local", update })), + ).toBe("forbidden"); + library.claim(docId, "w1"); + expect( + library.append(docId, "w1", { origin: "local", update }), + ).toBeGreaterThan(0); + expect(codeOf(() => library.claim(docId, "w2"))).toBe("already_open"); + expect(codeOf(() => library.claim(crypto.randomUUID(), "w2"))).toBe( + "not_found", + ); + }); +}); diff --git a/tests/unit/desktop/storage/daily-copy.test.ts b/tests/unit/desktop/storage/daily-copy.test.ts index 766f2d5b..fc4ba1a3 100644 --- a/tests/unit/desktop/storage/daily-copy.test.ts +++ b/tests/unit/desktop/storage/daily-copy.test.ts @@ -29,6 +29,7 @@ afterEach(() => { }); function write(docId: string, text: string) { + if (!library.isOpen(docId)) library.load(docId, "w1"); const doc = new Y.Doc(); doc.getText("t").insert(0, text); library.append(docId, "w1", { diff --git a/tests/unit/desktop/storage/library-index.test.ts b/tests/unit/desktop/storage/library-index.test.ts index 72774bbf..3478c473 100644 --- a/tests/unit/desktop/storage/library-index.test.ts +++ b/tests/unit/desktop/storage/library-index.test.ts @@ -152,6 +152,7 @@ describe("Trash and purge", () => { const ydoc = new Y.Doc(); ydoc.getText("t").insert(0, "x"); const update = Y.encodeStateAsUpdate(ydoc); + library.load(doc.id, "w1"); const seq = library.append(doc.id, "w1", { origin: "local", update }); library.compact(doc.id, "w1", update, seq); library.append(doc.id, "w1", { origin: "local", update }); From 57ed7ba620475626b366d0baaca7dafc926f1112 Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:29:25 +0330 Subject: [PATCH 05/19] feat(desktop): add the storage worker entry and its main-side client worker.ts runs the engine on node:sqlite in a worker_thread (D-A1) and answers one request at a time, so flush and shutdown run after every write sent before them. Replies are Result envelopes and bytes are transferred. Idle jobs run between requests, and index writes emit index_changed. StorageClient starts the worker, waits for ready, rejects pending calls with worker_restarting if the worker dies, and ends with the quit flush's shutdown (terminated after 2 s). createRpcDocStore turns those replies into the DocStore port, for IpcDocStore and the tests. Engine DocStore cases run in-process on bun:sqlite and, framework-free, in Electron. --- desktop/storage/client.ts | 177 ++++++++++++++ desktop/storage/protocol.ts | 231 ++++++++++++++++++ desktop/storage/worker.ts | 106 ++++++++ lib/platform/desktop/rpc-doc-store.ts | 68 ++++++ tests/helpers/engine-doc-store.ts | 48 ++++ tests/helpers/storage-doc-store-cases.ts | 186 ++++++++++++++ .../desktop/storage/doc-store-engine.test.ts | 68 ++++++ 7 files changed, 884 insertions(+) create mode 100644 desktop/storage/client.ts create mode 100644 desktop/storage/protocol.ts create mode 100644 desktop/storage/worker.ts create mode 100644 lib/platform/desktop/rpc-doc-store.ts create mode 100644 tests/helpers/engine-doc-store.ts create mode 100644 tests/helpers/storage-doc-store-cases.ts create mode 100644 tests/unit/desktop/storage/doc-store-engine.test.ts diff --git a/desktop/storage/client.ts b/desktop/storage/client.ts new file mode 100644 index 00000000..4373bde7 --- /dev/null +++ b/desktop/storage/client.ts @@ -0,0 +1,177 @@ +/** + * Main's handle on the storage worker (desktop architecture §2, §6.3). + * `doc-handlers` (P2.5) forwards validated `doc.*` and `index.*` calls + * through {@link StorageClient.call}; the quit coordinator ends with + * {@link StorageClient.shutdown}. + * + * Bytes in a request are transferred, not copied. If the worker dies, every + * pending call rejects with `worker_restarting`, which the renderer's + * batcher retries (resending an append is harmless). + */ +import { Worker } from "node:worker_threads"; + +import type { StorageErrorEnvelope } from "./engine/errors"; +import type { LibraryStatus } from "./engine/library"; +import { + type StorageMessage, + type StorageOp, + type StorageOps, + type StorageWorkerOptions, + transferablesOf, +} from "./protocol"; + +export class StorageCallError extends Error { + readonly code: StorageErrorEnvelope["code"]; + readonly retryable: boolean; + constructor(envelope: StorageErrorEnvelope) { + super(envelope.message); + this.name = "StorageCallError"; + this.code = envelope.code; + this.retryable = envelope.retryable; + } +} + +type Pending = { resolve(value: unknown): void; reject(error: Error): void }; + +export type StorageClientEvents = { + onIndexChanged?: (rev: number) => void; + /** The worker exited without being asked to (crash). */ + onExit?: (code: number) => void; +}; + +export class StorageClient { + readonly status: LibraryStatus; + /** Worker spawn to `ready`, and the open alone inside it. */ + readonly timings: { readyMs: number; openMs: number }; + private readonly worker: Worker; + private readonly pending = new Map(); + private nextId = 1; + private closing = false; + + private constructor( + worker: Worker, + status: LibraryStatus, + timings: { readyMs: number; openMs: number }, + events: StorageClientEvents, + ) { + this.worker = worker; + this.status = status; + this.timings = timings; + worker.on("message", (message: StorageMessage) => { + if (message.type === "reply") { + const pending = this.pending.get(message.id); + if (!pending) return; + this.pending.delete(message.id); + if (message.result.ok) pending.resolve(message.result.value); + else pending.reject(new StorageCallError(message.result.error)); + } else if (message.type === "index_changed") { + events.onIndexChanged?.(message.rev); + } + }); + worker.on("exit", (code) => { + this.failPending(); + if (!this.closing) events.onExit?.(code); + }); + } + + /** + * Starts the worker and waits for `ready`. Rejects with the open error + * (`cloud_folder`, `locked_elsewhere`, `corrupt`…) if the library can't + * open; a read-only library still resolves, with `status.readOnly` set. + */ + static start( + workerPath: string, + options: StorageWorkerOptions, + events: StorageClientEvents = {}, + ): Promise { + const started = performance.now(); + const worker = new Worker(workerPath, { workerData: options }); + return new Promise((resolve, reject) => { + const onError = (error: Error) => reject(error); + worker.once("error", onError); + worker.once("message", (message: StorageMessage) => { + worker.off("error", onError); + if (message.type === "ready") { + resolve( + new StorageClient( + worker, + message.status, + { + readyMs: performance.now() - started, + openMs: message.openMs, + }, + events, + ), + ); + } else if (message.type === "open_failed") { + void worker.terminate(); + reject(new StorageCallError(message.error)); + } else { + void worker.terminate(); + reject(new Error(`unexpected first message ${message.type}`)); + } + }); + }); + } + + call( + op: K, + args: StorageOps[K]["args"], + ): Promise { + if (this.closing && op !== "shutdown") + return Promise.reject( + new StorageCallError({ + code: "library_unavailable", + retryable: false, + message: "the library is closing", + }), + ); + const id = this.nextId++; + return new Promise((resolve, reject) => { + this.pending.set(id, { + resolve: resolve as (value: unknown) => void, + reject, + }); + this.worker.postMessage({ id, op, args }, transferablesOf(args)); + }); + } + + /** + * The quit flush's last step (architecture §8.2): every call already sent + * commits first, then the worker checkpoints, clears `unclean_exit` and + * closes. After `timeoutMs` it is terminated; WAL keeps every committed + * row, and the next open runs `quick_check`. + */ + async shutdown(timeoutMs = 2000): Promise<"clean" | "terminated"> { + if (this.closing) return "clean"; + this.closing = true; + const exited = new Promise((resolve) => + this.worker.once("exit", () => resolve()), + ); + let timer: ReturnType | undefined; + const timeout = new Promise<"terminated">((resolve) => { + timer = setTimeout(() => resolve("terminated"), timeoutMs); + }); + const clean = this.call("shutdown", {}) + .then(() => exited) + .then(() => "clean" as const); + const outcome = await Promise.race([clean, timeout]).catch( + () => "terminated" as const, + ); + clearTimeout(timer); + if (outcome === "terminated") await this.worker.terminate(); + return outcome; + } + + private failPending() { + for (const pending of this.pending.values()) + pending.reject( + new StorageCallError({ + code: "worker_restarting", + retryable: true, + message: "the storage worker stopped", + }), + ); + this.pending.clear(); + } +} diff --git a/desktop/storage/protocol.ts b/desktop/storage/protocol.ts new file mode 100644 index 00000000..f49d8cd4 --- /dev/null +++ b/desktop/storage/protocol.ts @@ -0,0 +1,231 @@ +/** + * Messages between main and the storage worker (desktop architecture §2, + * §7). Main has already validated every request at the IPC boundary; the + * worker trusts main. Nothing is thrown across: every reply is a `Result`. + * + * Bytes travel as `Uint8Array`s whose buffers are transferred, so a + * document crosses each boundary with one copy. + * + * No `node:*` imports: the dev storage host and unit tests dispatch to a + * `Library` in-process with {@link dispatch}. + */ +import type { AppendInput, LoadedContent } from "./engine/content"; +import { type StorageErrorEnvelope, toEnvelope } from "./engine/errors"; +import type { Library, LibraryStatus } from "./engine/library"; +import type { + CreateDocInput, + CreatedDoc, + SidebarSnapshot, +} from "./engine/library-index"; + +/** Every request: `args` in, `result` out. */ +export type StorageOps = { + load: { args: { docId: string; owner: string }; result: LoadedContent }; + append: { + args: { docId: string; owner: string; record: AppendInput }; + result: number; + }; + compact: { + args: { docId: string; owner: string; state: Uint8Array; upToSeq: number }; + result: { foreign: null }; + }; + /** Main replays its registry after a worker restart. */ + claim: { args: { docId: string; owner: string }; result: null }; + close: { args: { docId: string; owner: string }; result: null }; + releaseOwner: { args: { owner: string }; result: null }; + /** Resolves after every request sent before it: all durable. */ + flush: { args: Record; result: null }; + stateOf: { args: { docId: string }; result: Uint8Array }; + sidebar: { args: Record; result: SidebarSnapshot }; + createDoc: { args: CreateDocInput; result: CreatedDoc }; + createFolder: { + args: { name: string; parentId?: string | null }; + result: { id: string; rev: number }; + }; + renameDoc: { args: { id: string; title: string }; result: number }; + moveDoc: { + args: { id: string; folderId: string | null; sortKey: string }; + result: number; + }; + trashDoc: { args: { id: string; group?: string | null }; result: number }; + restoreDoc: { args: { id: string }; result: number }; + purgeDoc: { args: { id: string }; result: number }; + createTag: { + args: { name: string; color?: string | null }; + result: { id: string; rev: number }; + }; + setDocTags: { args: { docId: string; tagIds: string[] }; result: number }; + listDailyCopies: { args: Record; result: string[] }; + readDailyCopy: { args: { date: string; docId: string }; result: Uint8Array }; + runIdleJobs: { + args: Record; + result: { dailyCopy: string | null; purged: number }; + }; + stats: { args: Record; result: ReturnType }; + /** The quit flush's last step: checkpoint, clear `unclean_exit`, close. */ + shutdown: { args: Record; result: null }; +}; + +export type StorageOp = keyof StorageOps; + +export type Result = + | { ok: true; value: T } + | { ok: false; error: StorageErrorEnvelope }; + +/** main → worker */ +export type StorageRequest = { + [K in StorageOp]: { id: number; op: K; args: StorageOps[K]["args"] }; +}[StorageOp]; + +/** worker → main */ +export type StorageMessage = + | { type: "ready"; status: LibraryStatus; openMs: number } + | { type: "open_failed"; error: StorageErrorEnvelope } + | { type: "reply"; id: number; result: Result } + /** Metadata changed (architecture §7 `index.changed`); main fans it out. */ + | { type: "index_changed"; rev: number }; + +/** Options the worker is started with (`workerData`). */ +export type StorageWorkerOptions = { + dir: string; + fullfsync?: boolean; + synchronous?: "FULL" | "OFF"; + /** Durability soak only: protocol lines on fd 1. */ + trace?: boolean; + /** How often the worker looks for idle work, in ms (default 5000). */ + idlePollMs?: number; +}; + +/** Ops whose result is the new `rev`: each one emits `index_changed`. */ +export const REV_OPS: ReadonlySet = new Set([ + "renameDoc", + "moveDoc", + "trashDoc", + "restoreDoc", + "purgeDoc", + "setDocTags", +]); + +/** Ops whose result carries `rev` in an object. */ +const CREATE_OPS: ReadonlySet = new Set([ + "createDoc", + "createFolder", + "createTag", +]); + +function run(library: Library, request: StorageRequest): unknown { + switch (request.op) { + case "load": + return library.load(request.args.docId, request.args.owner); + case "append": + return library.append( + request.args.docId, + request.args.owner, + request.args.record, + ); + case "compact": + return library.compact( + request.args.docId, + request.args.owner, + request.args.state, + request.args.upToSeq, + ); + case "claim": + library.claim(request.args.docId, request.args.owner); + return null; + case "close": + library.close(request.args.docId, request.args.owner); + return null; + case "releaseOwner": + library.releaseOwner(request.args.owner); + return null; + case "flush": + return null; + case "stateOf": + return library.stateOf(request.args.docId); + case "sidebar": + return library.sidebar(); + case "createDoc": + return library.createDoc(request.args); + case "createFolder": + return library.createFolder(request.args); + case "renameDoc": + return library.renameDoc(request.args.id, request.args.title); + case "moveDoc": + return library.moveDoc( + request.args.id, + request.args.folderId, + request.args.sortKey, + ); + case "trashDoc": + return library.trashDoc(request.args.id, request.args.group ?? null); + case "restoreDoc": + return library.restoreDoc(request.args.id); + case "purgeDoc": + return library.purgeDoc(request.args.id); + case "createTag": + return library.createTag(request.args.name, request.args.color ?? null); + case "setDocTags": + return library.setDocTags(request.args.docId, request.args.tagIds); + case "listDailyCopies": + return library.listDailyCopies(); + case "readDailyCopy": + return library.readDailyCopy(request.args.date, request.args.docId); + case "runIdleJobs": + return library.runIdleJobs(); + case "stats": + return library.stats(); + case "shutdown": + library.shutdown(); + return null; + } +} + +/** + * Runs one request. Returns the reply, and the new `rev` when the request + * changed metadata (the caller emits `index_changed`). + */ +export function dispatch( + library: Library, + request: StorageRequest, +): { result: Result; rev: number | null } { + try { + const value = run(library, request); + let rev: number | null = null; + if (REV_OPS.has(request.op)) rev = value as number; + else if (CREATE_OPS.has(request.op)) rev = (value as { rev: number }).rev; + return { result: { ok: true, value }, rev }; + } catch (error) { + return { result: { ok: false, error: toEnvelope(error) }, rev: null }; + } +} + +/** The `ArrayBuffer`s in a value that can move instead of being copied. */ +export function transferablesOf(value: unknown): ArrayBuffer[] { + const found: ArrayBuffer[] = []; + const seen = new Set(); + const visit = (item: unknown, depth: number) => { + if (depth > 3 || item === null || typeof item !== "object") return; + if (item instanceof Uint8Array) { + const { buffer } = item; + // Only a buffer the view owns whole; never a pooled or shared one. + if ( + buffer instanceof ArrayBuffer && + item.byteOffset === 0 && + item.byteLength === buffer.byteLength && + !seen.has(buffer) + ) { + seen.add(buffer); + found.push(buffer); + } + return; + } + if (Array.isArray(item)) { + for (const entry of item) visit(entry, depth + 1); + return; + } + for (const entry of Object.values(item)) visit(entry, depth + 1); + }; + visit(value, 0); + return found; +} diff --git a/desktop/storage/worker.ts b/desktop/storage/worker.ts new file mode 100644 index 00000000..da84107d --- /dev/null +++ b/desktop/storage/worker.ts @@ -0,0 +1,106 @@ +/** + * The storage worker's entry (D-A1): a `worker_thread` inside Electron + * main, and the only SQLite connection to the library. Main starts it in + * parallel with the first window and sends it validated requests; it + * answers them one at a time, in order, so a `flush` or `shutdown` queued + * behind appends runs after they have committed. + * + * Messages: `ready` (or `open_failed`) once, then one `reply` per request, + * and `index_changed` after a metadata write. Idle jobs (daily copy, Trash) + * run between requests after 30 s without a write. + * + * With `trace`, the engine's protocol lines (`BATCH_BEGIN`, `ACK `, + * `COMPACT_BEGIN`, `COMPACT_ACK`) go straight to fd 1 with `writeSync`, so + * the durability soak knows what committed before it kills the process. + */ +import { writeSync } from "node:fs"; +import { parentPort, workerData } from "node:worker_threads"; + +import { openNodeSqlite } from "./drivers/node-sqlite"; +import { toEnvelope } from "./engine/errors"; +import { Library } from "./engine/library"; +import { createNodeHost } from "./node-host"; +import { + dispatch, + type StorageMessage, + type StorageRequest, + type StorageWorkerOptions, + transferablesOf, +} from "./protocol"; + +const port = parentPort; +if (!port) throw new Error("the storage worker must run in a worker_thread"); + +const options = workerData as StorageWorkerOptions; + +const post = (message: StorageMessage) => { + port.postMessage( + message, + message.type === "reply" && message.result.ok + ? transferablesOf(message.result.value) + : [], + ); +}; + +const line = (text: string) => { + writeSync(1, `${text}\n`); +}; + +let library: Library | null = null; +const started = performance.now(); +try { + library = Library.open({ + dir: options.dir, + host: createNodeHost(), + openSqlite: openNodeSqlite, + fullfsync: options.fullfsync, + synchronous: options.synchronous, + trace: options.trace ? line : undefined, + }); + post({ + type: "ready", + status: library.status, + openMs: performance.now() - started, + }); +} catch (error) { + post({ type: "open_failed", error: toEnvelope(error) }); +} + +let idleTimer: ReturnType | null = null; +if (library) { + const open = library; + idleTimer = setInterval(() => { + try { + open.runIdleJobs(); + } catch { + // A failed daily copy is reported through stats; the library is fine. + } + }, options.idlePollMs ?? 5000); + idleTimer.unref(); +} + +port.on("message", (request: StorageRequest) => { + if (!library) { + post({ + type: "reply", + id: request.id, + result: { + ok: false, + error: { + code: "library_unavailable", + retryable: true, + message: "the library did not open", + }, + }, + }); + return; + } + const { result, rev } = dispatch(library, request); + post({ type: "reply", id: request.id, result }); + if (rev !== null) post({ type: "index_changed", rev }); + if (request.op === "shutdown") { + if (idleTimer) clearInterval(idleTimer); + library = null; + port.close(); + } +}); diff --git a/lib/platform/desktop/rpc-doc-store.ts b/lib/platform/desktop/rpc-doc-store.ts new file mode 100644 index 00000000..5ba12925 --- /dev/null +++ b/lib/platform/desktop/rpc-doc-store.ts @@ -0,0 +1,68 @@ +/** + * A `DocStore` over request/reply calls to the storage worker. `IpcDocStore` + * (P2.5) passes a `call` that goes through `window.tinydocy`; tests and the + * Electron soak pass one that goes to a `StorageClient` (adding the window's + * owner id) or dispatches in-process. + * + * A rejection that carries a storage `code` becomes a `DocStoreError`; + * anything else (a dead transport) becomes `library_unavailable`, which is + * retryable, so the batcher keeps the batch and retries it in order. + */ +import { + type AppendRecord, + type CompactResult, + DOC_STORE_ERROR_CODES, + type DocStore, + DocStoreError, + type DocStoreErrorCode, + type LoadedDoc, +} from "@/lib/platform/doc-store"; + +export type DocStoreCalls = { + load: [{ docId: string }, LoadedDoc]; + append: [{ docId: string; record: AppendRecord }, number]; + compact: [ + { docId: string; state: Uint8Array; upToSeq: number }, + CompactResult, + ]; + close: [{ docId: string }, null]; + flush: [Record, null]; +}; + +export type DocStoreCall = ( + op: K, + args: DocStoreCalls[K][0], +) => Promise; + +const CODES: ReadonlySet = new Set(DOC_STORE_ERROR_CODES); + +export function toDocStoreError(error: unknown): DocStoreError { + if (error instanceof DocStoreError) return error; + const code = (error as { code?: unknown } | null)?.code; + const message = error instanceof Error ? error.message : String(error); + if (typeof code === "string" && CODES.has(code)) + return new DocStoreError(code as DocStoreErrorCode, message); + return new DocStoreError("library_unavailable", message); +} + +export function createRpcDocStore(call: DocStoreCall): DocStore { + const wrap = async (run: () => Promise): Promise => { + try { + return await run(); + } catch (error) { + throw toDocStoreError(error); + } + }; + return { + load: (docId) => wrap(() => call("load", { docId })), + append: (docId, record) => wrap(() => call("append", { docId, record })), + compact: (docId, state, upToSeq) => + wrap(() => call("compact", { docId, state, upToSeq })), + close: async (docId) => { + await wrap(() => call("close", { docId })); + }, + flush: async () => { + await wrap(() => call("flush", {})); + }, + }; +} diff --git a/tests/helpers/engine-doc-store.ts b/tests/helpers/engine-doc-store.ts new file mode 100644 index 00000000..c3f0d345 --- /dev/null +++ b/tests/helpers/engine-doc-store.ts @@ -0,0 +1,48 @@ +/** + * `DocStore`s over a real storage engine, in-process on `bun:sqlite`: what + * the shared contract suite runs against on every PR (architecture §11). + * Each store is one window (`owner`); two stores over one library model two + * windows, for `already_open`. + * + * Requests go through the worker's own `dispatch`, so the in-process path + * and the Electron worker share every line but the transport. Bytes are + * copied on the way in, as IPC would. + */ +import type { Library } from "@/desktop/storage/engine/library"; +import { dispatch, type StorageRequest } from "@/desktop/storage/protocol"; +import { + createRpcDocStore, + type DocStoreCall, +} from "@/lib/platform/desktop/rpc-doc-store"; +import type { DocStore } from "@/lib/platform/doc-store"; + +function copyBytes(value: T): T { + if (value instanceof Uint8Array) return value.slice() as T; + if (Array.isArray(value)) return value.map(copyBytes) as T; + if (value && typeof value === "object") + return Object.fromEntries( + Object.entries(value).map(([k, v]) => [k, copyBytes(v)]), + ) as T; + return value; +} + +let nextId = 1; + +export function createEngineDocStore( + library: Library, + owner: string, +): DocStore { + const call: DocStoreCall = async (op, args) => { + const request = { + id: nextId++, + op, + args: op === "flush" ? {} : { ...copyBytes(args), owner }, + } as StorageRequest; + // A microtask boundary, as a real transport has. + await Promise.resolve(); + const { result } = dispatch(library, request); + if (!result.ok) throw result.error; + return copyBytes(result.value) as never; + }; + return createRpcDocStore(call); +} diff --git a/tests/helpers/storage-doc-store-cases.ts b/tests/helpers/storage-doc-store-cases.ts new file mode 100644 index 00000000..fe406882 --- /dev/null +++ b/tests/helpers/storage-doc-store-cases.ts @@ -0,0 +1,186 @@ +/** + * `DocStore` cases for the desktop storage engine, written against the port + * only. Framework-free (`node:assert`), so `bun test` runs them on + * `bun:sqlite` and the Electron soak harness runs them on `node:sqlite` in + * the real storage worker. P2.1's shared contract suite joins these once it + * lands; these stay for the engine-specific rules (a library-wide seq, + * `not_found` for a document that was never created). + */ +import assert from "node:assert/strict"; + +import * as Y from "yjs"; + +import { type DocStore, DocStoreError } from "@/lib/platform/doc-store"; + +export type StoreCaseContext = { + /** One window's store; the same `owner` returns the same window. */ + store(owner: string): DocStore; + /** Creates a document through the index and returns its id. */ + createDoc(): Promise; +}; + +export type StoreCase = { + name: string; + run(context: StoreCaseContext): Promise; +}; + +function edit(doc: Y.Doc, text: string): Uint8Array { + let update: Uint8Array = new Uint8Array(); + doc.once("update", (u: Uint8Array) => { + update = u; + }); + const t = doc.getText("t"); + t.insert(t.length, text); + return update; +} + +function textOf(loaded: { + snapshot: Uint8Array | null; + updates: Uint8Array[]; +}) { + const doc = new Y.Doc(); + if (loaded.snapshot) Y.applyUpdate(doc, loaded.snapshot); + for (const update of loaded.updates) Y.applyUpdate(doc, update); + return doc.getText("t").toString(); +} + +async function rejection(promise: Promise): Promise { + try { + await promise; + } catch (error) { + assert.ok(error instanceof DocStoreError, `not a DocStoreError: ${error}`); + return error; + } + assert.fail("expected a rejection"); +} + +export const STORAGE_DOC_STORE_CASES: StoreCase[] = [ + { + name: "a new document loads empty", + async run({ store, createDoc }) { + const id = await createDoc(); + assert.deepEqual(await store("w1").load(id), { + snapshot: null, + updates: [], + maxSeq: 0, + schema: 0, + }); + }, + }, + { + name: "appends load back in order with increasing seqs", + async run({ store, createDoc }) { + const id = await createDoc(); + const w1 = store("w1"); + await w1.load(id); + const doc = new Y.Doc(); + const a = await w1.append(id, { + origin: "local", + update: edit(doc, "a"), + }); + const b = await w1.append(id, { + origin: "local", + update: edit(doc, "b"), + }); + assert.ok(b > a); + const loaded = await w1.load(id); + assert.equal(loaded.maxSeq, b); + assert.equal(textOf(loaded), "ab"); + }, + }, + { + name: "a resent append changes nothing", + async run({ store, createDoc }) { + const id = await createDoc(); + const w1 = store("w1"); + const update = edit(new Y.Doc(), "once"); + await w1.load(id); + await w1.append(id, { origin: "local", update }); + await w1.append(id, { origin: "local", update }); + assert.equal(textOf(await w1.load(id)), "once"); + }, + }, + { + name: "compaction folds covered rows into the snapshot and returns no foreign state", + async run({ store, createDoc }) { + const id = await createDoc(); + const w1 = store("w1"); + await w1.load(id); + const doc = new Y.Doc(); + await w1.append(id, { origin: "local", update: edit(doc, "a") }); + const covered = await w1.append(id, { + origin: "local", + update: edit(doc, "b"), + }); + const state = Y.encodeStateAsUpdate(doc); + const late = await w1.append(id, { + origin: "local", + update: edit(doc, "c"), + }); + assert.deepEqual(await w1.compact(id, state, covered), { foreign: null }); + const loaded = await w1.load(id); + assert.ok(loaded.snapshot); + assert.equal(loaded.updates.length, 1); + assert.equal(loaded.maxSeq, late); + assert.equal(textOf(loaded), "abc"); + }, + }, + { + name: "a second window gets already_open until the first closes", + async run({ store, createDoc }) { + const id = await createDoc(); + await store("w1").load(id); + const error = await rejection(store("w2").load(id)); + assert.equal(error.code, "already_open"); + assert.equal(error.retryable, false); + await store("w1").close(id); + assert.equal((await store("w2").load(id)).maxSeq, 0); + }, + }, + { + name: "flush resolves after every append already issued", + async run({ store, createDoc }) { + const id = await createDoc(); + const w1 = store("w1"); + await w1.load(id); + const doc = new Y.Doc(); + const sent = ["a", "b", "c"].map((text) => + w1.append(id, { origin: "local", update: edit(doc, text) }), + ); + await w1.flush(); + const seqs = await Promise.all(sent); + assert.equal((await w1.load(id)).maxSeq, Math.max(...seqs)); + }, + }, + { + name: "a rejected append writes nothing", + async run({ store, createDoc }) { + const id = await createDoc(); + const w1 = store("w1"); + await w1.load(id); + const error = await rejection( + w1.append(id, { origin: "local", update: new Uint8Array() }), + ); + assert.equal(error.code, "invalid_argument"); + assert.deepEqual((await w1.load(id)).updates, []); + }, + }, + { + name: "a window that hasn't opened a document can't write it", + async run({ store, createDoc }) { + const id = await createDoc(); + const update = edit(new Y.Doc(), "x"); + const error = await rejection( + store("w1").append(id, { origin: "local", update }), + ); + assert.equal(error.code, "forbidden"); + }, + }, + { + name: "an unknown document is not_found", + async run({ store }) { + const error = await rejection(store("w1").load(crypto.randomUUID())); + assert.equal(error.code, "not_found"); + }, + }, +]; diff --git a/tests/unit/desktop/storage/doc-store-engine.test.ts b/tests/unit/desktop/storage/doc-store-engine.test.ts new file mode 100644 index 00000000..124f11e6 --- /dev/null +++ b/tests/unit/desktop/storage/doc-store-engine.test.ts @@ -0,0 +1,68 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; + +import type { Library } from "@/desktop/storage/engine/library"; +import { dispatch } from "@/desktop/storage/protocol"; +import type { DocStore } from "@/lib/platform/doc-store"; +import { createEngineDocStore } from "@/tests/helpers/engine-doc-store"; +import { STORAGE_DOC_STORE_CASES } from "@/tests/helpers/storage-doc-store-cases"; +import { + createClock, + createTempDir, + createTestHost, + openTestLibrary, +} from "@/tests/helpers/storage-library"; + +let temp: ReturnType; +let library: Library; + +beforeEach(() => { + temp = createTempDir(); + library = openTestLibrary(temp.dir, createTestHost(createClock())); +}); +afterEach(() => { + library.shutdown(); + temp.cleanUp(); +}); + +describe("DocStore over the engine (bun:sqlite)", () => { + for (const testCase of STORAGE_DOC_STORE_CASES) + test(testCase.name, async () => { + const stores = new Map(); + await testCase.run({ + store(owner) { + let store = stores.get(owner); + if (!store) { + store = createEngineDocStore(library, owner); + stores.set(owner, store); + } + return store; + }, + createDoc: async () => library.createDoc().id, + }); + }); +}); + +describe("dispatch", () => { + test("a metadata write reports the new rev", () => { + const { result, rev } = dispatch(library, { + id: 1, + op: "createFolder", + args: { name: "A" }, + }); + expect(result.ok).toBe(true); + expect(rev).toBe(library.sidebar().rev); + }); + + test("a failure comes back as an envelope, never a throw", () => { + const { result, rev } = dispatch(library, { + id: 2, + op: "load", + args: { docId: "nope", owner: "w1" }, + }); + expect(result).toMatchObject({ + ok: false, + error: { code: "not_found", retryable: false }, + }); + expect(rev).toBeNull(); + }); +}); From 02af525b3e69d20a8c8ca2611983ad77f3424531 Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:29:58 +0330 Subject: [PATCH 06/19] test(ci): enforce the storage engine and worker import boundaries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The first rows of desktop architecture §3.3's import-boundaries test: the engine imports no electron, node:*, bun:*, React or Tiptap and stays in its folder; the worker reaches lib/ only through the allowlist and never bun:*; lib/platform/desktop never imports desktop/{main,preload,storage}. --- tests/unit/ci/import-boundaries.test.ts | 129 ++++++++++++++++++++++++ 1 file changed, 129 insertions(+) create mode 100644 tests/unit/ci/import-boundaries.test.ts diff --git a/tests/unit/ci/import-boundaries.test.ts b/tests/unit/ci/import-boundaries.test.ts new file mode 100644 index 00000000..63e28f0d --- /dev/null +++ b/tests/unit/ci/import-boundaries.test.ts @@ -0,0 +1,129 @@ +/** + * Import boundaries (desktop architecture §3.3). Imports are read with + * `Bun.Transpiler().scanImports()` (type-only imports are dropped, as at + * runtime) and followed transitively through relative and `@/` paths. + * + * Rows checked so far: the storage engine, the storage worker, and + * `lib/platform/desktop`. P2.5 adds the main, preload and renderer rows. + */ +import { describe, expect, test } from "bun:test"; +import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { dirname, join, relative, resolve } from "node:path"; + +const ROOT = resolve(import.meta.dir, "../../.."); +const transpilers = { + ts: new Bun.Transpiler({ loader: "ts" }), + tsx: new Bun.Transpiler({ loader: "tsx" }), +}; + +function filesUnder(dir: string): string[] { + const out: string[] = []; + for (const entry of readdirSync(dir)) { + const path = join(dir, entry); + if (statSync(path).isDirectory()) out.push(...filesUnder(path)); + else if (/\.tsx?$/.test(entry) && !entry.endsWith(".d.ts")) out.push(path); + } + return out; +} + +function resolveLocal(from: string, specifier: string): string | null { + let base: string; + if (specifier.startsWith("@/")) base = join(ROOT, specifier.slice(2)); + else if (specifier.startsWith(".")) base = resolve(dirname(from), specifier); + else return null; + for (const candidate of [ + base, + `${base}.ts`, + `${base}.tsx`, + join(base, "index.ts"), + join(base, "index.tsx"), + ]) + if (existsSync(candidate) && statSync(candidate).isFile()) return candidate; + throw new Error(`cannot resolve ${specifier} from ${relative(ROOT, from)}`); +} + +/** Every module `entry` reaches at runtime: local files and bare specifiers. */ +function reach(entries: string[]): { + files: Set; + bare: Map; +} { + const files = new Set(); + const bare = new Map(); + const queue = [...entries]; + while (queue.length > 0) { + const file = queue.pop() as string; + if (files.has(file)) continue; + files.add(file); + const transpiler = file.endsWith(".tsx") ? transpilers.tsx : transpilers.ts; + for (const { path } of transpiler.scanImports(readFileSync(file, "utf8"))) { + const local = resolveLocal(file, path); + if (local) queue.push(local); + else if (!bare.has(path)) bare.set(path, relative(ROOT, file)); + } + } + return { files, bare }; +} + +const rel = (path: string) => relative(ROOT, path).split("\\").join("/"); + +describe("desktop/storage/engine", () => { + const { files, bare } = reach( + filesUnder(join(ROOT, "desktop/storage/engine")), + ); + + test("imports no electron, node:*, bun:*, React or Tiptap", () => { + const banned = [...bare.entries()].filter( + ([spec]) => + /^(electron|node:|bun:|react|react-dom|@tiptap\/)/.test(spec) || + /^(fs|path|os|worker_threads|child_process)$/.test(spec), + ); + expect(banned).toEqual([]); + }); + + test("stays inside desktop/storage/engine", () => { + const outside = [...files] + .map(rel) + .filter((file) => !file.startsWith("desktop/storage/engine/")); + expect(outside).toEqual([]); + }); +}); + +describe("desktop/storage", () => { + // Architecture §3.3: main and storage may reach only these parts of lib/. + const LIB_ALLOWED = [ + /^lib\/security\/content-security-policy\.ts$/, + /^lib\/doc-migrations\//, + /^lib\/prefs\/schema\.ts$/, + /^lib\/comments\/validate\.ts$/, + /^lib\/assets\//, + // Pure name lists the import guard checks against (amended in §3.3). + /^lib\/schema-names\.ts$/, + ]; + + test("reaches lib/ only through the allowlist, and never React", () => { + const { files, bare } = reach(filesUnder(join(ROOT, "desktop/storage"))); + const libFiles = [...files] + .map(rel) + .filter((file) => file.startsWith("lib/")) + .filter((file) => !LIB_ALLOWED.some((pattern) => pattern.test(file))); + expect(libFiles).toEqual([]); + expect([...bare.keys()].filter((spec) => /^react/.test(spec))).toEqual([]); + }); + + test("never imports bun:* (it runs inside Electron)", () => { + const { bare } = reach(filesUnder(join(ROOT, "desktop/storage"))); + expect([...bare.keys()].filter((spec) => spec.startsWith("bun:"))).toEqual( + [], + ); + }); +}); + +describe("lib/platform/desktop", () => { + test("never imports desktop/main, preload or storage", () => { + const { files } = reach(filesUnder(join(ROOT, "lib/platform/desktop"))); + const crossing = [...files] + .map(rel) + .filter((file) => /^desktop\/(main|preload|storage)\//.test(file)); + expect(crossing).toEqual([]); + }); +}); From ba33787536db2b4f60b2cd775f4966a597dec15d Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:37:14 +0330 Subject: [PATCH 07/19] feat(desktop): report the library pragmas and SQLite version in stats Copy diagnostics and soak T3 read back journal_mode, synchronous, fullfsync, checkpoint_fullfsync and foreign_keys as the library runs. --- desktop/storage/engine/library.ts | 9 +++++++++ tests/unit/desktop/storage/library-open.test.ts | 14 +++++++++++--- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/desktop/storage/engine/library.ts b/desktop/storage/engine/library.ts index 3ed6f0a8..fc23116c 100644 --- a/desktop/storage/engine/library.ts +++ b/desktop/storage/engine/library.ts @@ -488,6 +488,15 @@ export class Library { ), lastDailyCopy: readMeta(db, "last_daily_copy"), today: localDate(this.host.now()), + // Read back, as set (D-A6; S2 confirmed them on all three OSes). + pragmas: { + journalMode: scalar(db, "PRAGMA journal_mode"), + synchronous: scalar(db, "PRAGMA synchronous"), + fullfsync: scalar(db, "PRAGMA fullfsync"), + checkpointFullfsync: scalar(db, "PRAGMA checkpoint_fullfsync"), + foreignKeys: scalar(db, "PRAGMA foreign_keys"), + }, + sqliteVersion: scalar(db, "SELECT sqlite_version()"), }; } diff --git a/tests/unit/desktop/storage/library-open.test.ts b/tests/unit/desktop/storage/library-open.test.ts index 2be72f48..5b230d67 100644 --- a/tests/unit/desktop/storage/library-open.test.ts +++ b/tests/unit/desktop/storage/library-open.test.ts @@ -4,7 +4,7 @@ import { readFileSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { StorageError } from "@/desktop/storage/engine/errors"; -import { LIBRARY_FILE, Library } from "@/desktop/storage/engine/library"; +import { LIBRARY_FILE } from "@/desktop/storage/engine/library"; import { LOCK_FILE } from "@/desktop/storage/engine/lock"; import type { FormatStep } from "@/desktop/storage/engine/migrations"; import { TABLES_V1 } from "@/desktop/storage/engine/schema"; @@ -338,6 +338,14 @@ describe("format migrations (T5)", () => { }); }); -test("Library.open is the only constructor", () => { - expect(typeof Library.open).toBe("function"); +test("stats reads the pragmas back as set", () => { + const library = openTestLibrary(temp.dir, createTestHost(clock)); + // checkpoint_fullfsync's default differs between SQLite builds. + expect(library.stats().pragmas).toMatchObject({ + journalMode: "wal", + synchronous: 2, + fullfsync: 0, + foreignKeys: 1, + }); + library.shutdown(); }); From 4cc17fe19ad0ffb450012f03ce2a6eb0c7b6a156 Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:37:15 +0330 Subject: [PATCH 08/19] =?UTF-8?q?test(desktop):=20add=20the=20storage=20du?= =?UTF-8?q?rability=20soak=20(T1=E2=80=93T8)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit tests/load/desktop-durability-soak.ts launches the real storage worker inside a windowless Electron (desktop/storage-harness, its own lockfile on Electron 45.3.2), kills it during appends, compactions and format migrations, quits it with edits in flight, opens a newer library with it, and runs a year of editing over 1,000 documents before timing startup to sidebar rows. Every library is verified by reopening it with the engine on bun:sqlite. T7 is the web library's. desktop-storage-soak.yml runs it on macOS, Windows and Linux: 20 kills per case on pull requests that touch storage, 200 nightly and on demand. --- .github/workflows/desktop-storage-soak.yml | 147 ++++ .github/workflows/nightly.yml | 10 + desktop/storage-harness/.gitignore | 2 + desktop/storage-harness/build.ts | 25 + desktop/storage-harness/bun.lock | 46 ++ desktop/storage-harness/bunfig.toml | 15 + desktop/storage-harness/format-steps.ts | 18 + desktop/storage-harness/gen.ts | 114 +++ desktop/storage-harness/main.ts | 298 ++++++++ desktop/storage-harness/package.json | 18 + desktop/storage-harness/tsconfig.json | 15 + eslint.config.mjs | 1 + tests/load/desktop-durability-soak.ts | 827 +++++++++++++++++++++ 13 files changed, 1536 insertions(+) create mode 100644 .github/workflows/desktop-storage-soak.yml create mode 100644 desktop/storage-harness/.gitignore create mode 100644 desktop/storage-harness/build.ts create mode 100644 desktop/storage-harness/bun.lock create mode 100644 desktop/storage-harness/bunfig.toml create mode 100644 desktop/storage-harness/format-steps.ts create mode 100644 desktop/storage-harness/gen.ts create mode 100644 desktop/storage-harness/main.ts create mode 100644 desktop/storage-harness/package.json create mode 100644 desktop/storage-harness/tsconfig.json create mode 100644 tests/load/desktop-durability-soak.ts diff --git a/.github/workflows/desktop-storage-soak.yml b/.github/workflows/desktop-storage-soak.yml new file mode 100644 index 00000000..861b93fd --- /dev/null +++ b/.github/workflows/desktop-storage-soak.yml @@ -0,0 +1,147 @@ +name: Desktop storage soak + +# Plan T1–T8 (P2.6): the storage worker on node:sqlite inside Electron, +# SIGKILLed during appends, compactions and migrations, quit with writes in +# flight, opened by an older app, and run through a simulated year with +# 1,000 documents. The verifier reopens every library with the engine on +# bun:sqlite. See tests/load/desktop-durability-soak.ts. +# +# Pull requests that touch storage run a short profile (20 kills per case); +# the nightly and manual runs use the plan's 200. Not a required check. +# Read-only token and no secrets, like CI. + +on: + pull_request: + paths: + - desktop/storage/** + - desktop/storage-harness/** + - desktop/dev/bun-sqlite-driver.ts + - lib/platform/desktop/** + - tests/load/desktop-durability-soak.ts + - .github/workflows/desktop-storage-soak.yml + workflow_call: + inputs: + kills: + type: number + default: 200 + workflow_dispatch: + inputs: + kills: + description: Kills per kill case (T1, T2; T1-dense runs half) + type: number + default: 200 + +permissions: + contents: read + +concurrency: + group: desktop-storage-soak-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + soak: + name: ${{ matrix.group.name }} (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + os: [macos-26, windows-latest, ubuntu-24.04] + group: + - name: contract, T3–T6 + cases: contract,T3,T4,T5,T6 + - name: T1 kill during append + cases: T1,T1-dense + - name: T2 kill during compaction + cases: T2 + - name: T8 year and startup + cases: T8 + defaults: + run: + shell: bash + env: + KILLS: ${{ github.event_name == 'pull_request' && 20 || inputs.kills || 200 }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: .tool-versions + + # The root install brings Yjs and the engine's dependencies; the + # harness has its own lockfile with Electron, whose binary downloads + # on first use. + - name: Install + run: | + bun install --frozen-lockfile + cd desktop/storage-harness + bun install --frozen-lockfile + bun -e 'require("electron")' + + # Everything after install runs on Bun. The runner image ships Node, + # so a stub makes any call to it fail loudly. Install comes first: + # lifecycle scripts (husky) call `node`. + - name: Block Node.js + run: | + mkdir -p "$RUNNER_TEMP/no-node" + cat > "$RUNNER_TEMP/no-node/node" <<'EOF' + #!/bin/sh + echo "::error::node was called: $*" >&2 + exit 1 + EOF + chmod +x "$RUNNER_TEMP/no-node/node" + echo "$RUNNER_TEMP/no-node" >> "$GITHUB_PATH" + + # Windows process spawns look for node.exe and never run the sh stub + # above, so put a compiled stub beside it (found in spike S3). + - name: Block Node.js (node.exe on Windows) + if: runner.os == 'Windows' + run: | + cat > "$RUNNER_TEMP/node-stub.ts" <<'EOF' + console.error(`::error::node was called: ${process.argv.slice(2).join(" ")}`); + process.exit(1); + EOF + bun build --compile "$RUNNER_TEMP/node-stub.ts" --outfile "$RUNNER_TEMP/no-node/node.exe" + if cmd.exe //d //c "node --version" > /dev/null 2>&1; then + echo "::error::the node.exe stub is not first on PATH" + exit 1 + fi + + - name: Audit the harness dependencies + working-directory: desktop/storage-harness + run: bun audit --audit-level=high + + - name: Typecheck and build the harness + working-directory: desktop/storage-harness + run: | + bun run typecheck + bun run build + + - name: Allow the Chromium sandbox (electron/electron#42510) + if: runner.os == 'Linux' + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + - name: Soak + env: + CASES: ${{ matrix.group.cases }} + run: | + if [ "$RUNNER_OS" = Linux ]; then + xvfb-run -a bun tests/load/desktop-durability-soak.ts --case="$CASES" --kills="$KILLS" --out=soak-out + else + bun tests/load/desktop-durability-soak.ts --case="$CASES" --kills="$KILLS" --out=soak-out + fi + + - name: Upload results + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: storage-soak-${{ matrix.os }}-${{ strategy.job-index }} + path: soak-out + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 242b8b14..667b34bd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -13,6 +13,8 @@ name: Nightly # the gate env at its default (spec §3.1 N+1 ship gate) # desktop-smoke the D-A12 Electron smoke (desktop-smoke.yml) on macOS, # Windows and Linux +# desktop-storage-soak plan T1–T8 against the storage worker in Electron +# (desktop-storage-soak.yml), 200 kills per case, 3 OSes # # Read-only token and no secrets, like CI. A red run means a regression in # offline editing, long sessions, concurrency or schema rollout safety that @@ -361,3 +363,11 @@ jobs: permissions: contents: read uses: ./.github/workflows/desktop-smoke.yml + + desktop-storage-soak: + name: Desktop storage soak (T1–T8) + permissions: + contents: read + uses: ./.github/workflows/desktop-storage-soak.yml + with: + kills: 200 diff --git a/desktop/storage-harness/.gitignore b/desktop/storage-harness/.gitignore new file mode 100644 index 00000000..8225baa4 --- /dev/null +++ b/desktop/storage-harness/.gitignore @@ -0,0 +1,2 @@ +/node_modules +/dist diff --git a/desktop/storage-harness/build.ts b/desktop/storage-harness/build.ts new file mode 100644 index 00000000..194b77bd --- /dev/null +++ b/desktop/storage-harness/build.ts @@ -0,0 +1,25 @@ +/** + * Bundles the harness main and the real storage worker as CJS for + * Electron, the way `desktop/build/build.ts` (P2.5) will: `node:*` and + * `electron` stay external, everything else (Yjs, the engine) is inlined. + */ +import path from "node:path"; + +const root = import.meta.dir; +const result = await Bun.build({ + entrypoints: [ + path.join(root, "main.ts"), + path.join(root, "../storage/worker.ts"), + ], + outdir: path.join(root, "dist"), + target: "node", + format: "cjs", + external: ["electron"], + naming: "[name].cjs", +}); +if (!result.success) { + for (const log of result.logs) console.error(log); + process.exit(1); +} +for (const output of result.outputs) + console.log(`${path.relative(root, output.path)} ${output.size} B`); diff --git a/desktop/storage-harness/bun.lock b/desktop/storage-harness/bun.lock new file mode 100644 index 00000000..8fb81cc4 --- /dev/null +++ b/desktop/storage-harness/bun.lock @@ -0,0 +1,46 @@ +{ + "lockfileVersion": 2, + "configVersion": 1, + "workspaces": { + "": { + "name": "tinydocy-storage-harness", + "devDependencies": { + "@types/node": "24.19.2", + "electron": "45.3.2", + "typescript": "6.0.3", + }, + }, + }, + "trustedDependencies": [ + "electron", + ], + "packages": { + "@electron-internal/extract-zip": ["@electron-internal/extract-zip@1.0.8", "", {}, "sha512-8iajeLFbzNwM0m3AsQZSZgYREWglAImyGvvFSWU34FgMW8sHlzdwj9QKvv9dgN7jMP591EFUF/sLOC3Jl9YIjQ=="], + + "@electron/get": ["@electron/get@5.1.0", "", { "dependencies": { "debug": "^4.1.1", "env-paths": "^3.0.0", "graceful-fs": "^4.2.11", "progress": "^2.0.3", "semver": "^7.6.3", "sumchecker": "^3.0.1" }, "optionalDependencies": { "undici": "^7.24.4" } }, "sha512-3kSBtG8ObcTVfXanm5vVJ6UnBLEVmVsRk1M+vGqCuMBV+XLCbJYuWQful+yIy0GQDsSlK0kHEriEHn7SPk4EnA=="], + + "@types/node": ["@types/node@24.19.2", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-93d49wCWJkCyN87MxLpNN6KXPCwFw7bVUg79mGUV1OS40JGWwVCr8n61DXCSqz7CV6/VfPkb7xec7uP2M+QHWQ=="], + + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" }, "peerDependencies": { "supports-color": "*" }, "optionalPeers": ["supports-color"] }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + + "electron": ["electron@45.3.2", "", { "dependencies": { "@electron-internal/extract-zip": "^1.0.1", "@electron/get": "^5.0.0", "@types/node": "^24.9.0" }, "bin": { "electron": "cli.js", "install-electron": "install.js" } }, "sha512-bPO0MMvJCdKdPfXgd3ES8zupu+ocr6XmSuRGwyZazTiigTz8jE7rwukSkck0iomS6UZeFuCwzu0uanMD/D34uw=="], + + "env-paths": ["env-paths@3.0.0", "", {}, "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A=="], + + "graceful-fs": ["graceful-fs@4.2.12", "", {}, "sha512-iuyGF2E4oafQfPsRrG2wtY2ILupMhvtdrhzrxB+0VcjsPpSPttpfYdjnGFGsTi5MaJFjI1jOKHPQ6UTbTj9TAw=="], + + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "progress": ["progress@2.0.3", "", {}, "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA=="], + + "semver": ["semver@7.8.6", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-9lpSSU4Mly3mrtruwLs1eLK6+gzbzqeBFwRNjIjE+9LuDpWdYctrQlQ3ej1Ilgi7gknpsTRnPoLP8DpCHqHIJw=="], + + "sumchecker": ["sumchecker@3.0.1", "", { "dependencies": { "debug": "^4.1.0" } }, "sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg=="], + + "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], + + "undici": ["undici@7.30.2", "", {}, "sha512-mLyPQnHwUxaM/rideoK/+KL8DaAJLRl4aBCfmX444aCSLnIaomqPLtNRe2jdCYcsk5wZzXrDIQC8ZeYNRAYyIA=="], + + "undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], + } +} diff --git a/desktop/storage-harness/bunfig.toml b/desktop/storage-harness/bunfig.toml new file mode 100644 index 00000000..d968ccb5 --- /dev/null +++ b/desktop/storage-harness/bunfig.toml @@ -0,0 +1,15 @@ +# Supply-chain guardrail: only resolve package versions that have been +# public for at least 3 days (value in seconds). Gives the registry and the +# community time to catch and unpublish compromised releases before +# `bun add` / `bun update` (local or in CI) can pick them up. +[install] +minimumReleaseAge = 259200 + +# Run package bins (next, eslint, playwright, ...) on the Bun runtime +# instead of Node, for `bun run` scripts and `bun `. Plain `bunx` +# ignores this setting; use `bunx --bun` for packages that are not installed. +[run] +bun = true + +# `bun test`: unit tests only (Playwright specs live in tests/e2e), with a +# happy-dom window preloaded. diff --git a/desktop/storage-harness/format-steps.ts b/desktop/storage-harness/format-steps.ts new file mode 100644 index 00000000..7fa8ef63 --- /dev/null +++ b/desktop/storage-harness/format-steps.ts @@ -0,0 +1,18 @@ +/** + * Format steps for soak T5 (migration interrupted). Each step builds a + * table of {@link STEP_ROWS} rows, slow enough that a kill lands inside one. + * The Electron run and the Bun verifier use the same list. + */ +import type { FormatStep } from "../storage/engine/migrations"; + +export const STEP_ROWS = 150_000; +export const T5_TARGET = 4; + +export const T5_STEPS: FormatStep[] = [2, 3, 4].map((to) => ({ + to, + up(db) { + db.exec(`CREATE TABLE soak_step_${to} (n INTEGER NOT NULL) STRICT`); + const insert = db.prepare(`INSERT INTO soak_step_${to} (n) VALUES (?)`); + for (let n = 0; n < STEP_ROWS; n += 1) insert.run(n); + }, +})); diff --git a/desktop/storage-harness/gen.ts b/desktop/storage-harness/gen.ts new file mode 100644 index 00000000..9ea200d8 --- /dev/null +++ b/desktop/storage-harness/gen.ts @@ -0,0 +1,114 @@ +/** + * Deterministic edits for the durability soak, shared by the Electron churn + * and the Bun verifier. Edit `k` appends `";"` to the text `t`, so a + * stored document is valid exactly when its text is `1;2;…;K;` for some K: + * a lost or half-applied batch shows up as a gap or a short K. + * + * `pad` is filler that makes compactions heavy (the plan's 5 MB document) + * without touching `t`. + */ +import * as Y from "yjs"; + +export const TEXT = "t"; +export const PAD = "pad"; + +/** The K of a stored text, or null if it isn't `1;2;…;K;`. */ +export function editCount(text: string): number | null { + if (text === "") return 0; + const parts = text.split(";"); + if (parts.pop() !== "") return null; + for (const [i, part] of parts.entries()) + if (part !== String(i + 1)) return null; + return parts.length; +} + +export function padUpdate(bytes: number): Uint8Array { + const doc = new Y.Doc(); + // A plain repeated string compresses nothing in Yjs: one item. + doc.getText(PAD).insert(0, "x".repeat(bytes)); + return Y.encodeStateAsUpdate(doc); +} + +export type BatchTiming = { debounceMs: number; maxWaitMs: number }; + +/** The product's batching (architecture §5.2), or the dense soak variant. */ +export const PRODUCT_TIMING: BatchTiming = { debounceMs: 250, maxWaitMs: 1000 }; +export const DENSE_TIMING: BatchTiming = { debounceMs: 5, maxWaitMs: 20 }; + +/** + * A minimal batcher: merges updates, flushes `debounceMs` after the last or + * `maxWaitMs` after the first, and keeps one send in flight. Stand-in for + * P2.1's `lib/platform/batcher.ts`, which the real app uses. + */ +export class SoakBatcher { + private pending: Uint8Array[] = []; + private pendingEdits = 0; + private firstAt = 0; + private timer: ReturnType | null = null; + private inFlight: Promise | null = null; + + constructor( + private readonly timing: BatchTiming, + private readonly send: ( + update: Uint8Array, + lastEdit: number, + ) => Promise, + ) {} + + push(update: Uint8Array, edit: number) { + if (this.pending.length === 0) this.firstAt = performance.now(); + this.pending.push(update); + this.pendingEdits = edit; + this.schedule(); + } + + private schedule() { + if (this.timer) clearTimeout(this.timer); + const sinceFirst = performance.now() - this.firstAt; + const wait = Math.max( + 0, + Math.min(this.timing.debounceMs, this.timing.maxWaitMs - sinceFirst), + ); + this.timer = setTimeout(() => void this.flush(), wait); + } + + get busy(): Promise | null { + return this.inFlight; + } + + /** Sends what is queued, after any send in flight. */ + async flush(): Promise { + if (this.timer) clearTimeout(this.timer); + this.timer = null; + while (this.inFlight) await this.inFlight; + if (this.pending.length === 0) return; + const update = Y.mergeUpdates(this.pending); + const lastEdit = this.pendingEdits; + this.pending = []; + this.inFlight = this.send(update, lastEdit).finally(() => { + this.inFlight = null; + }); + await this.inFlight; + } +} + +/** Parses `--key=value` and `--flag` arguments. */ +export function parseArgs(argv: string[]): Record { + const out: Record = {}; + for (const arg of argv) { + const match = arg.match(/^--([\w-]+)(?:=(.*))?$/); + if (match?.[1]) out[match[1]] = match[2] ?? "1"; + } + return out; +} + +/** A small seeded PRNG (mulberry32), so a soak run can be replayed. */ +export function seededRandom(seed: number): () => number { + let a = seed | 0; + return () => { + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} diff --git a/desktop/storage-harness/main.ts b/desktop/storage-harness/main.ts new file mode 100644 index 00000000..4a2c655e --- /dev/null +++ b/desktop/storage-harness/main.ts @@ -0,0 +1,298 @@ +/** + * Electron side of the storage soak (P2.6; plan T1–T8). Runs the real + * storage worker (`worker.cjs`, `node:sqlite`) from a windowless, + * dock-less Electron main. `tests/load/desktop-durability-soak.ts` (Bun) + * launches it, kills it, and verifies the library afterwards with the + * engine on `bun:sqlite`. + * + * Modes (`--mode=`): + * - `contract`: the engine `DocStore` cases over the worker (`CASE_OK` / + * `CASE_FAIL` lines). + * - `churn`: edits a document with product batching (or `--dense`) until + * killed; `--target=compaction` compacts after every ack; + * `--quit-after-ms` runs the quit flush instead (T4). The worker prints + * `BATCH_BEGIN` / `ACK ` / `COMPACT_BEGIN` / `COMPACT_ACK`; main + * prints `EDITS ` once it has the ack. + * - `migrate`: opens the library on main's thread with the T5 format steps. + * - `readonly`: opens a newer library (T6) and tries to write. + * - `commit-timing`: append round trips with `fullfsync` on or off (T3). + * - `startup`: worker spawn → ready → sidebar rows (T8, the §10 budget). + * + * An uncaught throw in Electron main opens a modal dialog, so every failure + * exits with `process.exit(1)` after printing `HARNESS_FAIL`. + */ +import { writeSync } from "node:fs"; +import path from "node:path"; + +import { app } from "electron"; +import * as Y from "yjs"; + +import { createRpcDocStore } from "../../lib/platform/desktop/rpc-doc-store"; +import type { DocStore } from "../../lib/platform/doc-store"; +import { STORAGE_DOC_STORE_CASES } from "../../tests/helpers/storage-doc-store-cases"; +import { StorageClient } from "../storage/client"; +import { openNodeSqlite } from "../storage/drivers/node-sqlite"; +import { Library } from "../storage/engine/library"; +import { createNodeHost } from "../storage/node-host"; +import { T5_STEPS, T5_TARGET } from "./format-steps"; +import { + DENSE_TIMING, + editCount, + PRODUCT_TIMING, + parseArgs, + SoakBatcher, + seededRandom, + TEXT, +} from "./gen"; + +const args = parseArgs(process.argv.slice(1)); +const say = (line: string) => writeSync(1, `${line}\n`); +const fail = (detail: unknown): never => { + const text = + detail instanceof Error + ? `${detail.message}\n${detail.stack}` + : String(detail); + writeSync(2, `HARNESS_FAIL ${text}\n`); + process.exit(1); +}; +process.on("uncaughtException", fail); +process.on("unhandledRejection", fail); + +app.disableHardwareAcceleration(); +// Only app.exit ends a run; there are no windows to close. +app.on("window-all-closed", () => {}); + +// Not __dirname: Bun.build inlines the source directory (S1, S2). +const workerPath = () => path.join(app.getAppPath(), "worker.cjs"); +const dir = () => args.dir ?? fail("--dir is required"); + +/** Copies bytes before they are transferred, as IPC would. */ +function copied(value: T): T { + if (value instanceof Uint8Array) return value.slice() as T; + if (Array.isArray(value)) return value.map(copied) as T; + if (value && typeof value === "object") + return Object.fromEntries( + Object.entries(value).map(([k, v]) => [k, copied(v)]), + ) as T; + return value; +} + +function storeFor(client: StorageClient, owner: string): DocStore { + return createRpcDocStore((op, callArgs) => + op === "flush" + ? (client.call("flush", {}) as never) + : (client.call(op, { ...copied(callArgs), owner } as never) as never), + ); +} + +async function contract() { + const client = await StorageClient.start(workerPath(), { dir: dir() }); + let failed = 0; + for (const [i, testCase] of STORAGE_DOC_STORE_CASES.entries()) { + const stores = new Map(); + try { + await testCase.run({ + store(owner) { + const key = `${i}:${owner}`; + let store = stores.get(key); + if (!store) { + store = storeFor(client, key); + stores.set(key, store); + } + return store; + }, + createDoc: async () => (await client.call("createDoc", {})).id, + }); + say(`CASE_OK ${testCase.name}`); + } catch (error) { + failed += 1; + say(`CASE_FAIL ${testCase.name}: ${String(error)}`); + } + } + say(`STATS ${JSON.stringify(await client.call("stats", {}))}`); + say(`SHUTDOWN ${await client.shutdown()}`); + if (failed > 0) fail(`${failed} contract case(s) failed`); +} + +async function churn() { + const docId = args.doc ?? fail("--doc is required"); + const random = seededRandom(Number(args.seed ?? 1)); + const client = await StorageClient.start(workerPath(), { + dir: dir(), + trace: true, + synchronous: args.synchronous === "OFF" ? "OFF" : "FULL", + fullfsync: args.fullfsync !== "0", + }); + const owner = "churn"; + const loaded = await client.call("load", { docId, owner }); + const doc = new Y.Doc(); + if (loaded.snapshot) Y.applyUpdate(doc, loaded.snapshot); + for (const update of loaded.updates) Y.applyUpdate(doc, update); + let k = + editCount(doc.getText(TEXT).toString()) ?? fail("stored text is broken"); + let ackedSeq = loaded.maxSeq; + const compactEvery = args.target === "compaction"; + + const batcher = new SoakBatcher( + args.dense ? DENSE_TIMING : PRODUCT_TIMING, + async (update, lastEdit) => { + ackedSeq = await client.call("append", { + docId, + owner, + record: { origin: "local", update }, + }); + say(`EDITS ${ackedSeq} ${lastEdit}`); + if (compactEvery) { + // Same task as the encode: no append is in flight (architecture §5.2). + const state = Y.encodeStateAsUpdate(doc); + await client.call("compact", { + docId, + owner, + state, + upToSeq: ackedSeq, + }); + } + }, + ); + let pendingEdit: Uint8Array | null = null; + doc.on("update", (update: Uint8Array) => { + pendingEdit = update; + }); + const quitAfter = args["quit-after-ms"] + ? Number(args["quit-after-ms"]) + : null; + const started = performance.now(); + say(`READY ${k}`); + for (;;) { + // A burst of 1–8 edits 2–10 ms apart, then a 260–400 ms pause. + const burst = 1 + Math.floor(random() * 8); + for (let i = 0; i < burst; i += 1) { + k += 1; + const text = doc.getText(TEXT); + text.insert(text.length, `${k};`); + if (pendingEdit) batcher.push(pendingEdit, k); + pendingEdit = null; + await new Promise((r) => setTimeout(r, 2 + random() * 8)); + } + if (quitAfter !== null && performance.now() - started >= quitAfter) break; + await new Promise((r) => setTimeout(r, 260 + random() * 140)); + } + // T4: the quit flush (architecture §8.2): flush every batch, then shut + // the worker down. + await batcher.flush(); + const outcome = await client.shutdown(); + say(`QUIT ${outcome} ${k} ${ackedSeq}`); +} + +async function migrate() { + say("READY"); + const library = Library.open({ + dir: dir(), + host: createNodeHost(), + openSqlite: openNodeSqlite, + formatVersion: T5_TARGET, + formatSteps: T5_STEPS, + }); + say(`MIGRATED ${library.status.formatVersion} ${library.status.readOnly}`); + library.shutdown(); +} + +async function readonly() { + const client = await StorageClient.start(workerPath(), { dir: dir() }); + say(`STATUS ${JSON.stringify(client.status)}`); + const sidebar = await client.call("sidebar", {}); + say(`SIDEBAR ${sidebar.docs.length}`); + try { + await client.call("createDoc", {}); + say("WRITE accepted"); + } catch (error) { + say(`WRITE refused ${(error as { code?: string }).code}`); + } + say(`SHUTDOWN ${await client.shutdown()}`); +} + +function percentile(sorted: number[], p: number) { + return ( + sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))] ?? 0 + ); +} + +async function commitTiming() { + const count = Number(args.count ?? 300); + const client = await StorageClient.start(workerPath(), { + dir: dir(), + fullfsync: args.fullfsync !== "0", + }); + const { id } = await client.call("createDoc", {}); + await client.call("load", { docId: id, owner: "timing" }); + const doc = new Y.Doc(); + const times: number[] = []; + for (let i = 0; i < count; i += 1) { + let update: Uint8Array = new Uint8Array(); + doc.once("update", (u: Uint8Array) => { + update = u; + }); + doc.getText(TEXT).insert(0, "y".repeat(4096)); + const t0 = performance.now(); + await client.call("append", { + docId: id, + owner: "timing", + record: { origin: "local", update }, + }); + times.push(performance.now() - t0); + } + times.sort((a, b) => a - b); + const stats = await client.call("stats", {}); + say( + `TIMING ${JSON.stringify({ + fullfsync: args.fullfsync !== "0", + count, + p50: percentile(times, 0.5), + p95: percentile(times, 0.95), + max: times.at(-1), + pragmas: stats.pragmas, + sqlite: stats.sqliteVersion, + })}`, + ); + await client.shutdown(); +} + +async function startup() { + // performance.now() counts from process start in Node and Electron main. + const spawnAt = performance.now(); + const client = await StorageClient.start(workerPath(), { dir: dir() }); + const readyAt = performance.now(); + const sidebar = await client.call("sidebar", {}); + const rowsAt = performance.now(); + say( + `STARTUP ${JSON.stringify({ + processToSpawnMs: spawnAt, + workerReadyMs: readyAt - spawnAt, + openMs: client.timings.openMs, + sidebarCallMs: rowsAt - readyAt, + processToSidebarMs: rowsAt, + spawnToSidebarMs: rowsAt - spawnAt, + docs: sidebar.docs.length, + folders: sidebar.folders.length, + uncleanExit: client.status.uncleanExit, + })}`, + ); + await client.shutdown(); +} + +const MODES: Record Promise> = { + contract, + churn, + migrate, + readonly, + "commit-timing": commitTiming, + startup, +}; + +app.whenReady().then(async () => { + app.dock?.hide(); + if (process.platform === "darwin") app.setActivationPolicy("accessory"); + const run = MODES[args.mode ?? ""] ?? fail(`unknown --mode=${args.mode}`); + await run(); + app.exit(0); +}, fail); diff --git a/desktop/storage-harness/package.json b/desktop/storage-harness/package.json new file mode 100644 index 00000000..ae4a58ab --- /dev/null +++ b/desktop/storage-harness/package.json @@ -0,0 +1,18 @@ +{ + "name": "tinydocy-storage-harness", + "private": true, + "type": "module", + "main": "dist/main.cjs", + "scripts": { + "build": "bun build.ts", + "typecheck": "tsc --noEmit -p ." + }, + "devDependencies": { + "@types/node": "24.19.2", + "electron": "45.3.2", + "typescript": "6.0.3" + }, + "trustedDependencies": [ + "electron" + ] +} diff --git a/desktop/storage-harness/tsconfig.json b/desktop/storage-harness/tsconfig.json new file mode 100644 index 00000000..bacd8dd1 --- /dev/null +++ b/desktop/storage-harness/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2023", + "lib": ["ES2023"], + "module": "Preserve", + "moduleResolution": "bundler", + "moduleDetection": "force", + "types": ["node"], + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "paths": { "@/*": ["../../*"] } + }, + "include": ["main.ts", "gen.ts", "format-steps.ts"] +} diff --git a/eslint.config.mjs b/eslint.config.mjs index b5665bd6..05f3f6b7 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -11,6 +11,7 @@ const eslintConfig = defineConfig([ "build/**", "next-env.d.ts", "desktop/smoke/dist/**", + "desktop/storage-harness/dist/**", ]), { settings: { diff --git a/tests/load/desktop-durability-soak.ts b/tests/load/desktop-durability-soak.ts new file mode 100644 index 00000000..0c4d9738 --- /dev/null +++ b/tests/load/desktop-durability-soak.ts @@ -0,0 +1,827 @@ +/** + * Desktop durability and growth soak (plan §9 T1–T8, P2.6's exit row). + * + * Launches the real storage worker inside Electron (the harness in + * `desktop/storage-harness`, built first), SIGKILLs it at chosen moments + * (`TerminateProcess` on Windows), waits until the pid is gone, then + * reopens the library with the engine on `bun:sqlite`, which also proves + * the two drivers read each other's files. The shape follows spike S2's + * kill harness. + * + * bun tests/load/desktop-durability-soak.ts --case=all|contract|T1|T2|T3|T4|T5|T6|T8 + * [--kills=200] [--out=soak-out] [--verbose] + * + * T7 (slow-hydrate overwrite) is a web-library case: it tests the session + * layer, not storage, and runs with W2. + * + * Writes `/.json` and a Markdown table to `$GITHUB_STEP_SUMMARY`. + * Exits 1 if any case fails. Run it only on CI runners (it starts + * Electron); on a desktop machine the windows stay hidden but it still + * takes the CPU for minutes. + */ + +import { Database } from "bun:sqlite"; +import { createHash } from "node:crypto"; +import { + appendFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import * as Y from "yjs"; + +import { openBunSqlite } from "@/desktop/dev/bun-sqlite-driver"; +import { DAILY_KEEP } from "@/desktop/storage/engine/daily-copy"; +import { + IDLE_MS, + LIBRARY_FILE, + Library, + type OpenLibraryOptions, +} from "@/desktop/storage/engine/library"; +import { LOCK_FILE } from "@/desktop/storage/engine/lock"; +import { createNodeHost } from "@/desktop/storage/node-host"; +import { + STEP_ROWS, + T5_STEPS, + T5_TARGET, +} from "@/desktop/storage-harness/format-steps"; +import { + editCount, + PAD, + padUpdate, + parseArgs, + seededRandom, + TEXT, +} from "@/desktop/storage-harness/gen"; + +const ROOT = join(import.meta.dir, "../.."); +const HARNESS = join(ROOT, "desktop/storage-harness"); +const args = parseArgs(process.argv.slice(2)); +const OUT = args.out ?? "soak-out"; +const KILLS = Number(args.kills ?? 200); +const isWindows = process.platform === "win32"; + +mkdirSync(OUT, { recursive: true }); + +const harnessMain = join(HARNESS, "dist", "main.cjs"); +/** The harness's own Electron (its package has the dependency, not the root). */ +function electronBinary(): string { + if (!existsSync(harnessMain)) + throw new Error( + "build the harness first: cd desktop/storage-harness && bun run build", + ); + return createRequire(join(HARNESS, "package.json"))("electron") as string; +} + +type CaseResult = { + name: string; + pass: boolean; + summary: string; + details: Record; + failures: string[]; +}; + +const note = (text: string) => { + if (args.verbose) console.log(text); +}; + +// ── Engine access (bun:sqlite) ────────────────────────────────────────── + +function openEngine( + dir: string, + options: Partial = {}, +): Library { + return Library.open({ + dir, + host: createNodeHost(), + openSqlite: openBunSqlite, + fullfsync: false, + ...options, + }); +} + +function textOf(library: Library, docId: string, owner = "verify") { + const loaded = library.load(docId, owner); + const doc = new Y.Doc(); + if (loaded.snapshot) Y.applyUpdate(doc, loaded.snapshot); + for (const update of loaded.updates) Y.applyUpdate(doc, update); + library.close(docId, owner); + return { + text: doc.getText(TEXT).toString(), + padLength: doc.getText(PAD).length, + maxSeq: loaded.maxSeq, + }; +} + +function seedLibrary(dir: string, padBytes: number): string { + const library = openEngine(dir); + const { id } = library.createDoc({ title: "Soak" }); + if (padBytes > 0) { + library.load(id, "seed"); + const seq = library.append(id, "seed", { + origin: "local", + update: padUpdate(padBytes), + }); + const doc = new Y.Doc(); + Y.applyUpdate(doc, padUpdate(padBytes)); + library.compact(id, "seed", Y.encodeStateAsUpdate(doc), seq); + library.close(id, "seed"); + } + library.shutdown(); + return id; +} + +// ── Electron runs ─────────────────────────────────────────────────────── + +type Run = { + lines: string[]; + exitCode: number | null; + stderr: string; + pid: number; +}; + +function pidGone(pid: number): boolean { + if (isWindows) { + const out = Bun.spawnSync(["tasklist", "/FI", `PID eq ${pid}`, "/NH"]); + return !out.stdout.toString().includes(` ${pid} `); + } + try { + process.kill(pid, 0); + return false; + } catch (error) { + return (error as { code?: string }).code === "ESRCH"; + } +} + +/** + * Starts the harness and feeds each stdout line to `onLine`, which may + * return "kill" to SIGKILL the process. Resolves once the process exited + * and its pid is gone. + */ +async function runHarness( + harnessArgs: string[], + onLine: (line: string, kill: () => void) => void = () => {}, + timeoutMs = 120_000, +): Promise { + const proc = Bun.spawn([electronBinary(), harnessMain, ...harnessArgs], { + stdout: "pipe", + stderr: "pipe", + env: { ...process.env, ELECTRON_ENABLE_STACK_DUMPING: "1" }, + }); + let killed = false; + const kill = () => { + if (killed) return; + killed = true; + proc.kill("SIGKILL"); + }; + const timer = setTimeout(kill, timeoutMs); + const lines: string[] = []; + const decoder = new TextDecoder(); + let buffer = ""; + for await (const chunk of proc.stdout) { + buffer += decoder.decode(chunk, { stream: true }); + let newline = buffer.indexOf("\n"); + while (newline >= 0) { + const line = buffer.slice(0, newline).trimEnd(); + buffer = buffer.slice(newline + 1); + lines.push(line); + if (!killed) onLine(line, kill); + newline = buffer.indexOf("\n"); + } + } + const exitCode = await proc.exited; + clearTimeout(timer); + const stderr = await new Response(proc.stderr).text(); + for (let i = 0; i < 50 && !pidGone(proc.pid); i += 1) await Bun.sleep(100); + if (!pidGone(proc.pid)) throw new Error(`pid ${proc.pid} still running`); + return { lines, exitCode, stderr, pid: proc.pid, killed }; +} + +// ── contract: the engine DocStore cases inside the Electron worker ────── + +async function caseContract(): Promise { + const dir = mkdtempSync(join(tmpdir(), "soak-contract-")); + const run = await runHarness([`--mode=contract`, `--dir=${dir}`]); + rmSync(dir, { recursive: true, force: true }); + const ok = run.lines.filter((l) => l.startsWith("CASE_OK")).length; + const failed = run.lines.filter((l) => l.startsWith("CASE_FAIL")); + const stats = run.lines.find((l) => l.startsWith("STATS ")); + return { + name: "contract", + pass: run.exitCode === 0 && failed.length === 0 && ok > 0, + summary: `${ok} DocStore cases passed on node:sqlite in the Electron worker`, + details: { + ok, + stats: stats ? JSON.parse(stats.slice(6)) : null, + exitCode: run.exitCode, + }, + failures: [ + ...failed, + ...(run.exitCode === 0 ? [] : [run.stderr.slice(0, 2000)]), + ], + }; +} + +// ── T1, T2, T3 control: kill during append or compaction ─────────────── + +type KillTarget = "append" | "compaction"; + +async function killLoop( + name: string, + target: KillTarget, + kills: number, + options: { dense?: boolean; synchronous?: "FULL" | "OFF" } = {}, +): Promise { + const dir = mkdtempSync(join(tmpdir(), `soak-${name}-`)); + const padBytes = target === "compaction" ? 5 * 1024 * 1024 : 64 * 1024; + const docId = seedLibrary(dir, padBytes); + const random = seededRandom(kills * 7919 + target.length); + const failures: string[] = []; + let inside = 0; + let afterFirstAck = 0; + let maxK = 0; + for (let i = 1; i <= kills; i += 1) { + let ackSeq = 0; + let ackedK = 0; + let last = ""; + let sawAck = false; + const delay = target === "append" ? 50 + random() * 1950 : random() * 20; + const run = await runHarness( + [ + "--mode=churn", + `--dir=${dir}`, + `--doc=${docId}`, + `--seed=${i}`, + `--target=${target}`, + `--synchronous=${options.synchronous ?? "FULL"}`, + ...(options.dense ? ["--dense"] : []), + ], + (line, kill) => { + const [word, a, b] = line.split(" "); + if (word === "ACK") { + ackSeq = Math.max(ackSeq, Number(a)); + sawAck = true; + } else if (word === "EDITS") ackedK = Math.max(ackedK, Number(b)); + if (/^(BATCH_BEGIN|ACK|COMPACT_BEGIN|COMPACT_ACK)$/.test(word ?? "")) + last = word ?? ""; + if (target === "append" && word === "READY") setTimeout(kill, delay); + if (target === "compaction" && word === "COMPACT_BEGIN") + setTimeout(kill, delay); + if (target === "compaction" && word === "READY") + setTimeout(kill, 15_000); // never compacted: still kill + }, + 60_000, + ); + if (!run.killed) { + failures.push( + `kill ${i}: exited by itself (${run.exitCode}) ${run.stderr.slice(0, 500)}`, + ); + continue; + } + if (last === "BATCH_BEGIN" || last === "COMPACT_BEGIN") inside += 1; + if (sawAck) afterFirstAck += 1; + try { + const library = openEngine(dir); + const { status } = library; + if (status.quickCheck !== null && status.quickCheck !== "ok") + failures.push(`kill ${i}: quick_check ${status.quickCheck}`); + const stored = textOf(library, docId); + library.shutdown(); + const k = editCount(stored.text); + if (k === null) failures.push(`kill ${i}: text has a gap or a torn edit`); + else { + maxK = k; + if (k < ackedK) + failures.push(`kill ${i}: lost acked edits (${k} < ${ackedK})`); + } + if (stored.maxSeq < ackSeq) + failures.push( + `kill ${i}: lost acked seq (${stored.maxSeq} < ${ackSeq})`, + ); + if (stored.padLength !== padBytes) + failures.push(`kill ${i}: pad is ${stored.padLength}, not ${padBytes}`); + } catch (error) { + failures.push(`kill ${i}: reopen failed: ${String(error)}`); + } + note( + `${name} ${i}/${kills} last=${last} acked=${ackedK} failures=${failures.length}`, + ); + } + rmSync(dir, { recursive: true, force: true }); + return { + name, + pass: failures.length === 0, + summary: `${kills} kills, ${afterFirstAck} after the first ack, ${inside} inside a transaction; ${failures.length} failures; final K ${maxK}`, + details: { kills, afterFirstAck, inside, maxK, target, ...options }, + failures: failures.slice(0, 50), + }; +} + +// ── T3: fullfsync on vs off, plus the synchronous=OFF control ───────── + +async function caseT3(): Promise { + const failures: string[] = []; + const timings: Record[] = []; + for (const fullfsync of ["1", "0"]) { + const dir = mkdtempSync(join(tmpdir(), "soak-t3-")); + const run = await runHarness([ + "--mode=commit-timing", + `--dir=${dir}`, + `--fullfsync=${fullfsync}`, + "--count=300", + ]); + rmSync(dir, { recursive: true, force: true }); + const line = run.lines.find((l) => l.startsWith("TIMING ")); + if (!line) { + failures.push( + `timing fullfsync=${fullfsync}: ${run.stderr.slice(0, 500)}`, + ); + continue; + } + const timing = JSON.parse(line.slice(7)) as { + pragmas: Record; + } & Record; + timings.push(timing); + const expectedFull = + process.platform === "darwin" && fullfsync === "1" ? 1 : 0; + if (timing.pragmas.journalMode !== "wal") + failures.push(`journal_mode ${timing.pragmas.journalMode}`); + if (timing.pragmas.synchronous !== 2) + failures.push(`synchronous ${timing.pragmas.synchronous}`); + if ( + process.platform === "darwin" && + timing.pragmas.fullfsync !== expectedFull + ) + failures.push( + `fullfsync ${timing.pragmas.fullfsync}, wanted ${expectedFull}`, + ); + } + const control = await killLoop("T3-control", "append", Math.min(KILLS, 50), { + synchronous: "OFF", + }); + failures.push(...control.failures); + return { + name: "T3", + pass: failures.length === 0, + summary: `pragmas read back as set; commit p50 ${timings + .map( + (t) => + `${Number(t.p50).toFixed(2)} ms (fullfsync ${t.fullfsync ? "on" : "off"})`, + ) + .join( + " vs ", + )}; synchronous=OFF control: ${control.summary}. Power loss itself needs owner hardware.`, + details: { timings, control: control.details }, + failures, + }; +} + +// ── T4: quit with writes in flight ───────────────────────────────────── + +async function caseT4(runs: number): Promise { + const dir = mkdtempSync(join(tmpdir(), "soak-t4-")); + const docId = seedLibrary(dir, 64 * 1024); + const random = seededRandom(4); + const failures: string[] = []; + let clean = 0; + for (let i = 1; i <= runs; i += 1) { + const quitAfter = Math.round(300 + random() * 2700); + const run = await runHarness([ + "--mode=churn", + `--dir=${dir}`, + `--doc=${docId}`, + `--seed=${100 + i}`, + `--quit-after-ms=${quitAfter}`, + ]); + const quit = run.lines.find((l) => l.startsWith("QUIT ")); + if (run.exitCode !== 0 || !quit) { + failures.push( + `run ${i}: exit ${run.exitCode} ${run.stderr.slice(0, 500)}`, + ); + continue; + } + const [, outcome, kText] = quit.split(" "); + if (outcome === "clean") clean += 1; + else failures.push(`run ${i}: shutdown ${outcome}`); + if (existsSync(join(dir, LOCK_FILE))) + failures.push(`run ${i}: lock left behind`); + const wal = join(dir, `${LIBRARY_FILE}-wal`); + if (existsSync(wal) && statSync(wal).size > 0) + failures.push(`run ${i}: WAL not checkpointed (${statSync(wal).size} B)`); + const library = openEngine(dir); + if (library.status.uncleanExit) + failures.push(`run ${i}: unclean_exit still set`); + const k = editCount(textOf(library, docId).text); + library.shutdown(); + if (k !== Number(kText)) + failures.push(`run ${i}: ${k} edits stored, ${kText} typed before quit`); + } + rmSync(dir, { recursive: true, force: true }); + return { + name: "T4", + pass: failures.length === 0, + summary: `${runs} quits with edits in flight: ${clean} clean shutdowns, every typed edit stored`, + details: { runs, clean }, + failures, + }; +} + +// ── T5: migration interrupted ────────────────────────────────────────── + +function stepTables(dir: string) { + const db = new Database(join(dir, LIBRARY_FILE)); + const counts: Record = {}; + for (const step of T5_STEPS) { + const exists = db + .query("SELECT 1 FROM sqlite_master WHERE name = ?") + .get(`soak_step_${step.to}`); + if (exists) + counts[step.to] = ( + db.query(`SELECT COUNT(*) AS n FROM soak_step_${step.to}`).get() as { + n: number; + } + ).n; + } + const check = ( + db.query("PRAGMA quick_check").get() as { quick_check: string } + ).quick_check; + db.close(); + return { counts, check }; +} + +async function caseT5(kills: number): Promise { + const failures: string[] = []; + const reached: Record = {}; + // Calibrate: how long a full migration takes on this runner. + const calibrationDir = mkdtempSync(join(tmpdir(), "soak-t5-")); + seedLibrary(calibrationDir, 0); + const t0 = performance.now(); + const calibration = await runHarness([ + "--mode=migrate", + `--dir=${calibrationDir}`, + ]); + const fullMs = performance.now() - t0; + if (!calibration.lines.some((l) => l === `MIGRATED ${T5_TARGET} null`)) + failures.push( + `calibration: ${calibration.lines.join(" | ")} ${calibration.stderr.slice(0, 500)}`, + ); + rmSync(calibrationDir, { recursive: true, force: true }); + const random = seededRandom(5); + for (let i = 1; i <= kills; i += 1) { + const dir = mkdtempSync(join(tmpdir(), "soak-t5-")); + const docId = seedLibrary(dir, 4096); + const delay = random() * fullMs; + await runHarness(["--mode=migrate", `--dir=${dir}`], (line, kill) => { + if (line === "READY") setTimeout(kill, delay); + }); + // As an app at format 1 sees it: a newer library opens read-only. + const probe = openEngine(dir); + const version = probe.status.formatVersion; + probe.shutdown(); + reached[version] = (reached[version] ?? 0) + 1; + const { counts, check } = stepTables(dir); + if (check !== "ok") failures.push(`kill ${i}: quick_check ${check}`); + for (const step of T5_STEPS) { + const rows = counts[step.to]; + if (step.to <= version && rows !== STEP_ROWS) + failures.push( + `kill ${i}: v${version} but step ${step.to} has ${rows} rows`, + ); + if (step.to > version && rows !== undefined) + failures.push( + `kill ${i}: v${version} but step ${step.to}'s table exists`, + ); + } + // The migrating app finishes the job. + const library = openEngine(dir, { + formatVersion: T5_TARGET, + formatSteps: T5_STEPS, + }); + if (library.status.formatVersion !== T5_TARGET || library.status.readOnly) + failures.push( + `kill ${i}: resume reached v${library.status.formatVersion} ${library.status.readOnly}`, + ); + const pad = textOf(library, docId).padLength; + library.shutdown(); + if (pad !== 4096) failures.push(`kill ${i}: document damaged (${pad})`); + rmSync(dir, { recursive: true, force: true }); + } + return { + name: "T5", + pass: failures.length === 0, + summary: `${kills} kills during a 3-step migration (${Math.round(fullMs)} ms); stopped at v1–v4: ${JSON.stringify(reached)}; each step all-or-nothing, every resume reached v${T5_TARGET}`, + details: { kills, fullMs, reached }, + failures, + }; +} + +// ── T6: a newer library is read-only ─────────────────────────────────── + +async function caseT6(): Promise { + const dir = mkdtempSync(join(tmpdir(), "soak-t6-")); + seedLibrary(dir, 1024); + const db = new Database(join(dir, LIBRARY_FILE)); + db.exec("UPDATE meta SET value = '9' WHERE key = 'format_version'"); + db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); + db.close(); + const hash = () => + createHash("sha256") + .update(readFileSync(join(dir, LIBRARY_FILE))) + .digest("hex"); + const before = hash(); + const run = await runHarness(["--mode=readonly", `--dir=${dir}`]); + const after = hash(); + rmSync(dir, { recursive: true, force: true }); + const failures: string[] = []; + const status = run.lines.find((l) => l.startsWith("STATUS ")); + if (!status || JSON.parse(status.slice(7)).readOnly !== "library_too_new") + failures.push(`status: ${status}`); + if (!run.lines.includes("WRITE refused library_too_new")) + failures.push(`write: ${run.lines.find((l) => l.startsWith("WRITE"))}`); + if (!run.lines.includes("SIDEBAR 1")) + failures.push("the newer library's list didn't load"); + if (before !== after) failures.push("library.sqlite changed"); + if (run.exitCode !== 0) + failures.push(`exit ${run.exitCode} ${run.stderr.slice(0, 500)}`); + return { + name: "T6", + pass: failures.length === 0, + summary: + "a format-9 library opened read-only in the Electron worker: list shown, writes refused, file byte-identical", + details: { before, after }, + failures, + }; +} + +// ── T8: a year of editing, then startup with 1,000 documents ────────── + +/** Architecture §10 "Sidebar populated", with the 1.5× CI slack of §10. */ +function sidebarBudgetMs() { + return process.platform === "win32" ? 1000 : 600; +} + +async function caseT8(): Promise { + const docs = Number(args.docs ?? 1000); + const docBytes = Number(args["doc-bytes"] ?? 200 * 1024); + const days = Number(args.days ?? 365); + const dir = mkdtempSync(join(tmpdir(), "soak-t8-")); + const failures: string[] = []; + const clock = { now: Date.UTC(2026, 0, 1, 9) }; + const host = createNodeHost({ now: () => clock.now }); + const open = () => + Library.open({ dir, host, openSqlite: openBunSqlite, fullfsync: false }); + let library = open(); + const random = seededRandom(8); + const started = performance.now(); + + // Seed: 1,000 documents of about 200 KB in 30 folders, with tags. + const folders = Array.from( + { length: 30 }, + (_, i) => library.createFolder({ name: `Folder ${i}` }).id, + ); + const tags = Array.from( + { length: 20 }, + (_, i) => library.createTag(`tag-${i}`).id, + ); + const template = new Y.Doc(); + template.getText(PAD).insert(0, "p".repeat(docBytes)); + const templateState = Y.encodeStateAsUpdate(template); + const ids: string[] = []; + const createSeeded = () => { + const { id } = library.createDoc({ + title: `Document ${ids.length}`, + folderId: folders[Math.floor(random() * folders.length)] ?? null, + }); + library.load(id, "seed"); + library.compact(id, "seed", templateState, 0); + library.close(id, "seed"); + const picked = tags.filter(() => random() < 0.08); + if (picked.length) library.setDocTags(id, picked); + ids.push(id); + return id; + }; + for (let i = 0; i < docs; i += 1) createSeeded(); + const seededBytes = library.stats().bytes; + + // References for a few documents, checked at the end. + const watched = new Map( + ids.slice(0, 5).map((id) => [id, ""]), + ); + const monthly: Record[] = []; + let appends = 0; + let compactions = 0; + let trashed = 0; + + for (let day = 0; day < days; day += 1) { + const today = new Set( + [...watched.keys()].filter(() => random() < 0.3), + ); + while (today.size < 20) + today.add(ids[Math.floor(random() * ids.length)] as string); + for (const id of today) { + // An editing session: hydrate, type in batches, compact on close. + const loaded = library.load(id, "w"); + const doc = new Y.Doc(); + if (loaded.snapshot) Y.applyUpdate(doc, loaded.snapshot); + for (const u of loaded.updates) Y.applyUpdate(doc, u); + let acked = loaded.maxSeq; + let rowsSinceSnapshot = loaded.updates.length; + const batches = 10 + Math.floor(random() * 30); + for (let b = 0; b < batches; b += 1) { + let update: Uint8Array = new Uint8Array(); + doc.once("update", (u: Uint8Array) => { + update = u; + }); + const words = `w${day}-${b} `.repeat(30); + doc.getText(TEXT).insert(doc.getText(TEXT).length, words); + acked = library.append(id, "w", { + origin: "local", + update, + title: `Document ${id.slice(0, 4)}`, + }); + appends += 1; + rowsSinceSnapshot += 1; + clock.now += 1000; + if (rowsSinceSnapshot > 500) { + library.compact(id, "w", Y.encodeStateAsUpdate(doc), acked); + compactions += 1; + rowsSinceSnapshot = 0; + } + } + library.compact(id, "w", Y.encodeStateAsUpdate(doc), acked); + compactions += 1; + library.close(id, "w"); + if (watched.has(id)) watched.set(id, doc.getText(TEXT).toString()); + } + // One new document a day, and one old one trashed (purged after 30 days). + createSeeded(); + const victim = ids.splice( + 5 + Math.floor(random() * (ids.length - 6)), + 1, + )[0]; + if (victim) { + library.trashDoc(victim); + trashed += 1; + } + // Idle: the daily copy and the Trash purge. + clock.now += IDLE_MS; + library.runIdleJobs(); + // Restart now and then, sometimes uncleanly. + if (day % 30 === 29) { + if (random() < 0.5) library.shutdown(); + else (library as unknown as { db: { close(): void } }).db.close(); + library = open(); + if ( + library.status.quickCheck !== null && + library.status.quickCheck !== "ok" + ) + failures.push(`day ${day}: quick_check ${library.status.quickCheck}`); + } + if (day % 30 === 0 || day === days - 1) { + const stats = library.stats(); + monthly.push({ + day, + mb: +(stats.bytes / 1048576).toFixed(1), + docs: stats.docs, + pendingRows: stats.pendingRows, + dailyCopies: library.listDailyCopies().length, + }); + } + clock.now = Date.UTC(2026, 0, 2 + day, 9); + } + + const stats = library.stats(); + const dailies = library.listDailyCopies(); + if (dailies.length !== Math.min(DAILY_KEEP, days)) + failures.push(`${dailies.length} daily copies kept, wanted ${DAILY_KEEP}`); + if (stats.pendingRows !== 0) + failures.push(`${stats.pendingRows} rows left uncompacted`); + for (const [id, expected] of watched) { + if (!ids.includes(id)) continue; + const { text } = textOf(library, id); + if (text !== expected) failures.push(`watched document ${id} lost text`); + } + // Growth: the year's text is small next to the seed; the library must not + // keep history it no longer needs. + const growth = stats.bytes / seededBytes; + if (growth > 1.5) + failures.push(`library grew ${growth.toFixed(2)}× in a year`); + const liveDocs = library.sidebar().docs.length; + library.shutdown(); + const simMs = performance.now() - started; + + // Startup with the year-end library: worker ready and sidebar rows. + // `--no-electron` checks the simulation alone (local runs). + const startups: Record[] = []; + for (let i = 0; i < (args["no-electron"] ? 0 : 5); i += 1) { + const run = await runHarness(["--mode=startup", `--dir=${dir}`]); + const line = run.lines.find((l) => l.startsWith("STARTUP ")); + if (!line) { + failures.push(`startup ${i}: ${run.stderr.slice(0, 500)}`); + continue; + } + startups.push(JSON.parse(line.slice(8))); + } + const budget = sidebarBudgetMs(); + const median = (key: string) => { + const values = startups.map((s) => Number(s[key])).sort((a, b) => a - b); + return values[Math.floor(values.length / 2)] ?? Number.NaN; + }; + const spawnToSidebar = median("spawnToSidebarMs"); + if (!args["no-electron"] && !(spawnToSidebar <= budget * 1.5)) + failures.push( + `sidebar ${spawnToSidebar.toFixed(0)} ms after worker spawn, budget ${budget} ms (× 1.5 CI slack)`, + ); + if (startups.some((s) => s.docs !== liveDocs)) + failures.push( + `startup listed ${startups.map((s) => s.docs)} documents, wanted ${liveDocs}`, + ); + rmSync(dir, { recursive: true, force: true }); + return { + name: "T8", + pass: failures.length === 0, + summary: `${days} days, ${appends} appends, ${compactions} compactions, ${trashed} trashed; ${(seededBytes / 1048576).toFixed(0)} → ${(stats.bytes / 1048576).toFixed(0)} MB; ${dailies.length} daily copies; ${liveDocs} documents listed ${spawnToSidebar.toFixed(0)} ms after worker spawn (median of 5; budget ${budget} ms)`, + details: { + days, + docs: liveDocs, + appends, + compactions, + trashed, + seededBytes, + finalBytes: stats.bytes, + monthly, + startups, + budgetMs: budget, + simMs, + }, + failures, + }; +} + +// ── Main ──────────────────────────────────────────────────────────────── + +const CASES: Record Promise> = { + contract: caseContract, + T1: () => killLoop("T1", "append", KILLS), + "T1-dense": () => + killLoop("T1-dense", "append", Math.ceil(KILLS / 2), { dense: true }), + T2: () => killLoop("T2", "compaction", KILLS), + T3: caseT3, + T4: () => caseT4(Number(args.quits ?? 20)), + T5: () => caseT5(Number(args["migration-kills"] ?? 40)), + T6: caseT6, + T8: caseT8, +}; + +const selected = + !args.case || args.case === "all" + ? Object.keys(CASES) + : args.case.split(",").map((name) => name.trim()); + +const results: CaseResult[] = []; +for (const name of selected) { + const run = CASES[name]; + if (!run) throw new Error(`unknown case ${name}`); + const t0 = performance.now(); + let result: CaseResult; + try { + result = await run(); + } catch (error) { + result = { + name, + pass: false, + summary: "crashed", + details: {}, + failures: [String(error instanceof Error ? error.stack : error)], + }; + } + result.details.seconds = Math.round((performance.now() - t0) / 1000); + results.push(result); + writeFileSync(join(OUT, `${name}.json`), JSON.stringify(result, null, 2)); + console.log(`${result.pass ? "PASS" : "FAIL"} ${name}: ${result.summary}`); + for (const failure of result.failures.slice(0, 10)) + console.log(` - ${failure}`); +} + +const table = [ + `### Desktop storage soak (${process.platform}-${process.arch})`, + "", + "| Case | Result | Summary |", + "|---|---|---|", + ...results.map( + (r) => + `| ${r.name} | ${r.pass ? "pass" : "**fail**"} | ${r.summary.replace(/\|/g, "\\|")} |`, + ), + "", +].join("\n"); +if (process.env.GITHUB_STEP_SUMMARY) + appendFileSync(process.env.GITHUB_STEP_SUMMARY, table); +process.exit(results.every((r) => r.pass) ? 0 : 1); From 0ba711a6e7f7fa265d3ae02f954f9b07d027910f Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:37:51 +0330 Subject: [PATCH 09/19] ci(desktop): run the full storage soak on a labelled pull request A pull request labelled storage-soak-full runs 200 kills per case, as the nightly does; workflow_dispatch only works once the file is on main. --- .github/workflows/desktop-storage-soak.yml | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/desktop-storage-soak.yml b/.github/workflows/desktop-storage-soak.yml index 861b93fd..517a042c 100644 --- a/.github/workflows/desktop-storage-soak.yml +++ b/.github/workflows/desktop-storage-soak.yml @@ -7,11 +7,14 @@ name: Desktop storage soak # bun:sqlite. See tests/load/desktop-durability-soak.ts. # # Pull requests that touch storage run a short profile (20 kills per case); -# the nightly and manual runs use the plan's 200. Not a required check. +# the nightly and manual runs use the plan's 200, and so does a pull request +# labelled `storage-soak-full` (workflow_dispatch needs the file on main). +# Not a required check. # Read-only token and no secrets, like CI. on: pull_request: + types: [opened, synchronize, reopened, labeled] paths: - desktop/storage/** - desktop/storage-harness/** @@ -62,7 +65,7 @@ jobs: run: shell: bash env: - KILLS: ${{ github.event_name == 'pull_request' && 20 || inputs.kills || 200 }} + KILLS: ${{ github.event_name != 'pull_request' && (inputs.kills || 200) || contains(github.event.pull_request.labels.*.name, 'storage-soak-full') && 200 || 20 }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 From 3a6cb911423a67b6616b809b91f74f06ba459242 Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:38:51 +0330 Subject: [PATCH 10/19] docs(desktop): record the P2.6 storage worker in the changelog and spec The architecture spec now lists where the engine, client and soak harness live, the open order as built, the worker-side registry check and claim, the compaction guards, the import guard, the lib/schema-names.ts boundary allowance, and that T7 belongs to the web library. --- CHANGELOG.md | 57 +++++++++++++++++++ .../specs/2026-11-15-desktop-architecture.md | 17 +++++- 2 files changed, 72 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4df91f15..d4dd7a81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,63 @@ since the previous tag. ## [Unreleased] +### Desktop P2.6: the library storage worker + +The SQLite engine behind the desktop library, where documents, folders and +tags live, and the `worker_thread` that runs it inside Electron main (D-A1). +Nothing in the web app or its build uses it yet; P2.5's shell starts the +worker and P2.7 builds the files sidebar on it. + +#### Added + +- **Desktop:** `desktop/storage/engine/`, with no `node:`, `bun:` or + Electron imports, behind a `SqliteDriver` seam (`node:sqlite` in the + worker, `bun:sqlite` in tests and the dev host): + - the format-1 schema, so the Local release needs no later migration: + `docs` (lineage and a `NOT NULL` `link_key` minted at insert, `kind`, the + list columns), `folders`, `tags`, `doc_tags`, `doc_view(doc_id, width, + place, remote_images)`, `import_sources`, `snapshots` and `updates` with + an `AUTOINCREMENT` seq; + - the open order: cloud-synced folders refused, `library.lock` with a pid + check, WAL with `synchronous=FULL` (`fullfsync` on macOS), and the + `format_version` guard before any write, so a newer library opens + read-only and untouched; `quick_check` after an unclean exit; + - format migrations with a `VACUUM INTO` copy first (2 kept), one + transaction per step, and read-only on failure; + - appends and renderer-driven compaction in one transaction each, with + one window per document (`already_open`, `forbidden`, `document_open`); + - documents, folders, tags, Trash (emptied after 30 days) and the sidebar + snapshot with `rev`; + - a daily `VACUUM INTO` copy on the first idle moment, keeping 7, and a + reader for restoring one document from a day; + - shutdown for the quit flush: WAL checkpoint, `unclean_exit` cleared, + lock released. +- **Desktop:** `desktop/storage/worker.ts` and `client.ts`: the worker + answers main one request at a time with `Result` envelopes and transferred + bytes; the client rejects pending calls with `worker_restarting` if the + worker dies and ends with a 2 s shutdown. +- **Platform:** `lib/doc-migrations/` (the open-time migration hook list, + empty so far, and `checkImport`, which refuses an import that is too + large, isn't Yjs, or carries a node, mark or root this build doesn't + know) and `lib/platform/desktop/rpc-doc-store.ts` (`DocStore` over worker + replies, for `IpcDocStore`). +- **Tests:** engine suites on `bun:sqlite` (open, content, index, daily + copies, codes, and a property test of random appends, compactions, + crashes and reopens), the import-boundary test's storage rows, and + `tests/load/desktop-durability-soak.ts` with its Electron harness in + `desktop/storage-harness/` for soak cases T1–T6 and T8. +- **CI:** `desktop-storage-soak.yml` on macOS, Windows and Linux: 20 kills + per case on pull requests that touch storage, 200 nightly or with the + `storage-soak-full` label. `bun run typecheck` also checks the new + `tsconfig.desktop-node.json`. + +#### Changed + +- **Docs:** the desktop architecture spec records where P2.6 put the engine + and the harness, the worker-side registry check and `claim`, the + import-boundary allowance for `lib/schema-names.ts`, and that T7 belongs to + the web library. + ### Desktop P2.1a: `DocSession` and `DocStore` port types The first half of desktop plan P2.1: the types the editor, the storage worker diff --git a/docs/superpowers/specs/2026-11-15-desktop-architecture.md b/docs/superpowers/specs/2026-11-15-desktop-architecture.md index 139e7560..b114fd27 100644 --- a/docs/superpowers/specs/2026-11-15-desktop-architecture.md +++ b/docs/superpowers/specs/2026-11-15-desktop-architecture.md @@ -126,8 +126,13 @@ desktop/ main/ # index, windows, app-protocol, csp, doc-handlers, quit-coordinator, menus, # dialogs, deep-links, backups, updater (lazy), diagnostics storage/worker.ts # worker_thread entry + storage/client.ts # main's handle on the worker: start, call, shutdown (P2.6) + storage/protocol.ts # request and reply types, dispatch (no node:* imports) + storage/node-host.ts # the StorageHost (fs, pid check, clock) the engine is given storage/engine/ # SQL logic over a SqliteDriver; no electron, node:* or bun:* imports storage/drivers/node-sqlite.ts + storage-harness/ # the soak's Electron main (own lockfile), like smoke/ (P2.6) + dev/bun-sqlite-driver.ts # the SqliteDriver over bun:sqlite, for tests and the dev host dev/storage-host.ts # dev and test only: doc-handlers + engine on bun:sqlite over a localhost WebSocket build/build.ts, build/electron-builder.yml ``` @@ -167,7 +172,7 @@ These are enforced by one test, `tests/unit/ci/import-boundaries.test.ts`. It re |---|---| | `desktop/shared` | anything outside itself | | `desktop/storage/engine` | `electron`, `node:*`, `bun:*`, React, `@tiptap/*` | -| `desktop/main`, `desktop/storage` | React, `lib/**` (except `lib/security/content-security-policy.ts`, `lib/doc-migrations`, `lib/prefs/schema.ts`, and the pure `lib/comments/validate.ts` and `lib/assets` sniffers) | +| `desktop/main`, `desktop/storage` | React, `lib/**` (except `lib/security/content-security-policy.ts`, `lib/doc-migrations`, `lib/prefs/schema.ts`, the pure `lib/comments/validate.ts` and `lib/assets` sniffers, and `lib/schema-names.ts`, the name lists the import guard checks; P2.6) | | `desktop/preload` | anything except `desktop/shared` and `electron` | | `lib/platform/desktop` | `desktop/{main,preload,storage}` | | `app`, `components`, `hooks`, `lib` | `desktop/**` except `desktop/shared` types; `lib/platform/desktop/**` except from `components/desktop/desktop-root.tsx` | @@ -289,6 +294,14 @@ INDEX import_sources(source_hash) | `assets.ts` (P2.6b) | Owns `/assets/`: the temp-fsync-rename write order, dedupe, the startup `tmp/` sweep, `recount` and GC with a 30-day grace (image store §3.2–§3.4). | | `jobs.ts` | One queue. Foreground calls run first; daily copy and mirror run on idle, meaning 30 s without an append. Each job yields between documents. | +**As built in P2.6.** +- **Modules.** `index.ts` is `library-index.ts` (not a barrel). `library.ts` holds the open order, the registry and the lifecycle; `lock.ts`, `cloud-folder.ts`, `sql.ts`, `driver.ts` (the `SqliteDriver` seam and SQLite error mapping) and `host.ts` (an injected `StorageHost` for files, the pid check and the clock) keep the engine free of `node:*`. `jobs.ts` is two lines in the worker: idle jobs run between requests after 30 s without a write. `mirror.ts`, `filenames.ts` and `assets.ts` come with P2.10 and P2.6b. +- **Open order.** Cloud-folder refusal, lock, pragmas, schema creation for a new file, then `format_version` before any other write (a newer library is reopened read-only and never written), then `quick_check` if `unclean_exit` was set (a failure is read-only, `corrupt`), then migrations, then `unclean_exit` is set. +- **Registry in the worker too.** `load` claims a document for a window; another window's `load` is `already_open`; an append or compaction from any window that doesn't hold the document is `forbidden`, as §7 requires of main; purge (and P2.8's import-into and restore) is `document_open` while any window holds it. After a worker restart, main replays its registry with `claim(docId, owner)`. +- **Compaction guards.** A compaction older than the stored snapshot is a no-op. An `upToSeq` past the document's last row is `invalid_argument`. `updates.seq` is `AUTOINCREMENT`, so a seq is never reused after the newest rows are compacted away. +- **`docs.schema`** defaults to 0, which is `DOC_SCHEMA` while `lib/doc-migrations` lists no steps. +- **Import guard.** `checkImport` (`lib/doc-migrations/import-guard.ts`) refuses a state over 64 MiB (`import_too_large`), bytes that aren't Yjs (`corrupt`), and nodes, marks or roots the build doesn't know (`unknown_schema`; legacy `link` and `code` pass, the import migrations rename them; roots: `default`, plus `comments` in P4.11). P2.8 calls it before anything is written. + **Backup failures.** EPERM, EACCES and Defender blocks on the backup folder raise app health `backups_failing` (§6.1; plan §5.5 "Backups failing"). The library itself is never affected. --- @@ -575,7 +588,7 @@ sequenceDiagram | Compaction invariant | property test: random appends, compactions and reopens, with the Y.Doc state compared against a reference | engine on `bun:sqlite` | | Guards and `doc-handlers` | fuzz plus golden messages, sender and registry checks | pure modules | | Reducer and chip | a table row for each plan §5.5 state; "every refusal ends in a stop or a backoff" | pure | -| Durability T1–T8, T16–T18 | `tests/load/desktop-durability-soak.ts`: SIGKILL Electron during append, compaction, migration, asset writes and image migration; a year of image churn | packaged app | +| Durability T1–T8, T16–T18 | `tests/load/desktop-durability-soak.ts`: SIGKILL Electron during append, compaction, migration, asset writes and image migration; a year of image churn | the storage worker in a windowless Electron (`desktop/storage-harness`, P2.6), verified on `bun:sqlite`; the packaged app once P2.5 lands. T7 (slow hydrate) is the web library's (W2): it tests the session layer, not storage. | | `BlobStore` contract and asset GC | one suite plus a property test: GC never deletes a referenced blob | `MemoryBlobStore`; the worker on `bun:sqlite`; `IdbBlobStore` in the web library | | Preferences boot script | parity with `resolveBoot()` over about 40 stored payloads; the desktop CSP hash matches the script (build test) | happy-dom; build | | Main | `argv` filter, navigation and `window.open` denial, fuses | `_electron` | From 207a5e7258dd2d3b1467ee83ab8bfa6dd893c1fd Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:47:24 +0330 Subject: [PATCH 11/19] fix(desktop): don't take a home folder named like a sync client for one A person whose user name is box or dropbox had their library refused as a cloud folder. The check now skips the home folder's own segments. --- desktop/storage/engine/cloud-folder.ts | 10 ++++++- .../storage/cloud-folder-and-codes.test.ts | 28 ++++++++++++++++++- 2 files changed, 36 insertions(+), 2 deletions(-) diff --git a/desktop/storage/engine/cloud-folder.ts b/desktop/storage/engine/cloud-folder.ts index 32401886..a87b1387 100644 --- a/desktop/storage/engine/cloud-folder.ts +++ b/desktop/storage/engine/cloud-folder.ts @@ -58,16 +58,24 @@ function isUnder(path: string, root: string, caseInsensitive: boolean) { /** The cloud provider syncing `dir`, or null. Symlinks are resolved first. */ export function cloudFolderOf( dir: string, - host: Pick, + host: Pick, ): CloudProvider | null { const resolved = host.realpath(dir); const caseInsensitive = host.platform !== "linux"; + // The home folder's own segments are a user name, not a sync folder: a + // person called "box" or "dropbox" must still be able to start the app. + const home = host.homeDir ? host.realpath(host.homeDir) : ""; + const skip = + home && isUnder(resolved, home, caseInsensitive) + ? segmentsOf(home).length + : 0; for (const name of ONEDRIVE_ENV) { const root = host.env[name]; if (root && isUnder(resolved, root, caseInsensitive)) return "onedrive"; } const parts = segmentsOf(resolved); for (const [i, part] of parts.entries()) { + if (i < skip) continue; if (part.toLowerCase() === "cloudstorage") { const next = parts[i + 1] ?? ""; for (const [pattern, provider] of CLOUD_STORAGE) diff --git a/tests/unit/desktop/storage/cloud-folder-and-codes.test.ts b/tests/unit/desktop/storage/cloud-folder-and-codes.test.ts index c1caed2e..7b522ba0 100644 --- a/tests/unit/desktop/storage/cloud-folder-and-codes.test.ts +++ b/tests/unit/desktop/storage/cloud-folder-and-codes.test.ts @@ -9,13 +9,39 @@ import { UPDATE_ORIGINS, } from "@/lib/platform/doc-store"; -const host = (platform: string, env: Record = {}) => ({ +const host = ( + platform: string, + env: Record = {}, + homeDir = "", +) => ({ platform, env, + homeDir, realpath: (path: string) => path, }); describe("cloudFolderOf", () => { + test("a user name that looks like a provider is not a sync folder", () => { + expect( + cloudFolderOf( + "/home/box/.config/TinyDocy", + host("linux", {}, "/home/box"), + ), + ).toBeNull(); + expect( + cloudFolderOf( + "/Users/dropbox/Library/Application Support/TinyDocy", + host("darwin", {}, "/Users/dropbox"), + ), + ).toBeNull(); + expect( + cloudFolderOf("/home/box/Dropbox/T", host("linux", {}, "/home/box")), + ).toBe("dropbox"); + expect( + cloudFolderOf("D:\\Dropbox\\TinyDocy", host("win32", {}, "C:\\Users\\a")), + ).toBe("dropbox"); + }); + const cases: [string, string, Record, string | null][] = [ ["darwin", "/Users/a/Library/Application Support/TinyDocy", {}, null], [ From 26b3bfd865a42c9954246e5df4a7553449a63329 Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:47:24 +0330 Subject: [PATCH 12/19] fix(desktop): announce every sidebar change in the index feed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Appends change a document's title, snippet and modified time, but did not move rev, and opening a document or the idle Trash purge moved rev without an event. The library now records each changed row, appends bump rev in their transaction, and the worker sends index_changed with the rev and the changed rows after every request and idle pass (architecture §7, §8.1). A failed daily copy is kept for stats instead of dropped. --- desktop/storage/client.ts | 5 +- desktop/storage/engine/content.ts | 4 +- desktop/storage/engine/library-index.ts | 19 ++--- desktop/storage/engine/library.ts | 79 ++++++++++++++++--- desktop/storage/engine/sql.ts | 12 +++ desktop/storage/protocol.ts | 37 ++------- desktop/storage/worker.ts | 15 +++- tests/helpers/engine-doc-store.ts | 2 +- .../desktop/storage/doc-store-engine.test.ts | 73 +++++++++++++---- 9 files changed, 168 insertions(+), 78 deletions(-) diff --git a/desktop/storage/client.ts b/desktop/storage/client.ts index 4373bde7..42129d82 100644 --- a/desktop/storage/client.ts +++ b/desktop/storage/client.ts @@ -12,6 +12,7 @@ import { Worker } from "node:worker_threads"; import type { StorageErrorEnvelope } from "./engine/errors"; import type { LibraryStatus } from "./engine/library"; +import type { IndexChange } from "./engine/library-index"; import { type StorageMessage, type StorageOp, @@ -34,7 +35,7 @@ export class StorageCallError extends Error { type Pending = { resolve(value: unknown): void; reject(error: Error): void }; export type StorageClientEvents = { - onIndexChanged?: (rev: number) => void; + onIndexChanged?: (rev: number, changes: IndexChange[]) => void; /** The worker exited without being asked to (crash). */ onExit?: (code: number) => void; }; @@ -65,7 +66,7 @@ export class StorageClient { if (message.result.ok) pending.resolve(message.result.value); else pending.reject(new StorageCallError(message.result.error)); } else if (message.type === "index_changed") { - events.onIndexChanged?.(message.rev); + events.onIndexChanged?.(message.rev, message.changes); } }); worker.on("exit", (code) => { diff --git a/desktop/storage/engine/content.ts b/desktop/storage/engine/content.ts index 73b16b59..b45ed2b5 100644 --- a/desktop/storage/engine/content.ts +++ b/desktop/storage/engine/content.ts @@ -12,7 +12,7 @@ import * as Y from "yjs"; import type { SqliteDriver } from "./driver"; import { StorageError } from "./errors"; -import { scalar, transaction } from "./sql"; +import { bumpRev, scalar, transaction } from "./sql"; /** The stored `updates.origin` values: `UPDATE_ORIGINS` in `lib/platform/doc-store.ts`. */ export const STORED_ORIGINS = [ @@ -140,6 +140,8 @@ export function append( ); if (updated.changes === 0) throw new StorageError("not_found", "no such document"); + // Title, snippet and modified time are on the sidebar (architecture §8.1). + bumpRev(db); return db .prepare( "INSERT INTO updates (doc_id, data, origin, created_at) VALUES (?, ?, ?, ?)", diff --git a/desktop/storage/engine/library-index.ts b/desktop/storage/engine/library-index.ts index 7ce23e65..e7ed44f7 100644 --- a/desktop/storage/engine/library-index.ts +++ b/desktop/storage/engine/library-index.ts @@ -11,7 +11,7 @@ */ import type { SqliteDriver, SqlRow } from "./driver"; import { StorageError } from "./errors"; -import { randomHex32, readMeta, scalar, transaction, writeMeta } from "./sql"; +import { bumpRev, randomHex32, readRev, scalar, transaction } from "./sql"; export type DocKind = "doc" | "template"; @@ -63,6 +63,12 @@ export type SidebarDoc = { export type SidebarTag = { id: string; name: string; color: string | null }; +/** + * One changed row, for the change feed (architecture §7 `index.changed`): + * a window patches that row instead of re-reading the whole library. + */ +export type IndexChange = { kind: "doc" | "folder" | "tag"; id: string }; + /** Everything the files sidebar and lists paint from (P2.7a/b). */ export type SidebarSnapshot = { rev: number; @@ -86,17 +92,6 @@ function cleanName(name: string): string { return name.trim().slice(0, MAX_NAME_LENGTH); } -export function readRev(db: SqliteDriver): number { - return Number.parseInt(readMeta(db, "rev") ?? "0", 10); -} - -/** Bumps `meta.rev`; call inside the write's transaction. */ -function bumpRev(db: SqliteDriver): number { - const rev = readRev(db) + 1; - writeMeta(db, "rev", String(rev)); - return rev; -} - /** Wraps a metadata write: one transaction, `rev` bumped, new rev returned. */ function indexWrite(db: SqliteDriver, body: () => void): number { return transaction(db, () => { diff --git a/desktop/storage/engine/library.ts b/desktop/storage/engine/library.ts index fc23116c..e36025d9 100644 --- a/desktop/storage/engine/library.ts +++ b/desktop/storage/engine/library.ts @@ -58,7 +58,7 @@ import { runFormatSteps, } from "./migrations"; import { FORMAT_VERSION, SCHEMA_V1, YJS_ENCODING } from "./schema"; -import { readMeta, scalar, transaction, writeMeta } from "./sql"; +import { readMeta, readRev, scalar, transaction, writeMeta } from "./sql"; export const LIBRARY_FILE = "library.sqlite"; /** Idle jobs (daily copy, Trash purge) wait for this long without a write. */ @@ -153,6 +153,8 @@ export class Library { private readonly holders = new Map(); private readonly trace: Trace | undefined; private lastWriteAt: number; + private changes = new Map(); + private lastDailyCopyError: string | null = null; private constructor(fields: { dir: string; @@ -289,6 +291,22 @@ export class Library { this.lastWriteAt = this.host.now(); } + private changed(kind: index.IndexChange["kind"], id: string) { + this.changes.set(`${kind}:${id}`, { kind, id }); + } + + /** + * The rows changed since the last call, with the current `rev`, or null + * when nothing changed. The worker drains this after every request and + * idle pass and sends `index_changed` (architecture §7, §8.1). + */ + takeIndexChanges(): { rev: number; changes: index.IndexChange[] } | null { + if (this.changes.size === 0 || !this.db) return null; + const changes = [...this.changes.values()]; + this.changes.clear(); + return { rev: readRev(this.conn), changes }; + } + // ── Content (DocStore) ──────────────────────────────────────────────── load(docId: string, owner: string): LoadedContent { @@ -300,6 +318,7 @@ export class Library { if (!this.status.readOnly) { try { index.markOpened(this.conn, docId, this.host.now()); + this.changed("doc", docId); } catch { // "Opened" sorting is a nicety; opening must not fail on it. } @@ -314,6 +333,7 @@ export class Library { const seq = append(this.conn, docId, input, this.host.now()); this.trace?.(`ACK ${seq}`); this.wrote(); + this.changed("doc", docId); return seq; } @@ -376,6 +396,7 @@ export class Library { this.assertWritable(); const created = index.createDoc(this.conn, input, this.host.now()); this.wrote(); + this.changed("doc", created.id); return created; } @@ -383,27 +404,36 @@ export class Library { this.assertWritable(); const created = index.createFolder(this.conn, input, this.host.now()); this.wrote(); + this.changed("folder", created.id); return created; } renameDoc(id: string, title: string): number { this.assertWritable(); - return index.renameDoc(this.conn, id, title); + const rev = index.renameDoc(this.conn, id, title); + this.changed("doc", id); + return rev; } moveDoc(id: string, folderId: string | null, sortKey: string): number { this.assertWritable(); - return index.moveDoc(this.conn, id, folderId, sortKey); + const rev = index.moveDoc(this.conn, id, folderId, sortKey); + this.changed("doc", id); + return rev; } trashDoc(id: string, group: string | null = null): number { this.assertWritable(); - return index.trashDoc(this.conn, id, this.host.now(), group); + const rev = index.trashDoc(this.conn, id, this.host.now(), group); + this.changed("doc", id); + return rev; } restoreDoc(id: string): number { this.assertWritable(); - return index.restoreDoc(this.conn, id); + const rev = index.restoreDoc(this.conn, id); + this.changed("doc", id); + return rev; } purgeDoc(id: string): number { @@ -411,17 +441,22 @@ export class Library { this.assertClosed(id); const rev = index.purgeDoc(this.conn, id); this.wrote(); + this.changed("doc", id); return rev; } createTag(name: string, color: string | null = null) { this.assertWritable(); - return index.createTag(this.conn, name, color); + const created = index.createTag(this.conn, name, color); + this.changed("tag", created.id); + return created; } setDocTags(docId: string, tagIds: readonly string[]): number { this.assertWritable(); - return index.setDocTags(this.conn, docId, tagIds); + const rev = index.setDocTags(this.conn, docId, tagIds); + this.changed("doc", docId); + return rev; } // ── Restore points ──────────────────────────────────────────────────── @@ -452,17 +487,36 @@ export class Library { /** * Runs the idle jobs when nothing was written for {@link IDLE_MS}: today's * daily copy, then Trash older than 30 days (skipping open documents). - * Returns what ran, for the worker's diagnostics. + * Returns what ran, for the worker's diagnostics. A failed daily copy is + * kept for `stats()` (app health `backups_failing`) and retried on the + * next idle pass; it never stops the purge. */ - runIdleJobs(): { dailyCopy: string | null; purged: number } { + runIdleJobs(): { + dailyCopy: string | null; + dailyCopyMs: number | null; + purged: number; + } { const now = this.host.now(); - const result = { dailyCopy: null as string | null, purged: 0 }; + const result = { + dailyCopy: null as string | null, + dailyCopyMs: null as number | null, + purged: 0, + }; if (this.status.readOnly || now - this.lastWriteAt < IDLE_MS) return result; - if (isDailyCopyDue(this.conn, now)) - result.dailyCopy = takeDailyCopy(this.conn, this.host, this.dir, now); + if (isDailyCopyDue(this.conn, now)) { + const started = performance.now(); + try { + result.dailyCopy = takeDailyCopy(this.conn, this.host, this.dir, now); + this.lastDailyCopyError = null; + } catch (error) { + this.lastDailyCopyError = storageErrorFrom(error).code; + } + result.dailyCopyMs = performance.now() - started; + } for (const id of index.expiredTrash(this.conn, now)) { if (this.holders.has(id)) continue; index.purgeDoc(this.conn, id); + this.changed("doc", id); result.purged += 1; } return result; @@ -487,6 +541,7 @@ export class Library { ), ), lastDailyCopy: readMeta(db, "last_daily_copy"), + lastDailyCopyError: this.lastDailyCopyError, today: localDate(this.host.now()), // Read back, as set (D-A6; S2 confirmed them on all three OSes). pragmas: { diff --git a/desktop/storage/engine/sql.ts b/desktop/storage/engine/sql.ts index 74b6bf14..c0b334a5 100644 --- a/desktop/storage/engine/sql.ts +++ b/desktop/storage/engine/sql.ts @@ -46,6 +46,18 @@ export function writeMeta(db: SqliteDriver, key: string, value: string) { ).run(key, value); } +/** `meta.rev`: moves on every change the files sidebar shows. */ +export function readRev(db: SqliteDriver): number { + return Number.parseInt(readMeta(db, "rev") ?? "0", 10); +} + +/** Bumps `meta.rev`; call inside the write's transaction. */ +export function bumpRev(db: SqliteDriver): number { + const rev = readRev(db) + 1; + writeMeta(db, "rev", String(rev)); + return rev; +} + /** 32 lowercase hex characters from the platform CSPRNG (`link_key`). */ export function randomHex32(): string { const bytes = new Uint8Array(16); diff --git a/desktop/storage/protocol.ts b/desktop/storage/protocol.ts index f49d8cd4..1d238cd3 100644 --- a/desktop/storage/protocol.ts +++ b/desktop/storage/protocol.ts @@ -15,6 +15,7 @@ import type { Library, LibraryStatus } from "./engine/library"; import type { CreateDocInput, CreatedDoc, + IndexChange, SidebarSnapshot, } from "./engine/library-index"; @@ -59,7 +60,7 @@ export type StorageOps = { readDailyCopy: { args: { date: string; docId: string }; result: Uint8Array }; runIdleJobs: { args: Record; - result: { dailyCopy: string | null; purged: number }; + result: ReturnType; }; stats: { args: Record; result: ReturnType }; /** The quit flush's last step: checkpoint, clear `unclean_exit`, close. */ @@ -83,7 +84,7 @@ export type StorageMessage = | { type: "open_failed"; error: StorageErrorEnvelope } | { type: "reply"; id: number; result: Result } /** Metadata changed (architecture §7 `index.changed`); main fans it out. */ - | { type: "index_changed"; rev: number }; + | { type: "index_changed"; rev: number; changes: IndexChange[] }; /** Options the worker is started with (`workerData`). */ export type StorageWorkerOptions = { @@ -96,23 +97,6 @@ export type StorageWorkerOptions = { idlePollMs?: number; }; -/** Ops whose result is the new `rev`: each one emits `index_changed`. */ -export const REV_OPS: ReadonlySet = new Set([ - "renameDoc", - "moveDoc", - "trashDoc", - "restoreDoc", - "purgeDoc", - "setDocTags", -]); - -/** Ops whose result carries `rev` in an object. */ -const CREATE_OPS: ReadonlySet = new Set([ - "createDoc", - "createFolder", - "createTag", -]); - function run(library: Library, request: StorageRequest): unknown { switch (request.op) { case "load": @@ -181,22 +165,15 @@ function run(library: Library, request: StorageRequest): unknown { } } -/** - * Runs one request. Returns the reply, and the new `rev` when the request - * changed metadata (the caller emits `index_changed`). - */ +/** Runs one request; a failure becomes an envelope, never a throw. */ export function dispatch( library: Library, request: StorageRequest, -): { result: Result; rev: number | null } { +): Result { try { - const value = run(library, request); - let rev: number | null = null; - if (REV_OPS.has(request.op)) rev = value as number; - else if (CREATE_OPS.has(request.op)) rev = (value as { rev: number }).rev; - return { result: { ok: true, value }, rev }; + return { ok: true, value: run(library, request) }; } catch (error) { - return { result: { ok: false, error: toEnvelope(error) }, rev: null }; + return { ok: false, error: toEnvelope(error) }; } } diff --git a/desktop/storage/worker.ts b/desktop/storage/worker.ts index da84107d..f4020017 100644 --- a/desktop/storage/worker.ts +++ b/desktop/storage/worker.ts @@ -66,6 +66,12 @@ try { post({ type: "open_failed", error: toEnvelope(error) }); } +/** Sends `index_changed` if anything the sidebar shows moved. */ +function announceChanges(open: Library) { + const changed = open.takeIndexChanges(); + if (changed) post({ type: "index_changed", ...changed }); +} + let idleTimer: ReturnType | null = null; if (library) { const open = library; @@ -73,8 +79,9 @@ if (library) { try { open.runIdleJobs(); } catch { - // A failed daily copy is reported through stats; the library is fine. + // Only the Trash purge can throw here; it retries on the next pass. } + announceChanges(open); }, options.idlePollMs ?? 5000); idleTimer.unref(); } @@ -95,12 +102,14 @@ port.on("message", (request: StorageRequest) => { }); return; } - const { result, rev } = dispatch(library, request); + const open = library; + const result = dispatch(open, request); post({ type: "reply", id: request.id, result }); - if (rev !== null) post({ type: "index_changed", rev }); if (request.op === "shutdown") { if (idleTimer) clearInterval(idleTimer); library = null; port.close(); + return; } + announceChanges(open); }); diff --git a/tests/helpers/engine-doc-store.ts b/tests/helpers/engine-doc-store.ts index c3f0d345..3dd5c287 100644 --- a/tests/helpers/engine-doc-store.ts +++ b/tests/helpers/engine-doc-store.ts @@ -40,7 +40,7 @@ export function createEngineDocStore( } as StorageRequest; // A microtask boundary, as a real transport has. await Promise.resolve(); - const { result } = dispatch(library, request); + const result = dispatch(library, request); if (!result.ok) throw result.error; return copyBytes(result.value) as never; }; diff --git a/tests/unit/desktop/storage/doc-store-engine.test.ts b/tests/unit/desktop/storage/doc-store-engine.test.ts index 124f11e6..1a14a9b2 100644 --- a/tests/unit/desktop/storage/doc-store-engine.test.ts +++ b/tests/unit/desktop/storage/doc-store-engine.test.ts @@ -1,6 +1,9 @@ import { afterEach, beforeEach, describe, expect, test } from "bun:test"; -import type { Library } from "@/desktop/storage/engine/library"; +import * as Y from "yjs"; + +import { IDLE_MS, type Library } from "@/desktop/storage/engine/library"; +import { TRASH_RETENTION_MS } from "@/desktop/storage/engine/library-index"; import { dispatch } from "@/desktop/storage/protocol"; import type { DocStore } from "@/lib/platform/doc-store"; import { createEngineDocStore } from "@/tests/helpers/engine-doc-store"; @@ -43,26 +46,62 @@ describe("DocStore over the engine (bun:sqlite)", () => { }); describe("dispatch", () => { - test("a metadata write reports the new rev", () => { - const { result, rev } = dispatch(library, { - id: 1, - op: "createFolder", - args: { name: "A" }, + test("a failure comes back as an envelope, never a throw", () => { + expect( + dispatch(library, { + id: 2, + op: "load", + args: { docId: "nope", owner: "w1" }, + }), + ).toMatchObject({ + ok: false, + error: { code: "not_found", retryable: false }, }); - expect(result.ok).toBe(true); - expect(rev).toBe(library.sidebar().rev); }); +}); - test("a failure comes back as an envelope, never a throw", () => { - const { result, rev } = dispatch(library, { - id: 2, - op: "load", - args: { docId: "nope", owner: "w1" }, +describe("the index change feed", () => { + test("an append that changes the title moves rev and names the document", () => { + const { id } = library.createDoc(); + library.load(id, "w1"); + library.takeIndexChanges(); + const before = library.sidebar().rev; + const doc = new Y.Doc(); + doc.getText("t").insert(0, "x"); + library.append(id, "w1", { + origin: "local", + update: Y.encodeStateAsUpdate(doc), + title: "New title", }); - expect(result).toMatchObject({ - ok: false, - error: { code: "not_found", retryable: false }, + expect(library.takeIndexChanges()).toEqual({ + rev: before + 1, + changes: [{ kind: "doc", id }], }); - expect(rev).toBeNull(); + expect(library.takeIndexChanges()).toBeNull(); + }); + + test("index writes report each changed row once", () => { + const folder = library.createFolder({ name: "A" }); + const { id } = library.createDoc({ folderId: folder.id }); + library.renameDoc(id, "x"); + library.moveDoc(id, null, "a"); + const tag = library.createTag("t"); + expect(library.takeIndexChanges()?.changes).toEqual([ + { kind: "folder", id: folder.id }, + { kind: "doc", id }, + { kind: "tag", id: tag.id }, + ]); + }); + + test("the idle Trash purge is announced", () => { + const clock = createClock(); + library.shutdown(); + library = openTestLibrary(temp.dir, createTestHost(clock)); + const { id } = library.createDoc(); + library.trashDoc(id); + library.takeIndexChanges(); + clock.advance(TRASH_RETENTION_MS + IDLE_MS); + expect(library.runIdleJobs().purged).toBe(1); + expect(library.takeIndexChanges()?.changes).toEqual([{ kind: "doc", id }]); }); }); From 8cb74eabb491b4ff9a29e09ab4ce63f9313d768c Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:47:25 +0330 Subject: [PATCH 13/19] test(desktop): time the soak's startup from process start MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit T8 now gates sidebar rows against architecture §10 from process start, reports worker ready against its own row, runs one startup after an unclean exit to time quick_check, sends realistic snippets, and records how long each daily VACUUM INTO blocks the worker. Temp-folder cleanup retries and never fails a case. --- desktop/storage-harness/main.ts | 1 + tests/load/desktop-durability-soak.ts | 92 +++++++++++++++++++++------ 2 files changed, 75 insertions(+), 18 deletions(-) diff --git a/desktop/storage-harness/main.ts b/desktop/storage-harness/main.ts index 4a2c655e..d0733c39 100644 --- a/desktop/storage-harness/main.ts +++ b/desktop/storage-harness/main.ts @@ -268,6 +268,7 @@ async function startup() { `STARTUP ${JSON.stringify({ processToSpawnMs: spawnAt, workerReadyMs: readyAt - spawnAt, + processToReadyMs: readyAt, openMs: client.timings.openMs, sidebarCallMs: rowsAt - readyAt, processToSidebarMs: rowsAt, diff --git a/tests/load/desktop-durability-soak.ts b/tests/load/desktop-durability-soak.ts index 0c4d9738..03be029f 100644 --- a/tests/load/desktop-durability-soak.ts +++ b/tests/load/desktop-durability-soak.ts @@ -93,6 +93,23 @@ const note = (text: string) => { if (args.verbose) console.log(text); }; +/** + * Removes a temp library. On Windows, Defender or the indexer may still hold + * a file for a moment; a cleanup failure is logged, never a failed case. + */ +function removeDir(dir: string) { + try { + rmSync(dir, { + recursive: true, + force: true, + maxRetries: 10, + retryDelay: 200, + }); + } catch (error) { + console.warn(`could not remove ${dir}: ${String(error)}`); + } +} + // ── Engine access (bun:sqlite) ────────────────────────────────────────── function openEngine( @@ -210,7 +227,7 @@ async function runHarness( async function caseContract(): Promise { const dir = mkdtempSync(join(tmpdir(), "soak-contract-")); const run = await runHarness([`--mode=contract`, `--dir=${dir}`]); - rmSync(dir, { recursive: true, force: true }); + removeDir(dir); const ok = run.lines.filter((l) => l.startsWith("CASE_OK")).length; const failed = run.lines.filter((l) => l.startsWith("CASE_FAIL")); const stats = run.lines.find((l) => l.startsWith("STATS ")); @@ -315,7 +332,7 @@ async function killLoop( `${name} ${i}/${kills} last=${last} acked=${ackedK} failures=${failures.length}`, ); } - rmSync(dir, { recursive: true, force: true }); + removeDir(dir); return { name, pass: failures.length === 0, @@ -338,7 +355,7 @@ async function caseT3(): Promise { `--fullfsync=${fullfsync}`, "--count=300", ]); - rmSync(dir, { recursive: true, force: true }); + removeDir(dir); const line = run.lines.find((l) => l.startsWith("TIMING ")); if (!line) { failures.push( @@ -424,7 +441,7 @@ async function caseT4(runs: number): Promise { if (k !== Number(kText)) failures.push(`run ${i}: ${k} edits stored, ${kText} typed before quit`); } - rmSync(dir, { recursive: true, force: true }); + removeDir(dir); return { name: "T4", pass: failures.length === 0, @@ -473,7 +490,7 @@ async function caseT5(kills: number): Promise { failures.push( `calibration: ${calibration.lines.join(" | ")} ${calibration.stderr.slice(0, 500)}`, ); - rmSync(calibrationDir, { recursive: true, force: true }); + removeDir(calibrationDir); const random = seededRandom(5); for (let i = 1; i <= kills; i += 1) { const dir = mkdtempSync(join(tmpdir(), "soak-t5-")); @@ -512,7 +529,7 @@ async function caseT5(kills: number): Promise { const pad = textOf(library, docId).padLength; library.shutdown(); if (pad !== 4096) failures.push(`kill ${i}: document damaged (${pad})`); - rmSync(dir, { recursive: true, force: true }); + removeDir(dir); } return { name: "T5", @@ -539,7 +556,7 @@ async function caseT6(): Promise { const before = hash(); const run = await runHarness(["--mode=readonly", `--dir=${dir}`]); const after = hash(); - rmSync(dir, { recursive: true, force: true }); + removeDir(dir); const failures: string[] = []; const status = run.lines.find((l) => l.startsWith("STATUS ")); if (!status || JSON.parse(status.slice(7)).readOnly !== "library_too_new") @@ -563,9 +580,15 @@ async function caseT6(): Promise { // ── T8: a year of editing, then startup with 1,000 documents ────────── -/** Architecture §10 "Sidebar populated", with the 1.5× CI slack of §10. */ -function sidebarBudgetMs() { - return process.platform === "win32" ? 1000 : 600; +/** + * Architecture §10 targets, from process start: "Worker ready" and + * "Sidebar populated" (Mac means an M1, Win a mid-range laptop; Linux uses + * the Windows row). CI applies §10's 1.5× slack to the sidebar gate. + */ +function budgets() { + return process.platform === "darwin" + ? { workerReadyMs: 150, sidebarMs: 600 } + : { workerReadyMs: 300, sidebarMs: 1000 }; } async function caseT8(): Promise { @@ -616,6 +639,9 @@ async function caseT8(): Promise { ids.slice(0, 5).map((id) => [id, ""]), ); const monthly: Record[] = []; + // How long each daily VACUUM INTO blocked the worker: a quit flush queues + // behind it (D-A2's "a clean quit loses nothing" needs it under ~3 s). + const copyMs: number[] = []; let appends = 0; let compactions = 0; let trashed = 0; @@ -646,6 +672,9 @@ async function caseT8(): Promise { origin: "local", update, title: `Document ${id.slice(0, 4)}`, + // A list-view snippet as long as the real one, so the sidebar + // payload is realistic. + snippet: words.repeat(2).slice(0, 200), }); appends += 1; rowsSinceSnapshot += 1; @@ -673,7 +702,12 @@ async function caseT8(): Promise { } // Idle: the daily copy and the Trash purge. clock.now += IDLE_MS; - library.runIdleJobs(); + const idle = library.runIdleJobs(); + if (idle.dailyCopyMs !== null) copyMs.push(idle.dailyCopyMs); + if (idle.dailyCopy === null && idle.dailyCopyMs !== null) + failures.push( + `day ${day}: daily copy failed (${library.stats().lastDailyCopyError})`, + ); // Restart now and then, sometimes uncleanly. if (day % 30 === 29) { if (random() < 0.5) library.shutdown(); @@ -730,25 +764,45 @@ async function caseT8(): Promise { } startups.push(JSON.parse(line.slice(8))); } - const budget = sidebarBudgetMs(); + const budget = budgets(); const median = (key: string) => { const values = startups.map((s) => Number(s[key])).sort((a, b) => a - b); return values[Math.floor(values.length / 2)] ?? Number.NaN; }; - const spawnToSidebar = median("spawnToSidebarMs"); - if (!args["no-electron"] && !(spawnToSidebar <= budget * 1.5)) + const toSidebar = median("processToSidebarMs"); + const toWorkerReady = median("processToReadyMs"); + if (!args["no-electron"] && !(toSidebar <= budget.sidebarMs * 1.5)) failures.push( - `sidebar ${spawnToSidebar.toFixed(0)} ms after worker spawn, budget ${budget} ms (× 1.5 CI slack)`, + `sidebar rows ${toSidebar.toFixed(0)} ms after process start, budget ${budget.sidebarMs} ms (× 1.5 CI slack)`, ); if (startups.some((s) => s.docs !== liveDocs)) failures.push( `startup listed ${startups.map((s) => s.docs)} documents, wanted ${liveDocs}`, ); - rmSync(dir, { recursive: true, force: true }); + // §10 "After an unclean exit": quick_check runs inside the open. + let unclean: Record | null = null; + if (!args["no-electron"]) { + const db = new Database(join(dir, LIBRARY_FILE)); + db.exec("UPDATE meta SET value = '1' WHERE key = 'unclean_exit'"); + db.close(); + const run = await runHarness(["--mode=startup", `--dir=${dir}`]); + const line = run.lines.find((l) => l.startsWith("STARTUP ")); + if (line) unclean = JSON.parse(line.slice(8)); + if (!unclean?.uncleanExit) + failures.push(`unclean startup: ${line ?? run.stderr.slice(0, 500)}`); + else if (unclean.openMs > 2000) + failures.push( + `quick_check open took ${unclean.openMs.toFixed(0)} ms, budget 2 s`, + ); + } + const sortedCopies = [...copyMs].sort((a, b) => a - b); + const copyMedian = sortedCopies[Math.floor(sortedCopies.length / 2)] ?? 0; + const copyMax = sortedCopies.at(-1) ?? 0; + removeDir(dir); return { name: "T8", pass: failures.length === 0, - summary: `${days} days, ${appends} appends, ${compactions} compactions, ${trashed} trashed; ${(seededBytes / 1048576).toFixed(0)} → ${(stats.bytes / 1048576).toFixed(0)} MB; ${dailies.length} daily copies; ${liveDocs} documents listed ${spawnToSidebar.toFixed(0)} ms after worker spawn (median of 5; budget ${budget} ms)`, + summary: `${days} days, ${appends} appends, ${compactions} compactions, ${trashed} trashed; ${(seededBytes / 1048576).toFixed(0)} → ${(stats.bytes / 1048576).toFixed(0)} MB; ${dailies.length} daily copies (VACUUM INTO median ${copyMedian.toFixed(0)} ms, max ${copyMax.toFixed(0)} ms); ${liveDocs} documents: worker ready ${toWorkerReady.toFixed(0)} ms and sidebar rows ${toSidebar.toFixed(0)} ms after process start (median of 5; §10 ${budget.workerReadyMs} / ${budget.sidebarMs} ms; excludes renderer IPC and paint); unclean-exit open with quick_check ${unclean ? unclean.openMs.toFixed(0) : "n/a"} ms`, details: { days, docs: liveDocs, @@ -759,7 +813,9 @@ async function caseT8(): Promise { finalBytes: stats.bytes, monthly, startups, - budgetMs: budget, + unclean, + budget, + dailyCopyMs: { median: copyMedian, max: copyMax, count: copyMs.length }, simMs, }, failures, From d0d82ed9c5261f5948fba469f94720e2ae6b7cbd Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:49:10 +0330 Subject: [PATCH 14/19] fix(desktop): finalize bun:sqlite statements before closing Bun's close() leaves a connection whose statements are still live open as a zombie, holding the library's file handles. On Windows the soak could not delete a library the verifier had closed (EBUSY). The driver now finalizes its cached statements first. --- desktop/dev/bun-sqlite-driver.ts | 13 ++++++++++++- tests/unit/desktop/storage/library-open.test.ts | 14 ++++++++++++++ 2 files changed, 26 insertions(+), 1 deletion(-) diff --git a/desktop/dev/bun-sqlite-driver.ts b/desktop/dev/bun-sqlite-driver.ts index 200801a8..4e915202 100644 --- a/desktop/dev/bun-sqlite-driver.ts +++ b/desktop/dev/bun-sqlite-driver.ts @@ -3,6 +3,10 @@ * (architecture §11). Normalizes Bun's differences from `node:sqlite`: a * missing row is `null` in Bun and `undefined` here, and integers stay * numbers (`safeIntegers` off). + * + * `close` finalizes every cached statement first: Bun's `close()` leaves a + * connection with live statements open as a zombie that keeps the file + * handles, which on Windows blocks deleting or replacing the library. */ import { Database } from "bun:sqlite"; @@ -19,12 +23,14 @@ export const openBunSqlite: OpenSqlite = (path, { readOnly }) => { ? new Database(path, { readonly: true }) : new Database(path, { create: true }); const cache = new Map(); + const statements: { finalize(): void }[] = []; const driver: SqliteDriver = { exec: (sql) => db.exec(sql), prepare(sql) { const cached = cache.get(sql); if (cached) return cached; const statement = db.prepare(sql); + statements.push(statement); const wrapped: SqliteStatement = { run: (...params) => { const result = statement.run(...params); @@ -39,7 +45,12 @@ export const openBunSqlite: OpenSqlite = (path, { readOnly }) => { cache.set(sql, wrapped); return wrapped; }, - close: () => db.close(), + close: () => { + for (const statement of statements) statement.finalize(); + statements.length = 0; + cache.clear(); + db.close(); + }, }; return driver; }; diff --git a/tests/unit/desktop/storage/library-open.test.ts b/tests/unit/desktop/storage/library-open.test.ts index 5b230d67..42366f65 100644 --- a/tests/unit/desktop/storage/library-open.test.ts +++ b/tests/unit/desktop/storage/library-open.test.ts @@ -349,3 +349,17 @@ test("stats reads the pragmas back as set", () => { }); library.shutdown(); }); + +test("shutdown releases every file handle (Windows can delete the folder)", () => { + const lsof = Bun.which("lsof"); + if (!lsof) return; + const library = openTestLibrary(temp.dir, createTestHost(clock)); + library.createDoc({ title: "x" }); + library.sidebar(); + library.shutdown(); + const open = Bun.spawnSync([lsof, "-p", String(process.pid)]) + .stdout.toString() + .split("\n") + .filter((line) => line.includes(temp.dir)); + expect(open).toEqual([]); +}); From f6e5306c7fe2edf50f2456fb32c9a8be8f1ead3c Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:52:55 +0330 Subject: [PATCH 15/19] feat(desktop): create documents at a given schema with any safe id createDoc takes the document's schema (DOC_SCHEMA for a new document, the file's on import) and accepts any id that is safe in SQL, paths and logs. Main's IPC guard still lets only UUIDs through; tests and imports use other ids. --- desktop/storage/engine/library-index.ts | 24 +++++++++++++++---- .../desktop/storage/library-index.test.ts | 15 ++++++++++++ 2 files changed, 35 insertions(+), 4 deletions(-) diff --git a/desktop/storage/engine/library-index.ts b/desktop/storage/engine/library-index.ts index e7ed44f7..e0946e87 100644 --- a/desktop/storage/engine/library-index.ts +++ b/desktop/storage/engine/library-index.ts @@ -16,8 +16,13 @@ import { bumpRev, randomHex32, readRev, scalar, transaction } from "./sql"; export type DocKind = "doc" | "template"; export type CreateDocInput = { - /** A UUID; minted when omitted. */ + /** + * Minted (a UUID) when omitted. Main's IPC guard only lets UUIDs through; + * the engine accepts any {@link SAFE_ID}, which tests and imports use. + */ id?: string; + /** `docs.schema`: `DOC_SCHEMA` for a new document, the file's on import. */ + schema?: number; title?: string; folderId?: string | null; sortKey?: string; @@ -81,10 +86,17 @@ export type SidebarSnapshot = { export const TRASH_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +/** Ids that are safe in SQL, file names and logs. */ +export const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/; const HEX32 = /^[0-9a-f]{32}$/; export const MAX_NAME_LENGTH = 200; function requireId(id: string, what: string) { + if (!SAFE_ID.test(id)) + throw new StorageError("invalid_argument", `bad ${what}`); +} + +function requireUuid(id: string, what: string) { if (!UUID.test(id)) throw new StorageError("invalid_argument", `bad ${what}`); } @@ -112,7 +124,10 @@ export function createDoc( const id = input.id ?? crypto.randomUUID(); requireId(id, "document id"); const lineage = input.lineage ?? crypto.randomUUID(); - requireId(lineage, "lineage"); + requireUuid(lineage, "lineage"); + const schema = input.schema ?? 0; + if (!Number.isInteger(schema) || schema < 0) + throw new StorageError("invalid_argument", "bad schema"); const linkKey = input.linkKey ?? randomHex32(); if (!HEX32.test(linkKey)) throw new StorageError("invalid_argument", "bad link key"); @@ -123,8 +138,8 @@ export function createDoc( const rev = indexWrite(db, () => { db.prepare( `INSERT INTO docs (id, title, folder_id, sort_key, created_at, modified_at, - lineage, link_key, kind) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + lineage, link_key, kind, schema) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, ).run( id, cleanName(input.title ?? ""), @@ -135,6 +150,7 @@ export function createDoc( lineage, linkKey, kind, + schema, ); }); return { id, lineage, linkKey, rev }; diff --git a/tests/unit/desktop/storage/library-index.test.ts b/tests/unit/desktop/storage/library-index.test.ts index 3478c473..0947972d 100644 --- a/tests/unit/desktop/storage/library-index.test.ts +++ b/tests/unit/desktop/storage/library-index.test.ts @@ -202,3 +202,18 @@ describe("Trash and purge", () => { expect(library.stats().docs).toBe(2); }); }); + +describe("ids and schema at creation", () => { + test("accepts any safe id and stores the given schema", () => { + const { id } = library.createDoc({ id: "doc-a_1", schema: 3 }); + expect(library.load(id, "w1").schema).toBe(3); + }); + + test("refuses ids that aren't safe in SQL, paths or logs", () => { + for (const id of ["", "../x", "a b", "a/b", "-x", "x".repeat(65)]) + expect(codeOf(() => library.createDoc({ id }))).toBe("invalid_argument"); + expect(codeOf(() => library.createDoc({ schema: -1 }))).toBe( + "invalid_argument", + ); + }); +}); From 7a3f09fa6453079bef1859b9b5286ae82421632a Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:52:55 +0330 Subject: [PATCH 16/19] test(desktop): run P2.1's DocStore contract on both storage drivers The shared contract now runs on the engine over bun:sqlite under bun test, and over node:sqlite in the real storage worker in the soak's contract case. It replaces the interim engine cases. --- .github/workflows/desktop-storage-soak.yml | 2 + desktop/storage-harness/main.ts | 64 ++++-- tests/helpers/storage-doc-store-cases.ts | 186 ------------------ tests/load/desktop-durability-soak.ts | 13 +- .../storage/doc-store-contract.test.ts | 43 ++++ .../desktop/storage/doc-store-engine.test.ts | 21 -- 6 files changed, 91 insertions(+), 238 deletions(-) delete mode 100644 tests/helpers/storage-doc-store-cases.ts create mode 100644 tests/unit/desktop/storage/doc-store-contract.test.ts diff --git a/.github/workflows/desktop-storage-soak.yml b/.github/workflows/desktop-storage-soak.yml index 517a042c..aca44f4d 100644 --- a/.github/workflows/desktop-storage-soak.yml +++ b/.github/workflows/desktop-storage-soak.yml @@ -20,6 +20,8 @@ on: - desktop/storage-harness/** - desktop/dev/bun-sqlite-driver.ts - lib/platform/desktop/** + - lib/platform/doc-store.ts + - tests/helpers/doc-store-contract.ts - tests/load/desktop-durability-soak.ts - .github/workflows/desktop-storage-soak.yml workflow_call: diff --git a/desktop/storage-harness/main.ts b/desktop/storage-harness/main.ts index d0733c39..29715741 100644 --- a/desktop/storage-harness/main.ts +++ b/desktop/storage-harness/main.ts @@ -6,8 +6,8 @@ * engine on `bun:sqlite`. * * Modes (`--mode=`): - * - `contract`: the engine `DocStore` cases over the worker (`CASE_OK` / - * `CASE_FAIL` lines). + * - `contract`: P2.1's shared `DocStore` contract over the worker + * (`CASE_OK` / `CASE_FAIL` lines). * - `churn`: edits a document with product batching (or `--dense`) until * killed; `--target=compaction` compacts after every ack; * `--quit-after-ms` runs the quit flush instead (T4). The worker prints @@ -21,7 +21,8 @@ * An uncaught throw in Electron main opens a modal dialog, so every failure * exits with `process.exit(1)` after printing `HARNESS_FAIL`. */ -import { writeSync } from "node:fs"; +import { mkdtempSync, rmSync, writeSync } from "node:fs"; +import { tmpdir } from "node:os"; import path from "node:path"; import { app } from "electron"; @@ -29,7 +30,10 @@ import * as Y from "yjs"; import { createRpcDocStore } from "../../lib/platform/desktop/rpc-doc-store"; import type { DocStore } from "../../lib/platform/doc-store"; -import { STORAGE_DOC_STORE_CASES } from "../../tests/helpers/storage-doc-store-cases"; +import { + DOC_STORE_CONTRACT, + type DocStoreHarness, +} from "../../tests/helpers/doc-store-contract"; import { StorageClient } from "../storage/client"; import { openNodeSqlite } from "../storage/drivers/node-sqlite"; import { Library } from "../storage/engine/library"; @@ -85,32 +89,50 @@ function storeFor(client: StorageClient, owner: string): DocStore { ); } +/** + * P2.1's shared DocStore contract on `node:sqlite` in the real worker: one + * worker and temp library per case, one owner id per connected client. + */ async function contract() { - const client = await StorageClient.start(workerPath(), { dir: dir() }); + let passed = 0; let failed = 0; - for (const [i, testCase] of STORAGE_DOC_STORE_CASES.entries()) { - const stores = new Map(); - try { - await testCase.run({ - store(owner) { - const key = `${i}:${owner}`; - let store = stores.get(key); - if (!store) { - store = storeFor(client, key); - stores.set(key, store); - } - return store; - }, - createDoc: async () => (await client.call("createDoc", {})).id, + const harness: DocStoreHarness = { + exclusiveOpen: true, + async createLibrary() { + const libraryDir = mkdtempSync(path.join(tmpdir(), "contract-")); + const client = await StorageClient.start(workerPath(), { + dir: libraryDir, }); + let clients = 0; + return { + connect: () => storeFor(client, `client-${++clients}`), + async createDoc(docId, schema) { + await client.call("createDoc", { id: docId, schema }); + }, + async dispose() { + await client.shutdown(); + rmSync(libraryDir, { + recursive: true, + force: true, + maxRetries: 10, + retryDelay: 200, + }); + }, + }; + }, + }; + for (const testCase of DOC_STORE_CONTRACT) { + if (testCase.requires && !harness[testCase.requires]) continue; + try { + await testCase.run(harness); + passed += 1; say(`CASE_OK ${testCase.name}`); } catch (error) { failed += 1; say(`CASE_FAIL ${testCase.name}: ${String(error)}`); } } - say(`STATS ${JSON.stringify(await client.call("stats", {}))}`); - say(`SHUTDOWN ${await client.shutdown()}`); + say(`CONTRACT ${passed} passed, ${failed} failed`); if (failed > 0) fail(`${failed} contract case(s) failed`); } diff --git a/tests/helpers/storage-doc-store-cases.ts b/tests/helpers/storage-doc-store-cases.ts deleted file mode 100644 index fe406882..00000000 --- a/tests/helpers/storage-doc-store-cases.ts +++ /dev/null @@ -1,186 +0,0 @@ -/** - * `DocStore` cases for the desktop storage engine, written against the port - * only. Framework-free (`node:assert`), so `bun test` runs them on - * `bun:sqlite` and the Electron soak harness runs them on `node:sqlite` in - * the real storage worker. P2.1's shared contract suite joins these once it - * lands; these stay for the engine-specific rules (a library-wide seq, - * `not_found` for a document that was never created). - */ -import assert from "node:assert/strict"; - -import * as Y from "yjs"; - -import { type DocStore, DocStoreError } from "@/lib/platform/doc-store"; - -export type StoreCaseContext = { - /** One window's store; the same `owner` returns the same window. */ - store(owner: string): DocStore; - /** Creates a document through the index and returns its id. */ - createDoc(): Promise; -}; - -export type StoreCase = { - name: string; - run(context: StoreCaseContext): Promise; -}; - -function edit(doc: Y.Doc, text: string): Uint8Array { - let update: Uint8Array = new Uint8Array(); - doc.once("update", (u: Uint8Array) => { - update = u; - }); - const t = doc.getText("t"); - t.insert(t.length, text); - return update; -} - -function textOf(loaded: { - snapshot: Uint8Array | null; - updates: Uint8Array[]; -}) { - const doc = new Y.Doc(); - if (loaded.snapshot) Y.applyUpdate(doc, loaded.snapshot); - for (const update of loaded.updates) Y.applyUpdate(doc, update); - return doc.getText("t").toString(); -} - -async function rejection(promise: Promise): Promise { - try { - await promise; - } catch (error) { - assert.ok(error instanceof DocStoreError, `not a DocStoreError: ${error}`); - return error; - } - assert.fail("expected a rejection"); -} - -export const STORAGE_DOC_STORE_CASES: StoreCase[] = [ - { - name: "a new document loads empty", - async run({ store, createDoc }) { - const id = await createDoc(); - assert.deepEqual(await store("w1").load(id), { - snapshot: null, - updates: [], - maxSeq: 0, - schema: 0, - }); - }, - }, - { - name: "appends load back in order with increasing seqs", - async run({ store, createDoc }) { - const id = await createDoc(); - const w1 = store("w1"); - await w1.load(id); - const doc = new Y.Doc(); - const a = await w1.append(id, { - origin: "local", - update: edit(doc, "a"), - }); - const b = await w1.append(id, { - origin: "local", - update: edit(doc, "b"), - }); - assert.ok(b > a); - const loaded = await w1.load(id); - assert.equal(loaded.maxSeq, b); - assert.equal(textOf(loaded), "ab"); - }, - }, - { - name: "a resent append changes nothing", - async run({ store, createDoc }) { - const id = await createDoc(); - const w1 = store("w1"); - const update = edit(new Y.Doc(), "once"); - await w1.load(id); - await w1.append(id, { origin: "local", update }); - await w1.append(id, { origin: "local", update }); - assert.equal(textOf(await w1.load(id)), "once"); - }, - }, - { - name: "compaction folds covered rows into the snapshot and returns no foreign state", - async run({ store, createDoc }) { - const id = await createDoc(); - const w1 = store("w1"); - await w1.load(id); - const doc = new Y.Doc(); - await w1.append(id, { origin: "local", update: edit(doc, "a") }); - const covered = await w1.append(id, { - origin: "local", - update: edit(doc, "b"), - }); - const state = Y.encodeStateAsUpdate(doc); - const late = await w1.append(id, { - origin: "local", - update: edit(doc, "c"), - }); - assert.deepEqual(await w1.compact(id, state, covered), { foreign: null }); - const loaded = await w1.load(id); - assert.ok(loaded.snapshot); - assert.equal(loaded.updates.length, 1); - assert.equal(loaded.maxSeq, late); - assert.equal(textOf(loaded), "abc"); - }, - }, - { - name: "a second window gets already_open until the first closes", - async run({ store, createDoc }) { - const id = await createDoc(); - await store("w1").load(id); - const error = await rejection(store("w2").load(id)); - assert.equal(error.code, "already_open"); - assert.equal(error.retryable, false); - await store("w1").close(id); - assert.equal((await store("w2").load(id)).maxSeq, 0); - }, - }, - { - name: "flush resolves after every append already issued", - async run({ store, createDoc }) { - const id = await createDoc(); - const w1 = store("w1"); - await w1.load(id); - const doc = new Y.Doc(); - const sent = ["a", "b", "c"].map((text) => - w1.append(id, { origin: "local", update: edit(doc, text) }), - ); - await w1.flush(); - const seqs = await Promise.all(sent); - assert.equal((await w1.load(id)).maxSeq, Math.max(...seqs)); - }, - }, - { - name: "a rejected append writes nothing", - async run({ store, createDoc }) { - const id = await createDoc(); - const w1 = store("w1"); - await w1.load(id); - const error = await rejection( - w1.append(id, { origin: "local", update: new Uint8Array() }), - ); - assert.equal(error.code, "invalid_argument"); - assert.deepEqual((await w1.load(id)).updates, []); - }, - }, - { - name: "a window that hasn't opened a document can't write it", - async run({ store, createDoc }) { - const id = await createDoc(); - const update = edit(new Y.Doc(), "x"); - const error = await rejection( - store("w1").append(id, { origin: "local", update }), - ); - assert.equal(error.code, "forbidden"); - }, - }, - { - name: "an unknown document is not_found", - async run({ store }) { - const error = await rejection(store("w1").load(crypto.randomUUID())); - assert.equal(error.code, "not_found"); - }, - }, -]; diff --git a/tests/load/desktop-durability-soak.ts b/tests/load/desktop-durability-soak.ts index 03be029f..5fa349ee 100644 --- a/tests/load/desktop-durability-soak.ts +++ b/tests/load/desktop-durability-soak.ts @@ -225,21 +225,14 @@ async function runHarness( // ── contract: the engine DocStore cases inside the Electron worker ────── async function caseContract(): Promise { - const dir = mkdtempSync(join(tmpdir(), "soak-contract-")); - const run = await runHarness([`--mode=contract`, `--dir=${dir}`]); - removeDir(dir); + const run = await runHarness(["--mode=contract"]); const ok = run.lines.filter((l) => l.startsWith("CASE_OK")).length; const failed = run.lines.filter((l) => l.startsWith("CASE_FAIL")); - const stats = run.lines.find((l) => l.startsWith("STATS ")); return { name: "contract", pass: run.exitCode === 0 && failed.length === 0 && ok > 0, - summary: `${ok} DocStore cases passed on node:sqlite in the Electron worker`, - details: { - ok, - stats: stats ? JSON.parse(stats.slice(6)) : null, - exitCode: run.exitCode, - }, + summary: `P2.1's DocStore contract: ${ok} cases passed on node:sqlite in the Electron worker`, + details: { ok, exitCode: run.exitCode }, failures: [ ...failed, ...(run.exitCode === 0 ? [] : [run.stderr.slice(0, 2000)]), diff --git a/tests/unit/desktop/storage/doc-store-contract.test.ts b/tests/unit/desktop/storage/doc-store-contract.test.ts new file mode 100644 index 00000000..89f4142c --- /dev/null +++ b/tests/unit/desktop/storage/doc-store-contract.test.ts @@ -0,0 +1,43 @@ +/** + * P2.1's shared `DocStore` contract (architecture §11) on the storage + * engine over `bun:sqlite`, through the same dispatch the worker uses. The + * Electron soak's `contract` case runs it again on `node:sqlite` inside the + * real worker. + */ +import { describe, it } from "bun:test"; + +import type { Library } from "@/desktop/storage/engine/library"; +import { defineDocStoreContract } from "@/tests/helpers/doc-store-contract"; +import { createEngineDocStore } from "@/tests/helpers/engine-doc-store"; +import { + createClock, + createTempDir, + createTestHost, + openTestLibrary, +} from "@/tests/helpers/storage-library"; + +defineDocStoreContract( + { describe, it }, + "DocStore contract on the engine (bun:sqlite)", + { + exclusiveOpen: true, + async createLibrary() { + const temp = createTempDir(); + const library: Library = openTestLibrary( + temp.dir, + createTestHost(createClock()), + ); + let clients = 0; + return { + connect: () => createEngineDocStore(library, `client-${++clients}`), + async createDoc(docId, schema) { + library.createDoc({ id: docId, schema }); + }, + async dispose() { + library.shutdown(); + temp.cleanUp(); + }, + }; + }, + }, +); diff --git a/tests/unit/desktop/storage/doc-store-engine.test.ts b/tests/unit/desktop/storage/doc-store-engine.test.ts index 1a14a9b2..5c1840b6 100644 --- a/tests/unit/desktop/storage/doc-store-engine.test.ts +++ b/tests/unit/desktop/storage/doc-store-engine.test.ts @@ -5,9 +5,6 @@ import * as Y from "yjs"; import { IDLE_MS, type Library } from "@/desktop/storage/engine/library"; import { TRASH_RETENTION_MS } from "@/desktop/storage/engine/library-index"; import { dispatch } from "@/desktop/storage/protocol"; -import type { DocStore } from "@/lib/platform/doc-store"; -import { createEngineDocStore } from "@/tests/helpers/engine-doc-store"; -import { STORAGE_DOC_STORE_CASES } from "@/tests/helpers/storage-doc-store-cases"; import { createClock, createTempDir, @@ -27,24 +24,6 @@ afterEach(() => { temp.cleanUp(); }); -describe("DocStore over the engine (bun:sqlite)", () => { - for (const testCase of STORAGE_DOC_STORE_CASES) - test(testCase.name, async () => { - const stores = new Map(); - await testCase.run({ - store(owner) { - let store = stores.get(owner); - if (!store) { - store = createEngineDocStore(library, owner); - stores.set(owner, store); - } - return store; - }, - createDoc: async () => library.createDoc().id, - }); - }); -}); - describe("dispatch", () => { test("a failure comes back as an envelope, never a throw", () => { expect( From 1390edba14d1fbd8632de0b7bf74fa7895ae87e1 Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:52:56 +0330 Subject: [PATCH 17/19] docs(desktop): note the shared contract and T7's home for P2.6 --- CHANGELOG.md | 14 +++++++++----- .../specs/2026-11-15-desktop-architecture.md | 2 +- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index adf045f5..b8deb66d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -35,7 +35,9 @@ worker and P2.7 builds the files sidebar on it. - appends and renderer-driven compaction in one transaction each, with one window per document (`already_open`, `forbidden`, `document_open`); - documents, folders, tags, Trash (emptied after 30 days) and the sidebar - snapshot with `rev`; + snapshot, with a change feed: every change the sidebar shows, appends + included, moves `rev`, and the worker sends `index_changed` with the + changed rows; - a daily `VACUUM INTO` copy on the first idle moment, keeping 7, and a reader for restoring one document from a day; - shutdown for the quit flush: WAL checkpoint, `unclean_exit` cleared, @@ -49,9 +51,11 @@ worker and P2.7 builds the files sidebar on it. large, isn't Yjs, or carries a node, mark or root this build doesn't know) and `lib/platform/desktop/rpc-doc-store.ts` (`DocStore` over worker replies, for `IpcDocStore`). -- **Tests:** engine suites on `bun:sqlite` (open, content, index, daily +- **Tests:** P2.1's shared `DocStore` contract on the engine, on + `bun:sqlite` under `bun test` and on `node:sqlite` inside the Electron + worker; engine suites (open, content, index and its change feed, daily copies, codes, and a property test of random appends, compactions, - crashes and reopens), the import-boundary test's storage rows, and + crashes and reopens); the import-boundary test's storage rows; and `tests/load/desktop-durability-soak.ts` with its Electron harness in `desktop/storage-harness/` for soak cases T1–T6 and T8. - **CI:** `desktop-storage-soak.yml` on macOS, Windows and Linux: 20 kills @@ -63,8 +67,8 @@ worker and P2.7 builds the files sidebar on it. - **Docs:** the desktop architecture spec records where P2.6 put the engine and the harness, the worker-side registry check and `claim`, the - import-boundary allowance for `lib/schema-names.ts`, and that T7 belongs to - the web library. + import-boundary allowance for `lib/schema-names.ts`, and that T7 is covered + by `local-session`'s tests, not the storage soak. ### Desktop P2.1b: `local-session`, the batcher and `MemoryDocStore` diff --git a/docs/superpowers/specs/2026-11-15-desktop-architecture.md b/docs/superpowers/specs/2026-11-15-desktop-architecture.md index 874ce912..8336ac75 100644 --- a/docs/superpowers/specs/2026-11-15-desktop-architecture.md +++ b/docs/superpowers/specs/2026-11-15-desktop-architecture.md @@ -597,7 +597,7 @@ sequenceDiagram | Compaction invariant | property test: random appends, compactions and reopens, with the Y.Doc state compared against a reference | engine on `bun:sqlite` | | Guards and `doc-handlers` | fuzz plus golden messages, sender and registry checks | pure modules | | Reducer and chip | a table row for each plan §5.5 state; "every refusal ends in a stop or a backoff" | pure | -| Durability T1–T8, T16–T18 | `tests/load/desktop-durability-soak.ts`: SIGKILL Electron during append, compaction, migration, asset writes and image migration; a year of image churn | the storage worker in a windowless Electron (`desktop/storage-harness`, P2.6), verified on `bun:sqlite`; the packaged app once P2.5 lands. T7 (slow hydrate) is the web library's (W2): it tests the session layer, not storage. | +| Durability T1–T8, T16–T18 | `tests/load/desktop-durability-soak.ts`: SIGKILL Electron during append, compaction, migration, asset writes and image migration; a year of image churn | the storage worker in a windowless Electron (`desktop/storage-harness`, P2.6), verified on `bun:sqlite`; the packaged app once P2.5 lands. T7 (slow hydrate) tests the session layer, not storage: P2.1 covers it in `local-session`'s tests, and W2 again over IndexedDB. | | `BlobStore` contract and asset GC | one suite plus a property test: GC never deletes a referenced blob | `MemoryBlobStore`; the worker on `bun:sqlite`; `IdbBlobStore` in the web library | | Preferences boot script | parity with `resolveBoot()` over about 40 stored payloads; the desktop CSP hash matches the script (build test) | happy-dom; build | | Main | `argv` filter, navigation and `window.open` denial, fuses | `_electron` | From de7a82cf1012498fcd8dc739ea8a21c8f93d2782 Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:54:59 +0330 Subject: [PATCH 18/19] refactor(platform): name the migration step migrate, as local-session takes it DOC_MIGRATIONS now plugs into LocalSessionOptions without an adapter. runDocMigrations stays for import, where a file migrates all or nothing. --- lib/doc-migrations/index.ts | 14 ++++++++------ .../unit/lib/doc-migrations/doc-migrations.test.ts | 8 +++++--- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/lib/doc-migrations/index.ts b/lib/doc-migrations/index.ts index d45ad5f7..bb1c718e 100644 --- a/lib/doc-migrations/index.ts +++ b/lib/doc-migrations/index.ts @@ -4,9 +4,10 @@ * * - **On open**, `local-session` runs them after hydrate and before the * editor binds (y-prosemirror deletes runs whose marks the schema doesn't - * know, `lib/yjs-mark-migration.ts`), inside one `doc.transact` with - * `MIGRATION_ORIGIN`, outside undo. Their update is appended as origin - * `migration` with `schemaTo`, which the storage engine writes to + * know, `lib/yjs-mark-migration.ts`). The composition root passes + * `DOC_SCHEMA` and `DOC_MIGRATIONS`; local-session runs each step in its + * own `MIGRATION_ORIGIN` transaction, outside undo, and appends it as + * origin `migration` with `schemaTo`, which the storage engine writes to * `docs.schema` in the same transaction. * - **At import** (P2.8), the same functions run on `.tinydoc` and web * downloads. @@ -21,7 +22,7 @@ export type DocMigration = { /** The schema this step produces: the previous step's `to` plus one. */ to: number; /** Edits `doc` in place. Runs inside the caller's transaction. */ - run(doc: Y.Doc): void; + migrate(doc: Y.Doc): void; }; export const DOC_MIGRATIONS: readonly DocMigration[] = []; @@ -54,7 +55,8 @@ export function planDocMigrations( /** * Runs the steps a plan lists in one transaction under `origin`, so they - * produce one update. Returns the schema reached, or null when there was + * produce one update. For import, where a file is all-or-nothing; the open + * hook in local-session runs one transaction per step instead. Returns the schema reached, or null when there was * nothing to do. */ export function runDocMigrations( @@ -64,7 +66,7 @@ export function runDocMigrations( ): number | null { if (plan.kind !== "migrate") return null; doc.transact(() => { - for (const step of plan.steps) step.run(doc); + for (const step of plan.steps) step.migrate(doc); }, origin); return plan.schemaTo; } diff --git a/tests/unit/lib/doc-migrations/doc-migrations.test.ts b/tests/unit/lib/doc-migrations/doc-migrations.test.ts index f67f15a2..e4bcc1bc 100644 --- a/tests/unit/lib/doc-migrations/doc-migrations.test.ts +++ b/tests/unit/lib/doc-migrations/doc-migrations.test.ts @@ -11,8 +11,8 @@ import { } from "@/lib/doc-migrations"; const steps: DocMigration[] = [ - { to: 1, run: (doc) => doc.getMap("m").set("one", true) }, - { to: 2, run: (doc) => doc.getMap("m").set("two", true) }, + { to: 1, migrate: (doc) => doc.getMap("m").set("one", true) }, + { to: 2, migrate: (doc) => doc.getMap("m").set("two", true) }, ]; describe("document migrations", () => { @@ -33,7 +33,9 @@ describe("document migrations", () => { }); test("refuses a gap in the list", () => { - expect(() => planDocMigrations(0, [{ to: 2, run: () => {} }])).toThrow(); + expect(() => + planDocMigrations(0, [{ to: 2, migrate: () => {} }]), + ).toThrow(); }); test("runs every step in one transaction under the given origin", () => { From c5e2613f7ddabcbcd2ed8258859e09193fe5b33e Mon Sep 17 00:00:00 2001 From: Harvey Date: Tue, 17 Nov 2026 12:55:19 +0330 Subject: [PATCH 19/19] docs(platform): rewrap the runDocMigrations comment --- lib/doc-migrations/index.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/doc-migrations/index.ts b/lib/doc-migrations/index.ts index bb1c718e..0d645b1d 100644 --- a/lib/doc-migrations/index.ts +++ b/lib/doc-migrations/index.ts @@ -55,9 +55,9 @@ export function planDocMigrations( /** * Runs the steps a plan lists in one transaction under `origin`, so they - * produce one update. For import, where a file is all-or-nothing; the open - * hook in local-session runs one transaction per step instead. Returns the schema reached, or null when there was - * nothing to do. + * produce one update. For import, where a file migrates all or nothing; + * the open hook in local-session runs one transaction per step instead. + * Returns the schema reached, or null when there was nothing to do. */ export function runDocMigrations( doc: Y.Doc,