diff --git a/.github/workflows/desktop-storage-soak.yml b/.github/workflows/desktop-storage-soak.yml new file mode 100644 index 00000000..aca44f4d --- /dev/null +++ b/.github/workflows/desktop-storage-soak.yml @@ -0,0 +1,152 @@ +name: Desktop storage soak + +# Plan T1–T8 (P2.6): the storage worker on node:sqlite inside Electron, +# SIGKILLed during appends, compactions and migrations, quit with writes in +# flight, opened by an older app, and run through a simulated year with +# 1,000 documents. The verifier reopens every library with the engine on +# bun:sqlite. See tests/load/desktop-durability-soak.ts. +# +# Pull requests that touch storage run a short profile (20 kills per case); +# the nightly and manual runs use the plan's 200, and so does a pull request +# labelled `storage-soak-full` (workflow_dispatch needs the file on main). +# Not a required check. +# Read-only token and no secrets, like CI. + +on: + pull_request: + types: [opened, synchronize, reopened, labeled] + paths: + - desktop/storage/** + - desktop/storage-harness/** + - desktop/dev/bun-sqlite-driver.ts + - lib/platform/desktop/** + - lib/platform/doc-store.ts + - tests/helpers/doc-store-contract.ts + - tests/load/desktop-durability-soak.ts + - .github/workflows/desktop-storage-soak.yml + workflow_call: + inputs: + kills: + type: number + default: 200 + workflow_dispatch: + inputs: + kills: + description: Kills per kill case (T1, T2; T1-dense runs half) + type: number + default: 200 + +permissions: + contents: read + +concurrency: + group: desktop-storage-soak-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }} + cancel-in-progress: true + +jobs: + soak: + name: ${{ matrix.group.name }} (${{ matrix.os }}) + runs-on: ${{ matrix.os }} + timeout-minutes: 90 + permissions: + contents: read + strategy: + fail-fast: false + matrix: + os: [macos-26, windows-latest, ubuntu-24.04] + group: + - name: contract, T3–T6 + cases: contract,T3,T4,T5,T6 + - name: T1 kill during append + cases: T1,T1-dense + - name: T2 kill during compaction + cases: T2 + - name: T8 year and startup + cases: T8 + defaults: + run: + shell: bash + env: + KILLS: ${{ github.event_name != 'pull_request' && (inputs.kills || 200) || contains(github.event.pull_request.labels.*.name, 'storage-soak-full') && 200 || 20 }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Set up Bun + uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 + with: + bun-version-file: .tool-versions + + # The root install brings Yjs and the engine's dependencies; the + # harness has its own lockfile with Electron, whose binary downloads + # on first use. + - name: Install + run: | + bun install --frozen-lockfile + cd desktop/storage-harness + bun install --frozen-lockfile + bun -e 'require("electron")' + + # Everything after install runs on Bun. The runner image ships Node, + # so a stub makes any call to it fail loudly. Install comes first: + # lifecycle scripts (husky) call `node`. + - name: Block Node.js + run: | + mkdir -p "$RUNNER_TEMP/no-node" + cat > "$RUNNER_TEMP/no-node/node" <<'EOF' + #!/bin/sh + echo "::error::node was called: $*" >&2 + exit 1 + EOF + chmod +x "$RUNNER_TEMP/no-node/node" + echo "$RUNNER_TEMP/no-node" >> "$GITHUB_PATH" + + # Windows process spawns look for node.exe and never run the sh stub + # above, so put a compiled stub beside it (found in spike S3). + - name: Block Node.js (node.exe on Windows) + if: runner.os == 'Windows' + run: | + cat > "$RUNNER_TEMP/node-stub.ts" <<'EOF' + console.error(`::error::node was called: ${process.argv.slice(2).join(" ")}`); + process.exit(1); + EOF + bun build --compile "$RUNNER_TEMP/node-stub.ts" --outfile "$RUNNER_TEMP/no-node/node.exe" + if cmd.exe //d //c "node --version" > /dev/null 2>&1; then + echo "::error::the node.exe stub is not first on PATH" + exit 1 + fi + + - name: Audit the harness dependencies + working-directory: desktop/storage-harness + run: bun audit --audit-level=high + + - name: Typecheck and build the harness + working-directory: desktop/storage-harness + run: | + bun run typecheck + bun run build + + - name: Allow the Chromium sandbox (electron/electron#42510) + if: runner.os == 'Linux' + run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0 + + - name: Soak + env: + CASES: ${{ matrix.group.cases }} + run: | + if [ "$RUNNER_OS" = Linux ]; then + xvfb-run -a bun tests/load/desktop-durability-soak.ts --case="$CASES" --kills="$KILLS" --out=soak-out + else + bun tests/load/desktop-durability-soak.ts --case="$CASES" --kills="$KILLS" --out=soak-out + fi + + - name: Upload results + if: always() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: storage-soak-${{ matrix.os }}-${{ strategy.job-index }} + path: soak-out + if-no-files-found: ignore + retention-days: 30 diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 242b8b14..667b34bd 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -13,6 +13,8 @@ name: Nightly # the gate env at its default (spec §3.1 N+1 ship gate) # desktop-smoke the D-A12 Electron smoke (desktop-smoke.yml) on macOS, # Windows and Linux +# desktop-storage-soak plan T1–T8 against the storage worker in Electron +# (desktop-storage-soak.yml), 200 kills per case, 3 OSes # # Read-only token and no secrets, like CI. A red run means a regression in # offline editing, long sessions, concurrency or schema rollout safety that @@ -361,3 +363,11 @@ jobs: permissions: contents: read uses: ./.github/workflows/desktop-smoke.yml + + desktop-storage-soak: + name: Desktop storage soak (T1–T8) + permissions: + contents: read + uses: ./.github/workflows/desktop-storage-soak.yml + with: + kills: 200 diff --git a/CHANGELOG.md b/CHANGELOG.md index d357f514..e2e957cf 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,67 @@ since the previous tag. ## [Unreleased] +### Desktop P2.6: the library storage worker + +The SQLite engine behind the desktop library, where documents, folders and +tags live, and the `worker_thread` that runs it inside Electron main (D-A1). +Nothing in the web app or its build uses it yet; P2.5's shell starts the +worker and P2.7 builds the files sidebar on it. + +#### Added + +- **Desktop:** `desktop/storage/engine/`, with no `node:`, `bun:` or + Electron imports, behind a `SqliteDriver` seam (`node:sqlite` in the + worker, `bun:sqlite` in tests and the dev host): + - the format-1 schema, so the Local release needs no later migration: + `docs` (lineage and a `NOT NULL` `link_key` minted at insert, `kind`, the + list columns), `folders`, `tags`, `doc_tags`, `doc_view(doc_id, width, + place, remote_images)`, `import_sources`, `snapshots` and `updates` with + an `AUTOINCREMENT` seq; + - the open order: cloud-synced folders refused, `library.lock` with a pid + check, WAL with `synchronous=FULL` (`fullfsync` on macOS), and the + `format_version` guard before any write, so a newer library opens + read-only and untouched; `quick_check` after an unclean exit; + - format migrations with a `VACUUM INTO` copy first (2 kept), one + transaction per step, and read-only on failure; + - appends and renderer-driven compaction in one transaction each, with + one window per document (`already_open`, `forbidden`, `document_open`); + - documents, folders, tags, Trash (emptied after 30 days) and the sidebar + snapshot, with a change feed: every change the sidebar shows, appends + included, moves `rev`, and the worker sends `index_changed` with the + changed rows; + - a daily `VACUUM INTO` copy on the first idle moment, keeping 7, and a + reader for restoring one document from a day; + - shutdown for the quit flush: WAL checkpoint, `unclean_exit` cleared, + lock released. +- **Desktop:** `desktop/storage/worker.ts` and `client.ts`: the worker + answers main one request at a time with `Result` envelopes and transferred + bytes; the client rejects pending calls with `worker_restarting` if the + worker dies and ends with a 2 s shutdown. +- **Platform:** `lib/doc-migrations/` (the open-time migration hook list, + empty so far, and `checkImport`, which refuses an import that is too + large, isn't Yjs, or carries a node, mark or root this build doesn't + know) and `lib/platform/desktop/rpc-doc-store.ts` (`DocStore` over worker + replies, for `IpcDocStore`). +- **Tests:** P2.1's shared `DocStore` contract on the engine, on + `bun:sqlite` under `bun test` and on `node:sqlite` inside the Electron + worker; engine suites (open, content, index and its change feed, daily + copies, codes, and a property test of random appends, compactions, + crashes and reopens); the import-boundary test's storage rows; and + `tests/load/desktop-durability-soak.ts` with its Electron harness in + `desktop/storage-harness/` for soak cases T1–T6 and T8. +- **CI:** `desktop-storage-soak.yml` on macOS, Windows and Linux: 20 kills + per case on pull requests that touch storage, 200 nightly or with the + `storage-soak-full` label. `bun run typecheck` also checks the new + `tsconfig.desktop-node.json`. + +#### Changed + +- **Docs:** the desktop architecture spec records where P2.6 put the engine + and the harness, the worker-side registry check and `claim`, the + import-boundary allowance for `lib/schema-names.ts`, and that T7 is covered + by `local-session`'s tests, not the storage soak. + ### Desktop P2.1b: `local-session`, the batcher and `MemoryDocStore` The second half of desktop plan P2.1: the platform-neutral storage path that diff --git a/desktop/dev/bun-sqlite-driver.ts b/desktop/dev/bun-sqlite-driver.ts new file mode 100644 index 00000000..4e915202 --- /dev/null +++ b/desktop/dev/bun-sqlite-driver.ts @@ -0,0 +1,56 @@ +/** + * `SqliteDriver` over `bun:sqlite`, for unit tests and the dev storage host + * (architecture §11). Normalizes Bun's differences from `node:sqlite`: a + * missing row is `null` in Bun and `undefined` here, and integers stay + * numbers (`safeIntegers` off). + * + * `close` finalizes every cached statement first: Bun's `close()` leaves a + * connection with live statements open as a zombie that keeps the file + * handles, which on Windows blocks deleting or replacing the library. + */ +import { Database } from "bun:sqlite"; + +import type { + OpenSqlite, + SqliteDriver, + SqliteStatement, + SqlRow, + SqlValue, +} from "@/desktop/storage/engine/driver"; + +export const openBunSqlite: OpenSqlite = (path, { readOnly }) => { + const db = readOnly + ? new Database(path, { readonly: true }) + : new Database(path, { create: true }); + const cache = new Map(); + const statements: { finalize(): void }[] = []; + const driver: SqliteDriver = { + exec: (sql) => db.exec(sql), + prepare(sql) { + const cached = cache.get(sql); + if (cached) return cached; + const statement = db.prepare(sql); + statements.push(statement); + const wrapped: SqliteStatement = { + run: (...params) => { + const result = statement.run(...params); + return { + changes: result.changes, + lastInsertRowid: Number(result.lastInsertRowid), + }; + }, + get: (...params) => statement.get(...params) ?? undefined, + all: (...params) => statement.all(...params), + }; + cache.set(sql, wrapped); + return wrapped; + }, + close: () => { + for (const statement of statements) statement.finalize(); + statements.length = 0; + cache.clear(); + db.close(); + }, + }; + return driver; +}; diff --git a/desktop/storage-harness/.gitignore b/desktop/storage-harness/.gitignore new file mode 100644 index 00000000..8225baa4 --- /dev/null +++ b/desktop/storage-harness/.gitignore @@ -0,0 +1,2 @@ +/node_modules +/dist diff --git a/desktop/storage-harness/build.ts b/desktop/storage-harness/build.ts new file mode 100644 index 00000000..194b77bd --- /dev/null +++ b/desktop/storage-harness/build.ts @@ -0,0 +1,25 @@ +/** + * Bundles the harness main and the real storage worker as CJS for + * Electron, the way `desktop/build/build.ts` (P2.5) will: `node:*` and + * `electron` stay external, everything else (Yjs, the engine) is inlined. + */ +import path from "node:path"; + +const root = import.meta.dir; +const result = await Bun.build({ + entrypoints: [ + path.join(root, "main.ts"), + path.join(root, "../storage/worker.ts"), + ], + outdir: path.join(root, "dist"), + target: "node", + format: "cjs", + external: ["electron"], + naming: "[name].cjs", +}); +if (!result.success) { + for (const log of result.logs) console.error(log); + process.exit(1); +} +for (const output of result.outputs) + console.log(`${path.relative(root, output.path)} ${output.size} B`); diff --git a/desktop/storage-harness/bun.lock b/desktop/storage-harness/bun.lock new file mode 100644 index 00000000..8fb81cc4 --- /dev/null +++ b/desktop/storage-harness/bun.lock @@ -0,0 +1,46 @@ +{ + "lockfileVersion": 2, + "configVersion": 1, + "workspaces": { + "": { + "name": "tinydocy-storage-harness", + "devDependencies": { + "@types/node": "24.19.2", + "electron": "45.3.2", + "typescript": "6.0.3", + }, + }, + }, + "trustedDependencies": [ + "electron", + ], + "packages": { + "@electron-internal/extract-zip": ["@electron-internal/extract-zip@1.0.8", "", {}, "sha512-8iajeLFbzNwM0m3AsQZSZgYREWglAImyGvvFSWU34FgMW8sHlzdwj9QKvv9dgN7jMP591EFUF/sLOC3Jl9YIjQ=="], + + "@electron/get": ["@electron/get@5.1.0", "", { "dependencies": { "debug": "^4.1.1", "env-paths": "^3.0.0", "graceful-fs": "^4.2.11", "progress": "^2.0.3", "semver": "^7.6.3", "sumchecker": "^3.0.1" }, "optionalDependencies": { "undici": "^7.24.4" } }, "sha512-3kSBtG8ObcTVfXanm5vVJ6UnBLEVmVsRk1M+vGqCuMBV+XLCbJYuWQful+yIy0GQDsSlK0kHEriEHn7SPk4EnA=="], + + "@types/node": ["@types/node@24.19.2", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-93d49wCWJkCyN87MxLpNN6KXPCwFw7bVUg79mGUV1OS40JGWwVCr8n61DXCSqz7CV6/VfPkb7xec7uP2M+QHWQ=="], + + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" }, "peerDependencies": { "supports-color": "*" }, "optionalPeers": ["supports-color"] }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + + "electron": ["electron@45.3.2", "", { "dependencies": { "@electron-internal/extract-zip": "^1.0.1", "@electron/get": "^5.0.0", "@types/node": "^24.9.0" }, "bin": { "electron": "cli.js", "install-electron": "install.js" } }, "sha512-bPO0MMvJCdKdPfXgd3ES8zupu+ocr6XmSuRGwyZazTiigTz8jE7rwukSkck0iomS6UZeFuCwzu0uanMD/D34uw=="], + + "env-paths": ["env-paths@3.0.0", "", {}, "sha512-dtJUTepzMW3Lm/NPxRf3wP4642UWhjL2sQxc+ym2YMj1m/H2zDNQOlezafzkHwn6sMstjHTwG6iQQsctDW/b1A=="], + + "graceful-fs": ["graceful-fs@4.2.12", "", {}, "sha512-iuyGF2E4oafQfPsRrG2wtY2ILupMhvtdrhzrxB+0VcjsPpSPttpfYdjnGFGsTi5MaJFjI1jOKHPQ6UTbTj9TAw=="], + + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "progress": ["progress@2.0.3", "", {}, "sha512-7PiHtLll5LdnKIMw100I+8xJXR5gW2QwWYkT6iJva0bXitZKa/XMrSbdmg3r2Xnaidz9Qumd0VPaMrZlF9V9sA=="], + + "semver": ["semver@7.8.6", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-9lpSSU4Mly3mrtruwLs1eLK6+gzbzqeBFwRNjIjE+9LuDpWdYctrQlQ3ej1Ilgi7gknpsTRnPoLP8DpCHqHIJw=="], + + "sumchecker": ["sumchecker@3.0.1", "", { "dependencies": { "debug": "^4.1.0" } }, "sha512-MvjXzkz/BOfyVDkG0oFOtBxHX2u3gKbMHIF/dXblZsgD3BWOFLmHovIpZY7BykJdAjcqRCBi1WYBNdEC9yI7vg=="], + + "typescript": ["typescript@6.0.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-y2TvuxSZPDyQakkFRPZHKFm+KKVqIisdg9/CZwm9ftvKXLP8NRWj38/ODjNbr43SsoXqNuAisEf1GdCxqWcdBw=="], + + "undici": ["undici@7.30.2", "", {}, "sha512-mLyPQnHwUxaM/rideoK/+KL8DaAJLRl4aBCfmX444aCSLnIaomqPLtNRe2jdCYcsk5wZzXrDIQC8ZeYNRAYyIA=="], + + "undici-types": ["undici-types@7.24.6", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="], + } +} diff --git a/desktop/storage-harness/bunfig.toml b/desktop/storage-harness/bunfig.toml new file mode 100644 index 00000000..d968ccb5 --- /dev/null +++ b/desktop/storage-harness/bunfig.toml @@ -0,0 +1,15 @@ +# Supply-chain guardrail: only resolve package versions that have been +# public for at least 3 days (value in seconds). Gives the registry and the +# community time to catch and unpublish compromised releases before +# `bun add` / `bun update` (local or in CI) can pick them up. +[install] +minimumReleaseAge = 259200 + +# Run package bins (next, eslint, playwright, ...) on the Bun runtime +# instead of Node, for `bun run` scripts and `bun `. Plain `bunx` +# ignores this setting; use `bunx --bun` for packages that are not installed. +[run] +bun = true + +# `bun test`: unit tests only (Playwright specs live in tests/e2e), with a +# happy-dom window preloaded. diff --git a/desktop/storage-harness/format-steps.ts b/desktop/storage-harness/format-steps.ts new file mode 100644 index 00000000..7fa8ef63 --- /dev/null +++ b/desktop/storage-harness/format-steps.ts @@ -0,0 +1,18 @@ +/** + * Format steps for soak T5 (migration interrupted). Each step builds a + * table of {@link STEP_ROWS} rows, slow enough that a kill lands inside one. + * The Electron run and the Bun verifier use the same list. + */ +import type { FormatStep } from "../storage/engine/migrations"; + +export const STEP_ROWS = 150_000; +export const T5_TARGET = 4; + +export const T5_STEPS: FormatStep[] = [2, 3, 4].map((to) => ({ + to, + up(db) { + db.exec(`CREATE TABLE soak_step_${to} (n INTEGER NOT NULL) STRICT`); + const insert = db.prepare(`INSERT INTO soak_step_${to} (n) VALUES (?)`); + for (let n = 0; n < STEP_ROWS; n += 1) insert.run(n); + }, +})); diff --git a/desktop/storage-harness/gen.ts b/desktop/storage-harness/gen.ts new file mode 100644 index 00000000..9ea200d8 --- /dev/null +++ b/desktop/storage-harness/gen.ts @@ -0,0 +1,114 @@ +/** + * Deterministic edits for the durability soak, shared by the Electron churn + * and the Bun verifier. Edit `k` appends `";"` to the text `t`, so a + * stored document is valid exactly when its text is `1;2;…;K;` for some K: + * a lost or half-applied batch shows up as a gap or a short K. + * + * `pad` is filler that makes compactions heavy (the plan's 5 MB document) + * without touching `t`. + */ +import * as Y from "yjs"; + +export const TEXT = "t"; +export const PAD = "pad"; + +/** The K of a stored text, or null if it isn't `1;2;…;K;`. */ +export function editCount(text: string): number | null { + if (text === "") return 0; + const parts = text.split(";"); + if (parts.pop() !== "") return null; + for (const [i, part] of parts.entries()) + if (part !== String(i + 1)) return null; + return parts.length; +} + +export function padUpdate(bytes: number): Uint8Array { + const doc = new Y.Doc(); + // A plain repeated string compresses nothing in Yjs: one item. + doc.getText(PAD).insert(0, "x".repeat(bytes)); + return Y.encodeStateAsUpdate(doc); +} + +export type BatchTiming = { debounceMs: number; maxWaitMs: number }; + +/** The product's batching (architecture §5.2), or the dense soak variant. */ +export const PRODUCT_TIMING: BatchTiming = { debounceMs: 250, maxWaitMs: 1000 }; +export const DENSE_TIMING: BatchTiming = { debounceMs: 5, maxWaitMs: 20 }; + +/** + * A minimal batcher: merges updates, flushes `debounceMs` after the last or + * `maxWaitMs` after the first, and keeps one send in flight. Stand-in for + * P2.1's `lib/platform/batcher.ts`, which the real app uses. + */ +export class SoakBatcher { + private pending: Uint8Array[] = []; + private pendingEdits = 0; + private firstAt = 0; + private timer: ReturnType | null = null; + private inFlight: Promise | null = null; + + constructor( + private readonly timing: BatchTiming, + private readonly send: ( + update: Uint8Array, + lastEdit: number, + ) => Promise, + ) {} + + push(update: Uint8Array, edit: number) { + if (this.pending.length === 0) this.firstAt = performance.now(); + this.pending.push(update); + this.pendingEdits = edit; + this.schedule(); + } + + private schedule() { + if (this.timer) clearTimeout(this.timer); + const sinceFirst = performance.now() - this.firstAt; + const wait = Math.max( + 0, + Math.min(this.timing.debounceMs, this.timing.maxWaitMs - sinceFirst), + ); + this.timer = setTimeout(() => void this.flush(), wait); + } + + get busy(): Promise | null { + return this.inFlight; + } + + /** Sends what is queued, after any send in flight. */ + async flush(): Promise { + if (this.timer) clearTimeout(this.timer); + this.timer = null; + while (this.inFlight) await this.inFlight; + if (this.pending.length === 0) return; + const update = Y.mergeUpdates(this.pending); + const lastEdit = this.pendingEdits; + this.pending = []; + this.inFlight = this.send(update, lastEdit).finally(() => { + this.inFlight = null; + }); + await this.inFlight; + } +} + +/** Parses `--key=value` and `--flag` arguments. */ +export function parseArgs(argv: string[]): Record { + const out: Record = {}; + for (const arg of argv) { + const match = arg.match(/^--([\w-]+)(?:=(.*))?$/); + if (match?.[1]) out[match[1]] = match[2] ?? "1"; + } + return out; +} + +/** A small seeded PRNG (mulberry32), so a soak run can be replayed. */ +export function seededRandom(seed: number): () => number { + let a = seed | 0; + return () => { + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; +} diff --git a/desktop/storage-harness/main.ts b/desktop/storage-harness/main.ts new file mode 100644 index 00000000..29715741 --- /dev/null +++ b/desktop/storage-harness/main.ts @@ -0,0 +1,321 @@ +/** + * Electron side of the storage soak (P2.6; plan T1–T8). Runs the real + * storage worker (`worker.cjs`, `node:sqlite`) from a windowless, + * dock-less Electron main. `tests/load/desktop-durability-soak.ts` (Bun) + * launches it, kills it, and verifies the library afterwards with the + * engine on `bun:sqlite`. + * + * Modes (`--mode=`): + * - `contract`: P2.1's shared `DocStore` contract over the worker + * (`CASE_OK` / `CASE_FAIL` lines). + * - `churn`: edits a document with product batching (or `--dense`) until + * killed; `--target=compaction` compacts after every ack; + * `--quit-after-ms` runs the quit flush instead (T4). The worker prints + * `BATCH_BEGIN` / `ACK ` / `COMPACT_BEGIN` / `COMPACT_ACK`; main + * prints `EDITS ` once it has the ack. + * - `migrate`: opens the library on main's thread with the T5 format steps. + * - `readonly`: opens a newer library (T6) and tries to write. + * - `commit-timing`: append round trips with `fullfsync` on or off (T3). + * - `startup`: worker spawn → ready → sidebar rows (T8, the §10 budget). + * + * An uncaught throw in Electron main opens a modal dialog, so every failure + * exits with `process.exit(1)` after printing `HARNESS_FAIL`. + */ +import { mkdtempSync, rmSync, writeSync } from "node:fs"; +import { tmpdir } from "node:os"; +import path from "node:path"; + +import { app } from "electron"; +import * as Y from "yjs"; + +import { createRpcDocStore } from "../../lib/platform/desktop/rpc-doc-store"; +import type { DocStore } from "../../lib/platform/doc-store"; +import { + DOC_STORE_CONTRACT, + type DocStoreHarness, +} from "../../tests/helpers/doc-store-contract"; +import { StorageClient } from "../storage/client"; +import { openNodeSqlite } from "../storage/drivers/node-sqlite"; +import { Library } from "../storage/engine/library"; +import { createNodeHost } from "../storage/node-host"; +import { T5_STEPS, T5_TARGET } from "./format-steps"; +import { + DENSE_TIMING, + editCount, + PRODUCT_TIMING, + parseArgs, + SoakBatcher, + seededRandom, + TEXT, +} from "./gen"; + +const args = parseArgs(process.argv.slice(1)); +const say = (line: string) => writeSync(1, `${line}\n`); +const fail = (detail: unknown): never => { + const text = + detail instanceof Error + ? `${detail.message}\n${detail.stack}` + : String(detail); + writeSync(2, `HARNESS_FAIL ${text}\n`); + process.exit(1); +}; +process.on("uncaughtException", fail); +process.on("unhandledRejection", fail); + +app.disableHardwareAcceleration(); +// Only app.exit ends a run; there are no windows to close. +app.on("window-all-closed", () => {}); + +// Not __dirname: Bun.build inlines the source directory (S1, S2). +const workerPath = () => path.join(app.getAppPath(), "worker.cjs"); +const dir = () => args.dir ?? fail("--dir is required"); + +/** Copies bytes before they are transferred, as IPC would. */ +function copied(value: T): T { + if (value instanceof Uint8Array) return value.slice() as T; + if (Array.isArray(value)) return value.map(copied) as T; + if (value && typeof value === "object") + return Object.fromEntries( + Object.entries(value).map(([k, v]) => [k, copied(v)]), + ) as T; + return value; +} + +function storeFor(client: StorageClient, owner: string): DocStore { + return createRpcDocStore((op, callArgs) => + op === "flush" + ? (client.call("flush", {}) as never) + : (client.call(op, { ...copied(callArgs), owner } as never) as never), + ); +} + +/** + * P2.1's shared DocStore contract on `node:sqlite` in the real worker: one + * worker and temp library per case, one owner id per connected client. + */ +async function contract() { + let passed = 0; + let failed = 0; + const harness: DocStoreHarness = { + exclusiveOpen: true, + async createLibrary() { + const libraryDir = mkdtempSync(path.join(tmpdir(), "contract-")); + const client = await StorageClient.start(workerPath(), { + dir: libraryDir, + }); + let clients = 0; + return { + connect: () => storeFor(client, `client-${++clients}`), + async createDoc(docId, schema) { + await client.call("createDoc", { id: docId, schema }); + }, + async dispose() { + await client.shutdown(); + rmSync(libraryDir, { + recursive: true, + force: true, + maxRetries: 10, + retryDelay: 200, + }); + }, + }; + }, + }; + for (const testCase of DOC_STORE_CONTRACT) { + if (testCase.requires && !harness[testCase.requires]) continue; + try { + await testCase.run(harness); + passed += 1; + say(`CASE_OK ${testCase.name}`); + } catch (error) { + failed += 1; + say(`CASE_FAIL ${testCase.name}: ${String(error)}`); + } + } + say(`CONTRACT ${passed} passed, ${failed} failed`); + if (failed > 0) fail(`${failed} contract case(s) failed`); +} + +async function churn() { + const docId = args.doc ?? fail("--doc is required"); + const random = seededRandom(Number(args.seed ?? 1)); + const client = await StorageClient.start(workerPath(), { + dir: dir(), + trace: true, + synchronous: args.synchronous === "OFF" ? "OFF" : "FULL", + fullfsync: args.fullfsync !== "0", + }); + const owner = "churn"; + const loaded = await client.call("load", { docId, owner }); + const doc = new Y.Doc(); + if (loaded.snapshot) Y.applyUpdate(doc, loaded.snapshot); + for (const update of loaded.updates) Y.applyUpdate(doc, update); + let k = + editCount(doc.getText(TEXT).toString()) ?? fail("stored text is broken"); + let ackedSeq = loaded.maxSeq; + const compactEvery = args.target === "compaction"; + + const batcher = new SoakBatcher( + args.dense ? DENSE_TIMING : PRODUCT_TIMING, + async (update, lastEdit) => { + ackedSeq = await client.call("append", { + docId, + owner, + record: { origin: "local", update }, + }); + say(`EDITS ${ackedSeq} ${lastEdit}`); + if (compactEvery) { + // Same task as the encode: no append is in flight (architecture §5.2). + const state = Y.encodeStateAsUpdate(doc); + await client.call("compact", { + docId, + owner, + state, + upToSeq: ackedSeq, + }); + } + }, + ); + let pendingEdit: Uint8Array | null = null; + doc.on("update", (update: Uint8Array) => { + pendingEdit = update; + }); + const quitAfter = args["quit-after-ms"] + ? Number(args["quit-after-ms"]) + : null; + const started = performance.now(); + say(`READY ${k}`); + for (;;) { + // A burst of 1–8 edits 2–10 ms apart, then a 260–400 ms pause. + const burst = 1 + Math.floor(random() * 8); + for (let i = 0; i < burst; i += 1) { + k += 1; + const text = doc.getText(TEXT); + text.insert(text.length, `${k};`); + if (pendingEdit) batcher.push(pendingEdit, k); + pendingEdit = null; + await new Promise((r) => setTimeout(r, 2 + random() * 8)); + } + if (quitAfter !== null && performance.now() - started >= quitAfter) break; + await new Promise((r) => setTimeout(r, 260 + random() * 140)); + } + // T4: the quit flush (architecture §8.2): flush every batch, then shut + // the worker down. + await batcher.flush(); + const outcome = await client.shutdown(); + say(`QUIT ${outcome} ${k} ${ackedSeq}`); +} + +async function migrate() { + say("READY"); + const library = Library.open({ + dir: dir(), + host: createNodeHost(), + openSqlite: openNodeSqlite, + formatVersion: T5_TARGET, + formatSteps: T5_STEPS, + }); + say(`MIGRATED ${library.status.formatVersion} ${library.status.readOnly}`); + library.shutdown(); +} + +async function readonly() { + const client = await StorageClient.start(workerPath(), { dir: dir() }); + say(`STATUS ${JSON.stringify(client.status)}`); + const sidebar = await client.call("sidebar", {}); + say(`SIDEBAR ${sidebar.docs.length}`); + try { + await client.call("createDoc", {}); + say("WRITE accepted"); + } catch (error) { + say(`WRITE refused ${(error as { code?: string }).code}`); + } + say(`SHUTDOWN ${await client.shutdown()}`); +} + +function percentile(sorted: number[], p: number) { + return ( + sorted[Math.min(sorted.length - 1, Math.floor(sorted.length * p))] ?? 0 + ); +} + +async function commitTiming() { + const count = Number(args.count ?? 300); + const client = await StorageClient.start(workerPath(), { + dir: dir(), + fullfsync: args.fullfsync !== "0", + }); + const { id } = await client.call("createDoc", {}); + await client.call("load", { docId: id, owner: "timing" }); + const doc = new Y.Doc(); + const times: number[] = []; + for (let i = 0; i < count; i += 1) { + let update: Uint8Array = new Uint8Array(); + doc.once("update", (u: Uint8Array) => { + update = u; + }); + doc.getText(TEXT).insert(0, "y".repeat(4096)); + const t0 = performance.now(); + await client.call("append", { + docId: id, + owner: "timing", + record: { origin: "local", update }, + }); + times.push(performance.now() - t0); + } + times.sort((a, b) => a - b); + const stats = await client.call("stats", {}); + say( + `TIMING ${JSON.stringify({ + fullfsync: args.fullfsync !== "0", + count, + p50: percentile(times, 0.5), + p95: percentile(times, 0.95), + max: times.at(-1), + pragmas: stats.pragmas, + sqlite: stats.sqliteVersion, + })}`, + ); + await client.shutdown(); +} + +async function startup() { + // performance.now() counts from process start in Node and Electron main. + const spawnAt = performance.now(); + const client = await StorageClient.start(workerPath(), { dir: dir() }); + const readyAt = performance.now(); + const sidebar = await client.call("sidebar", {}); + const rowsAt = performance.now(); + say( + `STARTUP ${JSON.stringify({ + processToSpawnMs: spawnAt, + workerReadyMs: readyAt - spawnAt, + processToReadyMs: readyAt, + openMs: client.timings.openMs, + sidebarCallMs: rowsAt - readyAt, + processToSidebarMs: rowsAt, + spawnToSidebarMs: rowsAt - spawnAt, + docs: sidebar.docs.length, + folders: sidebar.folders.length, + uncleanExit: client.status.uncleanExit, + })}`, + ); + await client.shutdown(); +} + +const MODES: Record Promise> = { + contract, + churn, + migrate, + readonly, + "commit-timing": commitTiming, + startup, +}; + +app.whenReady().then(async () => { + app.dock?.hide(); + if (process.platform === "darwin") app.setActivationPolicy("accessory"); + const run = MODES[args.mode ?? ""] ?? fail(`unknown --mode=${args.mode}`); + await run(); + app.exit(0); +}, fail); diff --git a/desktop/storage-harness/package.json b/desktop/storage-harness/package.json new file mode 100644 index 00000000..ae4a58ab --- /dev/null +++ b/desktop/storage-harness/package.json @@ -0,0 +1,18 @@ +{ + "name": "tinydocy-storage-harness", + "private": true, + "type": "module", + "main": "dist/main.cjs", + "scripts": { + "build": "bun build.ts", + "typecheck": "tsc --noEmit -p ." + }, + "devDependencies": { + "@types/node": "24.19.2", + "electron": "45.3.2", + "typescript": "6.0.3" + }, + "trustedDependencies": [ + "electron" + ] +} diff --git a/desktop/storage-harness/tsconfig.json b/desktop/storage-harness/tsconfig.json new file mode 100644 index 00000000..bacd8dd1 --- /dev/null +++ b/desktop/storage-harness/tsconfig.json @@ -0,0 +1,15 @@ +{ + "compilerOptions": { + "target": "ES2023", + "lib": ["ES2023"], + "module": "Preserve", + "moduleResolution": "bundler", + "moduleDetection": "force", + "types": ["node"], + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "paths": { "@/*": ["../../*"] } + }, + "include": ["main.ts", "gen.ts", "format-steps.ts"] +} diff --git a/desktop/storage/client.ts b/desktop/storage/client.ts new file mode 100644 index 00000000..42129d82 --- /dev/null +++ b/desktop/storage/client.ts @@ -0,0 +1,178 @@ +/** + * Main's handle on the storage worker (desktop architecture §2, §6.3). + * `doc-handlers` (P2.5) forwards validated `doc.*` and `index.*` calls + * through {@link StorageClient.call}; the quit coordinator ends with + * {@link StorageClient.shutdown}. + * + * Bytes in a request are transferred, not copied. If the worker dies, every + * pending call rejects with `worker_restarting`, which the renderer's + * batcher retries (resending an append is harmless). + */ +import { Worker } from "node:worker_threads"; + +import type { StorageErrorEnvelope } from "./engine/errors"; +import type { LibraryStatus } from "./engine/library"; +import type { IndexChange } from "./engine/library-index"; +import { + type StorageMessage, + type StorageOp, + type StorageOps, + type StorageWorkerOptions, + transferablesOf, +} from "./protocol"; + +export class StorageCallError extends Error { + readonly code: StorageErrorEnvelope["code"]; + readonly retryable: boolean; + constructor(envelope: StorageErrorEnvelope) { + super(envelope.message); + this.name = "StorageCallError"; + this.code = envelope.code; + this.retryable = envelope.retryable; + } +} + +type Pending = { resolve(value: unknown): void; reject(error: Error): void }; + +export type StorageClientEvents = { + onIndexChanged?: (rev: number, changes: IndexChange[]) => void; + /** The worker exited without being asked to (crash). */ + onExit?: (code: number) => void; +}; + +export class StorageClient { + readonly status: LibraryStatus; + /** Worker spawn to `ready`, and the open alone inside it. */ + readonly timings: { readyMs: number; openMs: number }; + private readonly worker: Worker; + private readonly pending = new Map(); + private nextId = 1; + private closing = false; + + private constructor( + worker: Worker, + status: LibraryStatus, + timings: { readyMs: number; openMs: number }, + events: StorageClientEvents, + ) { + this.worker = worker; + this.status = status; + this.timings = timings; + worker.on("message", (message: StorageMessage) => { + if (message.type === "reply") { + const pending = this.pending.get(message.id); + if (!pending) return; + this.pending.delete(message.id); + if (message.result.ok) pending.resolve(message.result.value); + else pending.reject(new StorageCallError(message.result.error)); + } else if (message.type === "index_changed") { + events.onIndexChanged?.(message.rev, message.changes); + } + }); + worker.on("exit", (code) => { + this.failPending(); + if (!this.closing) events.onExit?.(code); + }); + } + + /** + * Starts the worker and waits for `ready`. Rejects with the open error + * (`cloud_folder`, `locked_elsewhere`, `corrupt`…) if the library can't + * open; a read-only library still resolves, with `status.readOnly` set. + */ + static start( + workerPath: string, + options: StorageWorkerOptions, + events: StorageClientEvents = {}, + ): Promise { + const started = performance.now(); + const worker = new Worker(workerPath, { workerData: options }); + return new Promise((resolve, reject) => { + const onError = (error: Error) => reject(error); + worker.once("error", onError); + worker.once("message", (message: StorageMessage) => { + worker.off("error", onError); + if (message.type === "ready") { + resolve( + new StorageClient( + worker, + message.status, + { + readyMs: performance.now() - started, + openMs: message.openMs, + }, + events, + ), + ); + } else if (message.type === "open_failed") { + void worker.terminate(); + reject(new StorageCallError(message.error)); + } else { + void worker.terminate(); + reject(new Error(`unexpected first message ${message.type}`)); + } + }); + }); + } + + call( + op: K, + args: StorageOps[K]["args"], + ): Promise { + if (this.closing && op !== "shutdown") + return Promise.reject( + new StorageCallError({ + code: "library_unavailable", + retryable: false, + message: "the library is closing", + }), + ); + const id = this.nextId++; + return new Promise((resolve, reject) => { + this.pending.set(id, { + resolve: resolve as (value: unknown) => void, + reject, + }); + this.worker.postMessage({ id, op, args }, transferablesOf(args)); + }); + } + + /** + * The quit flush's last step (architecture §8.2): every call already sent + * commits first, then the worker checkpoints, clears `unclean_exit` and + * closes. After `timeoutMs` it is terminated; WAL keeps every committed + * row, and the next open runs `quick_check`. + */ + async shutdown(timeoutMs = 2000): Promise<"clean" | "terminated"> { + if (this.closing) return "clean"; + this.closing = true; + const exited = new Promise((resolve) => + this.worker.once("exit", () => resolve()), + ); + let timer: ReturnType | undefined; + const timeout = new Promise<"terminated">((resolve) => { + timer = setTimeout(() => resolve("terminated"), timeoutMs); + }); + const clean = this.call("shutdown", {}) + .then(() => exited) + .then(() => "clean" as const); + const outcome = await Promise.race([clean, timeout]).catch( + () => "terminated" as const, + ); + clearTimeout(timer); + if (outcome === "terminated") await this.worker.terminate(); + return outcome; + } + + private failPending() { + for (const pending of this.pending.values()) + pending.reject( + new StorageCallError({ + code: "worker_restarting", + retryable: true, + message: "the storage worker stopped", + }), + ); + this.pending.clear(); + } +} diff --git a/desktop/storage/drivers/node-sqlite.ts b/desktop/storage/drivers/node-sqlite.ts new file mode 100644 index 00000000..a3375a6c --- /dev/null +++ b/desktop/storage/drivers/node-sqlite.ts @@ -0,0 +1,64 @@ +/** + * `SqliteDriver` over `node:sqlite`, for the Electron storage worker + * (D-A1). `node:sqlite` returns `undefined` for a missing row and plain + * numbers by default, which is what the driver contract wants; BLOBs come + * back as fresh `Uint8Array`s. Loaded with `process.getBuiltinModule`, so a + * build of Electron without it fails here with a clear error (D-A12). + */ +import type { DatabaseSync } from "node:sqlite"; + +import type { + OpenSqlite, + SqliteDriver, + SqliteStatement, + SqlRow, + SqlValue, +} from "../engine/driver"; + +type NodeSqlite = typeof import("node:sqlite"); + +function loadNodeSqlite(): NodeSqlite { + const sqlite = process.getBuiltinModule("node:sqlite") as + | NodeSqlite + | undefined; + if (!sqlite) throw new Error("node:sqlite is not built into this runtime"); + return sqlite; +} + +function wrap(db: DatabaseSync): SqliteDriver { + const cache = new Map(); + return { + exec: (sql) => db.exec(sql), + prepare(sql) { + const cached = cache.get(sql); + if (cached) return cached; + const statement = db.prepare(sql); + const wrapped: SqliteStatement = { + run: (...params: SqlValue[]) => { + const result = statement.run(...params); + return { + changes: Number(result.changes), + lastInsertRowid: Number(result.lastInsertRowid), + }; + }, + get: (...params: SqlValue[]) => + statement.get(...params) as SqlRow | undefined, + all: (...params: SqlValue[]) => statement.all(...params) as SqlRow[], + }; + cache.set(sql, wrapped); + return wrapped; + }, + close: () => db.close(), + }; +} + +export const openNodeSqlite: OpenSqlite = (path, { readOnly }) => { + const { DatabaseSync } = loadNodeSqlite(); + return wrap( + new DatabaseSync(path, { + readOnly, + // The engine turns them on itself, the same on both drivers. + enableForeignKeyConstraints: false, + }), + ); +}; diff --git a/desktop/storage/engine/cloud-folder.ts b/desktop/storage/engine/cloud-folder.ts new file mode 100644 index 00000000..a87b1387 --- /dev/null +++ b/desktop/storage/engine/cloud-folder.ts @@ -0,0 +1,89 @@ +/** + * Refuses a library inside a cloud-synced folder (plan Appendix A, P2.6; + * architecture §5.4 `open.ts`). A sync client that uploads `library.sqlite` + * without its `-wal` file, or restores an older copy under a live + * connection, corrupts the library. `userData` never lives in one by + * default; this catches a moved home folder, OneDrive Known Folder Move and + * a `TINYDOCY_USER_DATA` override. + */ +import type { StorageHost } from "./host"; + +export type CloudProvider = + | "icloud" + | "onedrive" + | "dropbox" + | "google_drive" + | "box" + | "other"; + +// Folder names sync clients create. Compared case-insensitively against +// every path segment. +const SEGMENTS: ReadonlyArray<[RegExp, CloudProvider]> = [ + [/^mobile documents$/i, "icloud"], // ~/Library/Mobile Documents (iCloud Drive) + [/^icloud ?drive/i, "icloud"], // Windows iCloud for Windows + [/^onedrive( |$|-)/i, "onedrive"], // OneDrive, OneDrive - Contoso + [/^dropbox( |$|\()/i, "dropbox"], // Dropbox, Dropbox (Personal) + [/^google ?drive/i, "google_drive"], + [/^my drive$/i, "google_drive"], + [/^box( |$)/i, "box"], +]; + +// macOS File Provider puts every modern sync client under +// ~/Library/CloudStorage/-. +const CLOUD_STORAGE: ReadonlyArray<[RegExp, CloudProvider]> = [ + [/^onedrive/i, "onedrive"], + [/^dropbox/i, "dropbox"], + [/^googledrive/i, "google_drive"], + [/^box/i, "box"], + [/^icloud/i, "icloud"], +]; + +const ONEDRIVE_ENV = ["OneDrive", "OneDriveConsumer", "OneDriveCommercial"]; + +function segmentsOf(path: string): string[] { + return path.split(/[\\/]+/).filter(Boolean); +} + +function isUnder(path: string, root: string, caseInsensitive: boolean) { + const a = segmentsOf(path); + const b = segmentsOf(root); + if (b.length === 0 || b.length > a.length) return false; + return b.every((part, i) => + caseInsensitive + ? part.toLowerCase() === a[i]?.toLowerCase() + : part === a[i], + ); +} + +/** The cloud provider syncing `dir`, or null. Symlinks are resolved first. */ +export function cloudFolderOf( + dir: string, + host: Pick, +): CloudProvider | null { + const resolved = host.realpath(dir); + const caseInsensitive = host.platform !== "linux"; + // The home folder's own segments are a user name, not a sync folder: a + // person called "box" or "dropbox" must still be able to start the app. + const home = host.homeDir ? host.realpath(host.homeDir) : ""; + const skip = + home && isUnder(resolved, home, caseInsensitive) + ? segmentsOf(home).length + : 0; + for (const name of ONEDRIVE_ENV) { + const root = host.env[name]; + if (root && isUnder(resolved, root, caseInsensitive)) return "onedrive"; + } + const parts = segmentsOf(resolved); + for (const [i, part] of parts.entries()) { + if (i < skip) continue; + if (part.toLowerCase() === "cloudstorage") { + const next = parts[i + 1] ?? ""; + for (const [pattern, provider] of CLOUD_STORAGE) + if (pattern.test(next)) return provider; + return "other"; + } + for (const [pattern, provider] of SEGMENTS) + if (pattern.test(part)) return provider; + } + return null; +} diff --git a/desktop/storage/engine/content.ts b/desktop/storage/engine/content.ts new file mode 100644 index 00000000..b45ed2b5 --- /dev/null +++ b/desktop/storage/engine/content.ts @@ -0,0 +1,209 @@ +/** + * Document content: `load`, `append`, `compact` and `stateOf` (architecture + * §5.2, §5.4 `content.ts`). The renderer owns the Y.Doc and compacts it + * (D-A3); this module only stores bytes. It never parses an update, except + * in {@link stateOf}, which merges stored rows for export and restore. + * + * Rule one: a Yjs update is never dropped once acknowledged. `append` + * resolves only after COMMIT, and `compact` refuses an `upToSeq` that could + * cover a row the renderer has not seen. + */ +import * as Y from "yjs"; + +import type { SqliteDriver } from "./driver"; +import { StorageError } from "./errors"; +import { bumpRev, scalar, transaction } from "./sql"; + +/** The stored `updates.origin` values: `UPDATE_ORIGINS` in `lib/platform/doc-store.ts`. */ +export const STORED_ORIGINS = [ + "local", + "migration", + "remote", + "restore", +] as const; +export type StoredOrigin = (typeof STORED_ORIGINS)[number]; + +export type LoadedContent = { + snapshot: Uint8Array | null; + updates: Uint8Array[]; + maxSeq: number; + schema: number; +}; + +export type AppendInput = { + origin: StoredOrigin; + update: Uint8Array; + title?: string; + snippet?: string; + schemaTo?: number; +}; + +/** Title and snippet caps (architecture §7 `str(200)`; details spec §6.3). */ +export const MAX_TITLE_LENGTH = 200; +export const MAX_SNIPPET_LENGTH = 300; + +function docExists(db: SqliteDriver, docId: string): boolean { + return scalar(db, "SELECT 1 FROM docs WHERE id = ?", docId) !== undefined; +} + +function snapshotUpto(db: SqliteDriver, docId: string): number { + const value = scalar( + db, + "SELECT upto_seq FROM snapshots WHERE doc_id = ?", + docId, + ); + return typeof value === "number" ? value : 0; +} + +/** + * The highest seq this document has: of its rows, or of the rows its + * snapshot replaced. A compaction can't claim to cover more than this. + */ +function lastDocSeq(db: SqliteDriver, docId: string): number { + const value = scalar( + db, + "SELECT MAX(seq) FROM updates WHERE doc_id = ?", + docId, + ); + return Math.max( + typeof value === "number" ? value : 0, + snapshotUpto(db, docId), + ); +} + +export function load(db: SqliteDriver, docId: string): LoadedContent { + const doc = db.prepare("SELECT schema FROM docs WHERE id = ?").get(docId); + if (!doc) throw new StorageError("not_found", "no such document"); + const snap = db + .prepare("SELECT data, upto_seq FROM snapshots WHERE doc_id = ?") + .get(docId); + const upto = typeof snap?.upto_seq === "number" ? snap.upto_seq : 0; + const rows = db + .prepare( + "SELECT seq, data FROM updates WHERE doc_id = ? AND seq > ? ORDER BY seq", + ) + .all(docId, upto); + const last = rows.at(-1)?.seq; + return { + snapshot: snap ? (snap.data as Uint8Array) : null, + updates: rows.map((row) => row.data as Uint8Array), + maxSeq: typeof last === "number" ? last : upto, + schema: doc.schema as number, + }; +} + +function validateAppend(input: AppendInput) { + if (!STORED_ORIGINS.includes(input.origin)) + throw new StorageError("invalid_argument", "unknown origin"); + if (!(input.update instanceof Uint8Array) || input.update.byteLength === 0) + throw new StorageError("invalid_argument", "empty update"); + if ( + input.schemaTo !== undefined && + (!Number.isInteger(input.schemaTo) || input.schemaTo < 0) + ) + throw new StorageError("invalid_argument", "bad schemaTo"); +} + +/** + * One row, plus the document's list metadata, in one transaction. Returns + * the row's seq once committed. + */ +export function append( + db: SqliteDriver, + docId: string, + input: AppendInput, + now: number, +): number { + validateAppend(input); + return transaction(db, () => { + const updated = db + .prepare( + `UPDATE docs SET + modified_at = ?, + size_bytes = size_bytes + ?, + title = COALESCE(?, title), + snippet = COALESCE(?, snippet), + schema = COALESCE(?, schema) + WHERE id = ?`, + ) + .run( + now, + input.update.byteLength, + input.title === undefined + ? null + : input.title.slice(0, MAX_TITLE_LENGTH), + input.snippet === undefined + ? null + : input.snippet.slice(0, MAX_SNIPPET_LENGTH), + input.schemaTo ?? null, + docId, + ); + if (updated.changes === 0) + throw new StorageError("not_found", "no such document"); + // Title, snippet and modified time are on the sidebar (architecture §8.1). + bumpRev(db); + return db + .prepare( + "INSERT INTO updates (doc_id, data, origin, created_at) VALUES (?, ?, ?, ?)", + ) + .run(docId, input.update, input.origin, now).lastInsertRowid; + }); +} + +/** + * Stores `state` as the snapshot and deletes rows `seq ≤ upToSeq`, in one + * transaction (D-A3). A stale compaction (`upToSeq` below the stored + * snapshot's) changes nothing. An `upToSeq` past the document's last row + * is refused: it could claim rows the renderer has never seen. + */ +export function compact( + db: SqliteDriver, + docId: string, + state: Uint8Array, + upToSeq: number, +): void { + if (!Number.isInteger(upToSeq) || upToSeq < 0) + throw new StorageError("invalid_argument", "bad upToSeq"); + if (!(state instanceof Uint8Array) || state.byteLength === 0) + throw new StorageError("invalid_argument", "empty state"); + transaction(db, () => { + if (!docExists(db, docId)) + throw new StorageError("not_found", "no such document"); + if (upToSeq > lastDocSeq(db, docId)) + throw new StorageError( + "invalid_argument", + "upToSeq is past the document's rows", + ); + if (upToSeq < snapshotUpto(db, docId)) return; + db.prepare( + `INSERT INTO snapshots (doc_id, data, upto_seq) VALUES (?, ?, ?) + ON CONFLICT (doc_id) DO UPDATE SET data = excluded.data, upto_seq = excluded.upto_seq`, + ).run(docId, state, upToSeq); + db.prepare("DELETE FROM updates WHERE doc_id = ? AND seq <= ?").run( + docId, + upToSeq, + ); + db.prepare( + `UPDATE docs SET size_bytes = ? + COALESCE( + (SELECT SUM(LENGTH(data)) FROM updates WHERE doc_id = ?), 0) + WHERE id = ?`, + ).run(state.byteLength, docId, docId); + }); +} + +/** + * The document's whole state as one update (for `.tinydoc`, restore and + * the daily-copy reader). Merges without building a Y.Doc. + */ +export function stateOf(content: LoadedContent): Uint8Array { + const parts = content.snapshot + ? [content.snapshot, ...content.updates] + : content.updates; + if (parts.length === 0) return Y.encodeStateAsUpdate(new Y.Doc()); + return Y.mergeUpdates(parts); +} + +/** Rows waiting for compaction, for diagnostics and T8. */ +export function pendingRows(db: SqliteDriver): number { + return Number(scalar(db, "SELECT COUNT(*) FROM updates") ?? 0); +} diff --git a/desktop/storage/engine/daily-copy.ts b/desktop/storage/engine/daily-copy.ts new file mode 100644 index 00000000..884a4a70 --- /dev/null +++ b/desktop/storage/engine/daily-copy.ts @@ -0,0 +1,69 @@ +/** + * Daily copies (D-A6; architecture §5.4 `daily-copy.ts`): once a local day, + * on the first idle moment, `VACUUM INTO daily/library-.sqlite`, + * keeping the newest 7. They are the Local release's restore points + * ("Restore from a daily snapshot"), and its protection against a corrupt + * library. + * + * Each copy is written to `.partial` and renamed, so a kill mid-copy + * never leaves a half file under a real name. Retention is by count, so a + * clock jump can't delete good copies early. + */ +import type { SqliteDriver } from "./driver"; +import type { StorageHost } from "./host"; +import { readMeta, writeMeta } from "./sql"; + +export const DAILY_DIR = "daily"; +export const DAILY_KEEP = 7; +const DAILY = /^library-\d{4}-\d{2}-\d{2}\.sqlite$/; + +/** `YYYY-MM-DD` in local time: "today" is the person's day, not UTC's. */ +export function localDate(ms: number): string { + const date = new Date(ms); + const pad = (n: number) => String(n).padStart(2, "0"); + return `${date.getFullYear()}-${pad(date.getMonth() + 1)}-${pad(date.getDate())}`; +} + +export function dailyCopyName(date: string): string { + return `library-${date}.sqlite`; +} + +export function listDailyCopies(host: StorageHost, dir: string): string[] { + return host + .list(host.join(dir, DAILY_DIR)) + .filter((file) => DAILY.test(file)) + .sort(); +} + +export function isDailyCopyDue(db: SqliteDriver, now: number): boolean { + return readMeta(db, "last_daily_copy") !== localDate(now); +} + +/** + * Writes today's copy if there isn't one yet. Returns the copy's file name, + * or null when it was already taken today. + */ +export function takeDailyCopy( + db: SqliteDriver, + host: StorageHost, + dir: string, + now: number, +): string | null { + if (!isDailyCopyDue(db, now)) return null; + const date = localDate(now); + const folder = host.join(dir, DAILY_DIR); + host.mkdirp(folder); + for (const file of host.list(folder)) + if (file.endsWith(".partial")) host.remove(host.join(folder, file)); + const name = dailyCopyName(date); + const target = host.join(folder, name); + const partial = `${target}.partial`; + db.prepare("VACUUM INTO ?").run(partial); + if (host.exists(target)) host.remove(target); + host.rename(partial, target); + writeMeta(db, "last_daily_copy", date); + const copies = listDailyCopies(host, dir); + for (const old of copies.slice(0, -DAILY_KEEP)) + host.remove(host.join(folder, old)); + return name; +} diff --git a/desktop/storage/engine/driver.ts b/desktop/storage/engine/driver.ts new file mode 100644 index 00000000..7f28044d --- /dev/null +++ b/desktop/storage/engine/driver.ts @@ -0,0 +1,96 @@ +/** + * The engine's only view of SQLite (desktop architecture §3, §12 "Mobile"). + * Two drivers implement it: `node:sqlite` in the Electron storage worker + * (`desktop/storage/drivers/node-sqlite.ts`) and `bun:sqlite` for unit tests + * and the dev storage host (`desktop/dev/bun-sqlite-driver.ts`). + * + * Drivers normalize what differs between the two runtimes, so the engine + * behaves the same on both: + * - a missing row is `undefined` (Bun returns `null`); + * - integers are numbers (no `bigint`); + * - a BLOB is a `Uint8Array` the caller owns; + * - parameters are positional `?` only; + * - a failure carries the SQLite extended result code (see + * {@link sqliteResultCode}). + */ +import { StorageError, type StorageErrorCode } from "./errors"; + +export type SqlValue = string | number | null | Uint8Array; +export type SqlRow = Record; + +export interface SqliteStatement { + run(...params: SqlValue[]): { changes: number; lastInsertRowid: number }; + get(...params: SqlValue[]): SqlRow | undefined; + all(...params: SqlValue[]): SqlRow[]; +} + +export interface SqliteDriver { + exec(sql: string): void; + prepare(sql: string): SqliteStatement; + close(): void; +} + +export type OpenSqlite = ( + path: string, + options: { readOnly: boolean }, +) => SqliteDriver; + +/** + * The SQLite extended result code of a driver error, or null when the error + * did not come from SQLite. Bun puts it in `errno`, Node in `errcode`. + */ +export function sqliteResultCode(error: unknown): number | null { + if (typeof error !== "object" || error === null) return null; + const { errno, errcode } = error as { errno?: unknown; errcode?: unknown }; + if (typeof errcode === "number") return errcode; + // Bun's SQLiteError: its constructor reports as Error, its name doesn't. + if (typeof errno === "number" && (error as Error).name === "SQLiteError") + return errno; + return null; +} + +// Primary result codes (https://sqlite.org/rescode.html). +const SQLITE_PERM = 3; +const SQLITE_BUSY = 5; +const SQLITE_LOCKED = 6; +const SQLITE_READONLY = 8; +const SQLITE_IOERR = 10; +const SQLITE_CORRUPT = 11; +const SQLITE_FULL = 13; +const SQLITE_CANTOPEN = 14; +const SQLITE_CONSTRAINT = 19; +const SQLITE_NOTADB = 26; + +/** Maps a driver failure to the storage error a caller sees. */ +export function storageErrorFrom(error: unknown): StorageError { + if (error instanceof StorageError) return error; + const code = sqliteResultCode(error); + const message = error instanceof Error ? error.message : String(error); + let mapped: StorageErrorCode = "io_error"; + switch (code === null ? null : code & 0xff) { + case SQLITE_FULL: + mapped = "disk_full"; + break; + case SQLITE_CORRUPT: + case SQLITE_NOTADB: + mapped = "corrupt"; + break; + case SQLITE_BUSY: + case SQLITE_LOCKED: + mapped = "locked_elsewhere"; + break; + case SQLITE_READONLY: + case SQLITE_PERM: + mapped = "library_read_only"; + break; + case SQLITE_CONSTRAINT: + // A duplicate tag name or link key: the caller's input, not the disk. + mapped = "invalid_argument"; + break; + case SQLITE_CANTOPEN: + case SQLITE_IOERR: + mapped = "io_error"; + break; + } + return new StorageError(mapped, message); +} diff --git a/desktop/storage/engine/errors.ts b/desktop/storage/engine/errors.ts new file mode 100644 index 00000000..3304884a --- /dev/null +++ b/desktop/storage/engine/errors.ts @@ -0,0 +1,66 @@ +/** + * Storage error codes (desktop architecture §9). The engine rejects with a + * {@link StorageError}; the worker sends `{ code, retryable }` across, never + * a thrown object. This list must stay a superset of `DOC_STORE_ERROR_CODES` + * in `lib/platform/doc-store.ts` minus the web library's `quota_exceeded` + * (a unit test compares them): the engine cannot import `lib/`. + */ +export const STORAGE_ERROR_CODES = [ + "invalid_argument", + "forbidden", + "not_found", + "already_open", + "document_open", + "library_read_only", + "library_too_new", + "migration_failed", + "disk_full", + "io_error", + "corrupt", + "locked_elsewhere", + "cloud_folder", + "worker_restarting", + "library_unavailable", + "import_too_large", + "unknown_schema", +] as const; + +export type StorageErrorCode = (typeof STORAGE_ERROR_CODES)[number]; + +const RETRYABLE: ReadonlySet = new Set([ + "disk_full", + "io_error", + "worker_restarting", + "library_unavailable", +]); + +export class StorageError extends Error { + readonly code: StorageErrorCode; + readonly retryable: boolean; + + constructor(code: StorageErrorCode, message: string = code) { + super(message); + this.name = "StorageError"; + this.code = code; + this.retryable = RETRYABLE.has(code); + } +} + +/** What crosses the worker boundary in place of a thrown error. */ +export type StorageErrorEnvelope = { + code: StorageErrorCode; + retryable: boolean; + message: string; +}; + +export function toEnvelope(error: unknown): StorageErrorEnvelope { + const storage = + error instanceof StorageError + ? error + : new StorageError("io_error", String(error)); + return { + code: storage.code, + retryable: storage.retryable, + message: storage.message, + }; +} diff --git a/desktop/storage/engine/host.ts b/desktop/storage/engine/host.ts new file mode 100644 index 00000000..4269ad49 --- /dev/null +++ b/desktop/storage/engine/host.ts @@ -0,0 +1,32 @@ +/** + * What the engine needs from the operating system, injected so the engine + * imports no `node:*` or `bun:*` module (desktop architecture §3.3). The + * worker passes `createNodeHost()` (`desktop/storage/node-host.ts`); tests + * pass the same host with a fake clock, or a fake pid check. + */ +export interface StorageHost { + /** `process.platform`. */ + readonly platform: string; + /** This process's pid, written to `library.lock`. */ + readonly pid: number; + /** The home folder, used only to recognise cloud-synced folders. */ + readonly homeDir: string; + /** Environment variables (OneDrive sets `OneDrive*` on Windows). */ + readonly env: Readonly>; + /** Milliseconds since the epoch. */ + now(): number; + join(...parts: string[]): string; + /** Resolves symlinks; returns the path unchanged when it doesn't exist. */ + realpath(path: string): string; + mkdirp(path: string): void; + exists(path: string): boolean; + /** File names in `dir`, or `[]` when it doesn't exist. */ + list(dir: string): string[]; + remove(path: string): void; + rename(from: string, to: string): void; + readText(path: string): string | null; + /** Creates `path` with `text`; false when it already exists. */ + createExclusive(path: string, text: string): boolean; + /** Whether a process with this pid is running. */ + isPidAlive(pid: number): boolean; +} diff --git a/desktop/storage/engine/library-index.ts b/desktop/storage/engine/library-index.ts new file mode 100644 index 00000000..e0946e87 --- /dev/null +++ b/desktop/storage/engine/library-index.ts @@ -0,0 +1,398 @@ +/** + * The library's metadata: documents, folders and tags (architecture §5.4 + * `index.ts`; named `library-index.ts` so it isn't taken for a barrel). + * `IpcDocumentIndex` (P2.3's `DocumentIndex` port) sits on top of these + * operations; the row shapes here are the storage side and stay neutral. + * + * Every metadata write goes through this module (seam S10) and bumps + * `meta.rev` in the same transaction, so the change feed can say "changed + * since rev N". Inserting a document mints its `lineage` and `link_key` + * (seam S16): never lazily, so two synced devices can't mint different keys. + */ +import type { SqliteDriver, SqlRow } from "./driver"; +import { StorageError } from "./errors"; +import { bumpRev, randomHex32, readRev, scalar, transaction } from "./sql"; + +export type DocKind = "doc" | "template"; + +export type CreateDocInput = { + /** + * Minted (a UUID) when omitted. Main's IPC guard only lets UUIDs through; + * the engine accepts any {@link SAFE_ID}, which tests and imports use. + */ + id?: string; + /** `docs.schema`: `DOC_SCHEMA` for a new document, the file's on import. */ + schema?: number; + title?: string; + folderId?: string | null; + sortKey?: string; + kind?: DocKind; + /** Kept only by a backup restore (S8c); everything else gets a new one. */ + lineage?: string; + /** Kept only by a backup zip or `library.json` restore (S8c). */ + linkKey?: string; +}; + +export type CreatedDoc = { + id: string; + lineage: string; + linkKey: string; + rev: number; +}; + +export type SidebarFolder = { + id: string; + parentId: string | null; + name: string; + sortKey: string; + sort: string | null; + view: string | null; + group: string | null; +}; + +export type SidebarDoc = { + id: string; + title: string; + folderId: string | null; + sortKey: string; + pinKey: string | null; + pinned: boolean; + createdAt: number; + modifiedAt: number; + openedAt: number | null; + sizeBytes: number; + snippet: string; + kind: DocKind; + tagIds: string[]; +}; + +export type SidebarTag = { id: string; name: string; color: string | null }; + +/** + * One changed row, for the change feed (architecture §7 `index.changed`): + * a window patches that row instead of re-reading the whole library. + */ +export type IndexChange = { kind: "doc" | "folder" | "tag"; id: string }; + +/** Everything the files sidebar and lists paint from (P2.7a/b). */ +export type SidebarSnapshot = { + rev: number; + folders: SidebarFolder[]; + docs: SidebarDoc[]; + tags: SidebarTag[]; +}; + +/** Trash is emptied after this long (details spec §9). */ +export const TRASH_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; + +const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i; +/** Ids that are safe in SQL, file names and logs. */ +export const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9_-]{0,63}$/; +const HEX32 = /^[0-9a-f]{32}$/; +export const MAX_NAME_LENGTH = 200; + +function requireId(id: string, what: string) { + if (!SAFE_ID.test(id)) + throw new StorageError("invalid_argument", `bad ${what}`); +} + +function requireUuid(id: string, what: string) { + if (!UUID.test(id)) throw new StorageError("invalid_argument", `bad ${what}`); +} + +function cleanName(name: string): string { + return name.trim().slice(0, MAX_NAME_LENGTH); +} + +/** Wraps a metadata write: one transaction, `rev` bumped, new rev returned. */ +function indexWrite(db: SqliteDriver, body: () => void): number { + return transaction(db, () => { + body(); + return bumpRev(db); + }); +} + +function requireChanged(changes: number, what: string) { + if (changes === 0) throw new StorageError("not_found", `no such ${what}`); +} + +export function createDoc( + db: SqliteDriver, + input: CreateDocInput, + now: number, +): CreatedDoc { + const id = input.id ?? crypto.randomUUID(); + requireId(id, "document id"); + const lineage = input.lineage ?? crypto.randomUUID(); + requireUuid(lineage, "lineage"); + const schema = input.schema ?? 0; + if (!Number.isInteger(schema) || schema < 0) + throw new StorageError("invalid_argument", "bad schema"); + const linkKey = input.linkKey ?? randomHex32(); + if (!HEX32.test(linkKey)) + throw new StorageError("invalid_argument", "bad link key"); + if (input.folderId) requireId(input.folderId, "folder id"); + const kind = input.kind ?? "doc"; + if (kind !== "doc" && kind !== "template") + throw new StorageError("invalid_argument", "bad kind"); + const rev = indexWrite(db, () => { + db.prepare( + `INSERT INTO docs (id, title, folder_id, sort_key, created_at, modified_at, + lineage, link_key, kind, schema) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + ).run( + id, + cleanName(input.title ?? ""), + input.folderId ?? null, + input.sortKey ?? "", + now, + now, + lineage, + linkKey, + kind, + schema, + ); + }); + return { id, lineage, linkKey, rev }; +} + +export function createFolder( + db: SqliteDriver, + input: { + id?: string; + name: string; + parentId?: string | null; + sortKey?: string; + }, + now: number, +): { id: string; rev: number } { + const id = input.id ?? crypto.randomUUID(); + requireId(id, "folder id"); + if (input.parentId) requireId(input.parentId, "parent id"); + const rev = indexWrite(db, () => { + db.prepare( + `INSERT INTO folders (id, parent_id, name, sort_key, created_at, modified_at) + VALUES (?, ?, ?, ?, ?, ?)`, + ).run( + id, + input.parentId ?? null, + cleanName(input.name), + input.sortKey ?? "", + now, + now, + ); + }); + return { id, rev }; +} + +export function renameDoc(db: SqliteDriver, id: string, title: string): number { + return indexWrite(db, () => { + requireChanged( + db + .prepare("UPDATE docs SET title = ? WHERE id = ?") + .run(cleanName(title), id).changes, + "document", + ); + }); +} + +export function moveDoc( + db: SqliteDriver, + id: string, + folderId: string | null, + sortKey: string, +): number { + if (folderId) requireId(folderId, "folder id"); + return indexWrite(db, () => { + requireChanged( + db + .prepare("UPDATE docs SET folder_id = ?, sort_key = ? WHERE id = ?") + .run(folderId, sortKey, id).changes, + "document", + ); + }); +} + +export function markOpened(db: SqliteDriver, id: string, now: number): number { + return indexWrite(db, () => { + requireChanged( + db.prepare("UPDATE docs SET opened_at = ? WHERE id = ?").run(now, id) + .changes, + "document", + ); + }); +} + +export function trashDoc( + db: SqliteDriver, + id: string, + now: number, + group: string | null = null, +): number { + return indexWrite(db, () => { + requireChanged( + db + .prepare( + `UPDATE docs SET trashed_at = ?, trash_group = ?, trashed_from = folder_id + WHERE id = ? AND trashed_at IS NULL`, + ) + .run(now, group, id).changes, + "document", + ); + }); +} + +export function restoreDoc(db: SqliteDriver, id: string): number { + return indexWrite(db, () => { + requireChanged( + db + .prepare( + `UPDATE docs SET trashed_at = NULL, trash_group = NULL, + folder_id = CASE WHEN trashed_from IS NULL + OR EXISTS (SELECT 1 FROM folders WHERE id = docs.trashed_from AND trashed_at IS NULL) + THEN trashed_from ELSE NULL END, + trashed_from = NULL + WHERE id = ? AND trashed_at IS NOT NULL`, + ) + .run(id).changes, + "trashed document", + ); + }); +} + +/** + * Deletes a document and everything that cascades from it: rows, snapshot, + * tags, view and import source. The caller (`Library`) refuses a document + * that is open (`document_open`). + */ +export function purgeDoc(db: SqliteDriver, id: string): number { + return indexWrite(db, () => { + requireChanged( + db.prepare("DELETE FROM docs WHERE id = ?").run(id).changes, + "document", + ); + }); +} + +/** Ids of documents in Trash for longer than {@link TRASH_RETENTION_MS}. */ +export function expiredTrash(db: SqliteDriver, now: number): string[] { + return db + .prepare( + "SELECT id FROM docs WHERE trashed_at IS NOT NULL AND trashed_at <= ?", + ) + .all(now - TRASH_RETENTION_MS) + .map((row) => row.id as string); +} + +export function createTag( + db: SqliteDriver, + name: string, + color: string | null = null, +): { id: string; rev: number } { + const clean = cleanName(name); + if (!clean) throw new StorageError("invalid_argument", "empty tag name"); + const id = crypto.randomUUID(); + const rev = indexWrite(db, () => { + db.prepare("INSERT INTO tags (id, name, color) VALUES (?, ?, ?)").run( + id, + clean, + color, + ); + }); + return { id, rev }; +} + +export function setDocTags( + db: SqliteDriver, + docId: string, + tagIds: readonly string[], +): number { + for (const tagId of tagIds) requireId(tagId, "tag id"); + return indexWrite(db, () => { + if (scalar(db, "SELECT 1 FROM docs WHERE id = ?", docId) === undefined) + throw new StorageError("not_found", "no such document"); + db.prepare("DELETE FROM doc_tags WHERE doc_id = ?").run(docId); + const insert = db.prepare( + "INSERT INTO doc_tags (doc_id, tag_id) VALUES (?, ?)", + ); + for (const tagId of new Set(tagIds)) insert.run(docId, tagId); + }); +} + +const nullableString = (value: SqlRow[string]) => + typeof value === "string" ? value : null; +const nullableNumber = (value: SqlRow[string]) => + typeof value === "number" ? value : null; + +/** + * The files sidebar's whole picture in one read: every live (untrashed) + * folder and document, and the tags. Reads no BLOB column. P2.6's exit + * budget (architecture §10): populated within 600 ms on an M1 and 1.0 s on + * Windows with 1,000 documents. + */ +export function sidebarSnapshot(db: SqliteDriver): SidebarSnapshot { + const tagsByDoc = new Map(); + for (const row of db + .prepare( + `SELECT doc_tags.doc_id, doc_tags.tag_id FROM doc_tags + JOIN docs ON docs.id = doc_tags.doc_id WHERE docs.trashed_at IS NULL`, + ) + .all()) { + const docId = row.doc_id as string; + const list = tagsByDoc.get(docId); + if (list) list.push(row.tag_id as string); + else tagsByDoc.set(docId, [row.tag_id as string]); + } + const docs = db + .prepare( + `SELECT id, title, folder_id, sort_key, pin_key, pinned, created_at, + modified_at, opened_at, size_bytes, snippet, kind + FROM docs WHERE trashed_at IS NULL`, + ) + .all() + .map( + (row): SidebarDoc => ({ + id: row.id as string, + title: row.title as string, + folderId: nullableString(row.folder_id), + sortKey: row.sort_key as string, + pinKey: nullableString(row.pin_key), + pinned: row.pinned === 1, + createdAt: row.created_at as number, + modifiedAt: row.modified_at as number, + openedAt: nullableNumber(row.opened_at), + sizeBytes: row.size_bytes as number, + snippet: row.snippet as string, + kind: row.kind as DocKind, + tagIds: tagsByDoc.get(row.id as string) ?? [], + }), + ); + const folders = db + .prepare( + `SELECT id, parent_id, name, sort_key, sort, view, "group" + FROM folders WHERE trashed_at IS NULL`, + ) + .all() + .map( + (row): SidebarFolder => ({ + id: row.id as string, + parentId: nullableString(row.parent_id), + name: row.name as string, + sortKey: row.sort_key as string, + sort: nullableString(row.sort), + view: nullableString(row.view), + group: nullableString(row.group), + }), + ); + const tags = db + .prepare("SELECT id, name, color FROM tags") + .all() + .map( + (row): SidebarTag => ({ + id: row.id as string, + name: row.name as string, + color: nullableString(row.color), + }), + ); + return { rev: readRev(db), folders, docs, tags }; +} diff --git a/desktop/storage/engine/library.ts b/desktop/storage/engine/library.ts new file mode 100644 index 00000000..e36025d9 --- /dev/null +++ b/desktop/storage/engine/library.ts @@ -0,0 +1,578 @@ +/** + * One open library: the storage worker's whole state (architecture §5.4). + * The worker owns exactly one `Library`; tests and the dev storage host + * build one directly over `bun:sqlite`. + * + * **Open order** (architecture §5.4 `open.ts`, D-A6). Nothing that writes + * runs before the `format_version` check, so a library made by a newer app + * is never touched (soak T6): + * 1. refuse a cloud-synced folder (`cloud_folder`); + * 2. take `library.lock` (`locked_elsewhere`); + * 3. pragmas: WAL, `synchronous=FULL`, foreign keys, and on macOS + * `fullfsync` and `checkpoint_fullfsync`; + * 4. create the schema if the file is new (lineage and `library_id` from day + * one); + * 5. `format_version` newer than the app → reopen read-only, + * `library_too_new`; + * 6. `unclean_exit` still set → `quick_check`; a failure is read-only, + * `corrupt`; + * 7. older → pre-migration copy, then the format steps (`migration_failed` + * is read-only); + * 8. set `unclean_exit`; {@link Library.shutdown} clears it. + * + * **Registry** (architecture §5.2, §6.2): a document is open in at most one + * window (D-A3). `load` claims it for an owner (a window id) and rejects + * another owner with `already_open`; appends and compactions from any + * other owner are `forbidden` (and {@link Library.claim} re-registers holds + * after a worker restart); main's own content writes (purge, and P2.8's + * import-into and restore) refuse an open document with `document_open`. + */ +import { cloudFolderOf } from "./cloud-folder"; +import { + type AppendInput, + append, + compact, + type LoadedContent, + load, + pendingRows, + stateOf, +} from "./content"; +import { + DAILY_DIR, + dailyCopyName, + isDailyCopyDue, + listDailyCopies, + localDate, + takeDailyCopy, +} from "./daily-copy"; +import type { OpenSqlite, SqliteDriver } from "./driver"; +import { storageErrorFrom } from "./driver"; +import { StorageError } from "./errors"; +import type { StorageHost } from "./host"; +import * as index from "./library-index"; +import { acquireLock, releaseLock } from "./lock"; +import { + FORMAT_STEPS, + type FormatStep, + readFormatVersion, + runFormatSteps, +} from "./migrations"; +import { FORMAT_VERSION, SCHEMA_V1, YJS_ENCODING } from "./schema"; +import { readMeta, readRev, scalar, transaction, writeMeta } from "./sql"; + +export const LIBRARY_FILE = "library.sqlite"; +/** Idle jobs (daily copy, Trash purge) wait for this long without a write. */ +export const IDLE_MS = 30_000; + +export type ReadOnlyReason = "library_too_new" | "migration_failed" | "corrupt"; + +export type LibraryStatus = { + readOnly: ReadOnlyReason | null; + formatVersion: number; + /** The previous session did not shut down cleanly. */ + uncleanExit: boolean; + /** `quick_check`'s verdict after an unclean exit; null when it didn't run. */ + quickCheck: "ok" | string | null; + libraryId: string; +}; + +/** Protocol lines for the durability soak (written to fd 1 by the worker). */ +export type Trace = (line: string) => void; + +export type OpenLibraryOptions = { + dir: string; + host: StorageHost; + openSqlite: OpenSqlite; + /** Default on; the T3 control turns it off. macOS only. */ + fullfsync?: boolean; + /** Default `FULL`; the soak's atomicity control uses `OFF`. */ + synchronous?: "FULL" | "OFF"; + /** For tests: the format this "app" writes, and the steps that reach it. */ + formatVersion?: number; + formatSteps?: readonly FormatStep[]; + trace?: Trace; +}; + +function applyPragmas( + db: SqliteDriver, + host: StorageHost, + options: OpenLibraryOptions, +) { + db.exec("PRAGMA journal_mode = WAL"); + db.exec(`PRAGMA synchronous = ${options.synchronous ?? "FULL"}`); + db.exec("PRAGMA foreign_keys = ON"); + if (host.platform === "darwin" && options.fullfsync !== false) { + db.exec("PRAGMA fullfsync = 1"); + db.exec("PRAGMA checkpoint_fullfsync = 1"); + } +} + +function hasTable(db: SqliteDriver, name: string): boolean { + return ( + scalar( + db, + "SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = ?", + name, + ) !== undefined + ); +} + +function createLibrary(db: SqliteDriver, now: number) { + const tables = Number( + scalar(db, "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table'"), + ); + if (tables > 0) + throw new StorageError("corrupt", "not a TinyDocy library (no meta)"); + transaction(db, () => { + db.exec(SCHEMA_V1); + writeMeta(db, "format_version", String(FORMAT_VERSION)); + writeMeta(db, "yjs_encoding", YJS_ENCODING); + writeMeta(db, "library_id", crypto.randomUUID()); + writeMeta(db, "created_at", String(now)); + writeMeta(db, "rev", "0"); + writeMeta(db, "unclean_exit", "0"); + }); +} + +function quickCheck(db: SqliteDriver): string { + const rows = db.prepare("PRAGMA quick_check").all(); + const messages = rows.map((row) => String(Object.values(row)[0])); + return messages.length === 1 && messages[0] === "ok" + ? "ok" + : messages.slice(0, 5).join("; "); +} + +export class Library { + readonly dir: string; + readonly path: string; + readonly status: LibraryStatus; + private db: SqliteDriver | null; + private readonly host: StorageHost; + private readonly openSqlite: OpenSqlite; + private readonly lockPath: string; + private readonly holders = new Map(); + private readonly trace: Trace | undefined; + private lastWriteAt: number; + private changes = new Map(); + private lastDailyCopyError: string | null = null; + + private constructor(fields: { + dir: string; + path: string; + status: LibraryStatus; + db: SqliteDriver; + host: StorageHost; + openSqlite: OpenSqlite; + lockPath: string; + trace: Trace | undefined; + }) { + this.dir = fields.dir; + this.path = fields.path; + this.status = fields.status; + this.db = fields.db; + this.host = fields.host; + this.openSqlite = fields.openSqlite; + this.lockPath = fields.lockPath; + this.trace = fields.trace; + this.lastWriteAt = fields.host.now(); + } + + static open(options: OpenLibraryOptions): Library { + const { dir, host, openSqlite } = options; + const target = options.formatVersion ?? FORMAT_VERSION; + host.mkdirp(dir); + const cloud = cloudFolderOf(dir, host); + if (cloud) throw new StorageError("cloud_folder", cloud); + const lockPath = acquireLock(host, dir); + const path = host.join(dir, LIBRARY_FILE); + let db: SqliteDriver | null = null; + try { + db = openSqlite(path, { readOnly: false }); + applyPragmas(db, host, options); + if (!hasTable(db, "meta")) createLibrary(db, host.now()); + const version = readFormatVersion(db); + const libraryId = readMeta(db, "library_id") ?? ""; + const status: LibraryStatus = { + readOnly: null, + formatVersion: version, + uncleanExit: false, + quickCheck: null, + libraryId, + }; + if (version > target) { + db.close(); + db = openSqlite(path, { readOnly: true }); + status.readOnly = "library_too_new"; + } else { + status.uncleanExit = readMeta(db, "unclean_exit") !== "0"; + if (status.uncleanExit) { + status.quickCheck = quickCheck(db); + if (status.quickCheck !== "ok") status.readOnly = "corrupt"; + } + if (!status.readOnly && version < target) { + try { + status.formatVersion = runFormatSteps( + db, + host, + dir, + target, + options.formatSteps ?? FORMAT_STEPS, + ); + } catch (error) { + if ( + !(error instanceof StorageError) || + error.code !== "migration_failed" + ) + throw error; + status.readOnly = "migration_failed"; + status.formatVersion = readFormatVersion(db); + } + } + if (!status.readOnly) writeMeta(db, "unclean_exit", "1"); + } + return new Library({ + dir, + path, + status, + db, + host, + openSqlite, + lockPath, + trace: options.trace, + }); + } catch (error) { + try { + db?.close(); + } catch { + // Already failing; the original error matters. + } + releaseLock(host, lockPath); + throw storageErrorFrom(error); + } + } + + private get conn(): SqliteDriver { + if (!this.db) + throw new StorageError("library_unavailable", "library is closed"); + return this.db; + } + + private assertWritable() { + if (this.status.readOnly) + throw new StorageError( + this.status.readOnly === "library_too_new" + ? "library_too_new" + : "library_read_only", + this.status.readOnly, + ); + } + + /** Content writes from the renderer: refused while another owner holds the doc. */ + /** + * Content writes from the renderer: only the window holding the document + * may write it (architecture §7: "the registry must hold docId open for + * this window"). Main checks this first; the worker checks again. + */ + private assertWriter(docId: string, owner: string) { + if (this.holders.get(docId) !== owner) + throw new StorageError( + "forbidden", + "this window has not opened the document", + ); + } + + /** Main's own content writes: refused while any window holds the doc. */ + private assertClosed(docId: string) { + if (this.holders.has(docId)) + throw new StorageError("document_open", "document is open"); + } + + private wrote() { + this.lastWriteAt = this.host.now(); + } + + private changed(kind: index.IndexChange["kind"], id: string) { + this.changes.set(`${kind}:${id}`, { kind, id }); + } + + /** + * The rows changed since the last call, with the current `rev`, or null + * when nothing changed. The worker drains this after every request and + * idle pass and sends `index_changed` (architecture §7, §8.1). + */ + takeIndexChanges(): { rev: number; changes: index.IndexChange[] } | null { + if (this.changes.size === 0 || !this.db) return null; + const changes = [...this.changes.values()]; + this.changes.clear(); + return { rev: readRev(this.conn), changes }; + } + + // ── Content (DocStore) ──────────────────────────────────────────────── + + load(docId: string, owner: string): LoadedContent { + const holder = this.holders.get(docId); + if (holder !== undefined && holder !== owner) + throw new StorageError("already_open", "document is open elsewhere"); + const content = load(this.conn, docId); + this.holders.set(docId, owner); + if (!this.status.readOnly) { + try { + index.markOpened(this.conn, docId, this.host.now()); + this.changed("doc", docId); + } catch { + // "Opened" sorting is a nicety; opening must not fail on it. + } + } + return content; + } + + append(docId: string, owner: string, input: AppendInput): number { + this.assertWritable(); + this.assertWriter(docId, owner); + this.trace?.(`BATCH_BEGIN ${docId}`); + const seq = append(this.conn, docId, input, this.host.now()); + this.trace?.(`ACK ${seq}`); + this.wrote(); + this.changed("doc", docId); + return seq; + } + + compact( + docId: string, + owner: string, + state: Uint8Array, + upToSeq: number, + ): { foreign: null } { + this.assertWritable(); + this.assertWriter(docId, owner); + this.trace?.(`COMPACT_BEGIN ${upToSeq} ${state.byteLength}`); + compact(this.conn, docId, state, upToSeq); + this.trace?.(`COMPACT_ACK ${upToSeq}`); + this.wrote(); + return { foreign: null }; + } + + /** + * Re-registers a hold without loading: after a worker restart, main + * replays its registry so the windows' resent batches are accepted. + */ + claim(docId: string, owner: string): void { + const holder = this.holders.get(docId); + if (holder !== undefined && holder !== owner) + throw new StorageError("already_open", "document is open elsewhere"); + if ( + scalar(this.conn, "SELECT 1 FROM docs WHERE id = ?", docId) === undefined + ) + throw new StorageError("not_found", "no such document"); + this.holders.set(docId, owner); + } + + close(docId: string, owner: string): void { + if (this.holders.get(docId) === owner) this.holders.delete(docId); + } + + /** A window closed or its renderer died: every hold it had is released. */ + releaseOwner(owner: string): void { + for (const [docId, holder] of this.holders) + if (holder === owner) this.holders.delete(docId); + } + + isOpen(docId: string): boolean { + return this.holders.has(docId); + } + + /** The whole state of a closed document, for export (P2.8). */ + stateOf(docId: string): Uint8Array { + return stateOf(load(this.conn, docId)); + } + + // ── Index ───────────────────────────────────────────────────────────── + + sidebar(): index.SidebarSnapshot { + return index.sidebarSnapshot(this.conn); + } + + createDoc(input: index.CreateDocInput = {}): index.CreatedDoc { + this.assertWritable(); + const created = index.createDoc(this.conn, input, this.host.now()); + this.wrote(); + this.changed("doc", created.id); + return created; + } + + createFolder(input: { name: string; parentId?: string | null }) { + this.assertWritable(); + const created = index.createFolder(this.conn, input, this.host.now()); + this.wrote(); + this.changed("folder", created.id); + return created; + } + + renameDoc(id: string, title: string): number { + this.assertWritable(); + const rev = index.renameDoc(this.conn, id, title); + this.changed("doc", id); + return rev; + } + + moveDoc(id: string, folderId: string | null, sortKey: string): number { + this.assertWritable(); + const rev = index.moveDoc(this.conn, id, folderId, sortKey); + this.changed("doc", id); + return rev; + } + + trashDoc(id: string, group: string | null = null): number { + this.assertWritable(); + const rev = index.trashDoc(this.conn, id, this.host.now(), group); + this.changed("doc", id); + return rev; + } + + restoreDoc(id: string): number { + this.assertWritable(); + const rev = index.restoreDoc(this.conn, id); + this.changed("doc", id); + return rev; + } + + purgeDoc(id: string): number { + this.assertWritable(); + this.assertClosed(id); + const rev = index.purgeDoc(this.conn, id); + this.wrote(); + this.changed("doc", id); + return rev; + } + + createTag(name: string, color: string | null = null) { + this.assertWritable(); + const created = index.createTag(this.conn, name, color); + this.changed("tag", created.id); + return created; + } + + setDocTags(docId: string, tagIds: readonly string[]): number { + this.assertWritable(); + const rev = index.setDocTags(this.conn, docId, tagIds); + this.changed("doc", docId); + return rev; + } + + // ── Restore points ──────────────────────────────────────────────────── + + listDailyCopies(): string[] { + return listDailyCopies(this.host, this.dir).map((file) => + file.slice("library-".length, -".sqlite".length), + ); + } + + /** One document's state in the daily copy of `date` (`YYYY-MM-DD`). */ + readDailyCopy(date: string, docId: string): Uint8Array { + if (!/^\d{4}-\d{2}-\d{2}$/.test(date)) + throw new StorageError("invalid_argument", "bad date"); + const file = this.host.join(this.dir, DAILY_DIR, dailyCopyName(date)); + if (!this.host.exists(file)) + throw new StorageError("not_found", "no copy for that day"); + const copy = this.openSqlite(file, { readOnly: true }); + try { + return stateOf(load(copy, docId)); + } finally { + copy.close(); + } + } + + // ── Jobs and lifecycle ──────────────────────────────────────────────── + + /** + * Runs the idle jobs when nothing was written for {@link IDLE_MS}: today's + * daily copy, then Trash older than 30 days (skipping open documents). + * Returns what ran, for the worker's diagnostics. A failed daily copy is + * kept for `stats()` (app health `backups_failing`) and retried on the + * next idle pass; it never stops the purge. + */ + runIdleJobs(): { + dailyCopy: string | null; + dailyCopyMs: number | null; + purged: number; + } { + const now = this.host.now(); + const result = { + dailyCopy: null as string | null, + dailyCopyMs: null as number | null, + purged: 0, + }; + if (this.status.readOnly || now - this.lastWriteAt < IDLE_MS) return result; + if (isDailyCopyDue(this.conn, now)) { + const started = performance.now(); + try { + result.dailyCopy = takeDailyCopy(this.conn, this.host, this.dir, now); + this.lastDailyCopyError = null; + } catch (error) { + this.lastDailyCopyError = storageErrorFrom(error).code; + } + result.dailyCopyMs = performance.now() - started; + } + for (const id of index.expiredTrash(this.conn, now)) { + if (this.holders.has(id)) continue; + index.purgeDoc(this.conn, id); + this.changed("doc", id); + result.purged += 1; + } + return result; + } + + /** Forces today's daily copy now (diagnostics, tests). */ + takeDailyCopyNow(): string | null { + this.assertWritable(); + return takeDailyCopy(this.conn, this.host, this.dir, this.host.now()); + } + + stats() { + const db = this.conn; + return { + formatVersion: this.status.formatVersion, + docs: Number(scalar(db, "SELECT COUNT(*) FROM docs")), + pendingRows: pendingRows(db), + bytes: Number( + scalar( + db, + "SELECT page_count * page_size FROM pragma_page_count(), pragma_page_size()", + ), + ), + lastDailyCopy: readMeta(db, "last_daily_copy"), + lastDailyCopyError: this.lastDailyCopyError, + today: localDate(this.host.now()), + // Read back, as set (D-A6; S2 confirmed them on all three OSes). + pragmas: { + journalMode: scalar(db, "PRAGMA journal_mode"), + synchronous: scalar(db, "PRAGMA synchronous"), + fullfsync: scalar(db, "PRAGMA fullfsync"), + checkpointFullfsync: scalar(db, "PRAGMA checkpoint_fullfsync"), + foreignKeys: scalar(db, "PRAGMA foreign_keys"), + }, + sqliteVersion: scalar(db, "SELECT sqlite_version()"), + }; + } + + /** + * The quit flush's last step (architecture §8.2): every request queued + * before this one has already committed (the worker handles one message + * at a time), so checkpoint the WAL into the main file, clear + * `unclean_exit`, close and release the lock. Safe to call twice. + */ + shutdown(): void { + const db = this.db; + if (!db) return; + try { + if (!this.status.readOnly) { + writeMeta(db, "unclean_exit", "0"); + db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); + } + } finally { + this.db = null; + db.close(); + releaseLock(this.host, this.lockPath); + } + } +} diff --git a/desktop/storage/engine/lock.ts b/desktop/storage/engine/lock.ts new file mode 100644 index 00000000..b67d704e --- /dev/null +++ b/desktop/storage/engine/lock.ts @@ -0,0 +1,34 @@ +/** + * `library.lock` (architecture §5.4 `open.ts`): one process opens a library + * at a time. The file holds the owner's pid; a lock whose pid is no longer + * running is stale (a crash or a SIGKILL) and is taken over. Electron's + * single-instance lock covers the usual case; this one also covers a second + * build (internal and release side by side) pointed at the same folder. + */ +import { StorageError } from "./errors"; +import type { StorageHost } from "./host"; + +export const LOCK_FILE = "library.lock"; + +export function acquireLock(host: StorageHost, dir: string): string { + const path = host.join(dir, LOCK_FILE); + for (let attempt = 0; attempt < 3; attempt += 1) { + if (host.createExclusive(path, String(host.pid))) return path; + const holder = Number.parseInt(host.readText(path) ?? "", 10); + if ( + Number.isInteger(holder) && + holder > 0 && + holder !== host.pid && + host.isPidAlive(holder) + ) { + throw new StorageError("locked_elsewhere", `library held by ${holder}`); + } + // Stale (dead pid, unreadable, or our own from a restarted worker). + host.remove(path); + } + throw new StorageError("locked_elsewhere", "could not take library.lock"); +} + +export function releaseLock(host: StorageHost, path: string) { + if (host.readText(path) === String(host.pid)) host.remove(path); +} diff --git a/desktop/storage/engine/migrations.ts b/desktop/storage/engine/migrations.ts new file mode 100644 index 00000000..efbac6a6 --- /dev/null +++ b/desktop/storage/engine/migrations.ts @@ -0,0 +1,100 @@ +/** + * The `format_version` runner (architecture §5.4 `migrations.ts`, D-A6). + * + * - **Newer than the app:** the library opens read-only with + * `library_too_new`, and nothing is written (checked before any other + * write; see `library.ts`). + * - **Older:** `VACUUM INTO library.pre-v-.sqlite` first (the last 2 + * are kept), then each step in its own transaction, which also bumps + * `format_version`. A failed step leaves the library read-only with + * `migration_failed`; a kill between steps resumes at the next one. + * + * Format 1 is the first, so {@link FORMAT_STEPS} is empty. Tests pass their + * own steps to exercise the runner (soak T5). + */ +import type { SqliteDriver } from "./driver"; +import { StorageError } from "./errors"; +import type { StorageHost } from "./host"; +import { readMeta, transaction, writeMeta } from "./sql"; + +export type FormatStep = { + /** The `format_version` this step produces. */ + to: number; + up(db: SqliteDriver): void; +}; + +/** Steps from format 1 to the current `FORMAT_VERSION`, in order. */ +export const FORMAT_STEPS: readonly FormatStep[] = []; + +export const PRE_MIGRATION_KEEP = 2; +const PRE_MIGRATION = /^library\.pre-v\d+-[0-9TZ]+\.sqlite$/; + +/** `20261117T031700Z`: sortable, and valid in a file name on every OS. */ +export function utcStamp(ms: number): string { + return new Date(ms).toISOString().replace(/[-:]/g, "").replace(/\.\d+/, ""); +} + +export function readFormatVersion(db: SqliteDriver): number { + const value = Number.parseInt(readMeta(db, "format_version") ?? "", 10); + if (!Number.isInteger(value) || value < 1) + throw new StorageError("corrupt", "meta.format_version is missing"); + return value; +} + +/** Takes the pre-migration copy and prunes older ones. */ +export function backUpBeforeMigration( + db: SqliteDriver, + host: StorageHost, + dir: string, + from: number, +): string { + const name = `library.pre-v${from}-${utcStamp(host.now())}.sqlite`; + const target = host.join(dir, name); + const partial = `${target}.partial`; + if (host.exists(partial)) host.remove(partial); + db.prepare("VACUUM INTO ?").run(partial); + host.rename(partial, target); + const copies = host + .list(dir) + .filter((file) => PRE_MIGRATION.test(file)) + .sort(); + for (const old of copies.slice(0, -PRE_MIGRATION_KEEP)) + host.remove(host.join(dir, old)); + return target; +} + +/** + * Migrates `db` from its `format_version` to `target`. Returns the version + * reached; throws `migration_failed` when a step fails (that step rolled + * back, earlier ones stay). + */ +export function runFormatSteps( + db: SqliteDriver, + host: StorageHost, + dir: string, + target: number, + steps: readonly FormatStep[], +): number { + let version = readFormatVersion(db); + if (version >= target) return version; + backUpBeforeMigration(db, host, dir, version); + for (const step of steps) { + if (step.to <= version) continue; + if (step.to > target) break; + try { + transaction(db, () => { + step.up(db); + writeMeta(db, "format_version", String(step.to)); + }); + } catch (error) { + throw new StorageError( + "migration_failed", + `step to v${step.to}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + version = step.to; + } + if (version !== target) + throw new StorageError("migration_failed", `no step reaches v${target}`); + return version; +} diff --git a/desktop/storage/engine/schema.ts b/desktop/storage/engine/schema.ts new file mode 100644 index 00000000..ce6a6636 --- /dev/null +++ b/desktop/storage/engine/schema.ts @@ -0,0 +1,120 @@ +/** + * The library schema, `format_version` 1 (desktop architecture §5.3). + * + * Everything the Local release needs is created here, so no later migration + * is needed for it: `tags`, `doc_tags` and `doc_view` (D7, D10), the details + * spec §6.3 list columns, and the v7/v8 seams `docs.link_key` (NOT NULL, + * minted at insert), `docs.kind` and `import_sources`. `blobs` and + * `doc_blobs` arrive with P2.6b, and the Live release adds `remotes` and + * `versions`; each comes with a `format_version` step. + * + * `updates.seq` is `AUTOINCREMENT`, so a seq is never reused after + * compaction deletes the newest rows: a reused seq at or below a snapshot's + * `upto_seq` would be skipped on load. + */ +export const FORMAT_VERSION = 1; + +/** The Yjs update encoding of every stored BLOB (`meta.yjs_encoding`). */ +export const YJS_ENCODING = "v1"; + +export const SCHEMA_V1 = ` +CREATE TABLE meta ( + key TEXT PRIMARY KEY, + value TEXT NOT NULL +) STRICT; + +CREATE TABLE folders ( + id TEXT PRIMARY KEY, + parent_id TEXT, + name TEXT NOT NULL, + sort_key TEXT NOT NULL DEFAULT '', + created_at INTEGER NOT NULL, + modified_at INTEGER NOT NULL, + trashed_at INTEGER, + sort TEXT, + view TEXT, + "group" TEXT +) STRICT; + +CREATE TABLE docs ( + id TEXT PRIMARY KEY, + title TEXT NOT NULL DEFAULT '', + folder_id TEXT, + sort_key TEXT NOT NULL DEFAULT '', + pin_key TEXT, + pinned INTEGER NOT NULL DEFAULT 0, + created_at INTEGER NOT NULL, + modified_at INTEGER NOT NULL, + opened_at INTEGER, + trashed_at INTEGER, + trash_group TEXT, + trashed_from TEXT, + schema INTEGER NOT NULL DEFAULT 0, + lineage TEXT NOT NULL, + size_bytes INTEGER NOT NULL DEFAULT 0, + snippet TEXT NOT NULL DEFAULT '', + mirrored_at INTEGER, + link_key TEXT NOT NULL UNIQUE, + kind TEXT NOT NULL DEFAULT 'doc' CHECK (kind IN ('doc', 'template')) +) STRICT; + +CREATE TABLE import_sources ( + doc_id TEXT PRIMARY KEY REFERENCES docs(id) ON DELETE CASCADE, + relative_path TEXT, + source_app TEXT, + source_hash TEXT, + imported_at INTEGER +) STRICT; + +CREATE TABLE snapshots ( + doc_id TEXT PRIMARY KEY REFERENCES docs(id) ON DELETE CASCADE, + data BLOB NOT NULL, + upto_seq INTEGER NOT NULL +) STRICT; + +CREATE TABLE updates ( + seq INTEGER PRIMARY KEY AUTOINCREMENT, + doc_id TEXT NOT NULL REFERENCES docs(id) ON DELETE CASCADE, + data BLOB NOT NULL, + origin TEXT NOT NULL, + created_at INTEGER NOT NULL +) STRICT; + +CREATE TABLE tags ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL UNIQUE COLLATE NOCASE, + color TEXT +) STRICT; + +CREATE TABLE doc_tags ( + doc_id TEXT NOT NULL REFERENCES docs(id) ON DELETE CASCADE, + tag_id TEXT NOT NULL REFERENCES tags(id) ON DELETE CASCADE, + PRIMARY KEY (doc_id, tag_id) +) STRICT, WITHOUT ROWID; + +CREATE TABLE doc_view ( + doc_id TEXT PRIMARY KEY REFERENCES docs(id) ON DELETE CASCADE, + width TEXT CHECK (width IS NULL OR width IN ('narrow', 'standard', 'wide', 'full')), + place TEXT, + remote_images INTEGER NOT NULL DEFAULT 0 +) STRICT; + +CREATE INDEX updates_doc_seq ON updates(doc_id, seq); +CREATE INDEX docs_folder_list ON docs(folder_id, trashed_at, opened_at); +CREATE INDEX folders_parent ON folders(parent_id); +CREATE INDEX doc_tags_tag ON doc_tags(tag_id); +CREATE INDEX import_sources_hash ON import_sources(source_hash); +`; + +/** Every table format 1 creates; the schema test checks the file against it. */ +export const TABLES_V1 = [ + "doc_tags", + "doc_view", + "docs", + "folders", + "import_sources", + "meta", + "snapshots", + "tags", + "updates", +] as const; diff --git a/desktop/storage/engine/sql.ts b/desktop/storage/engine/sql.ts new file mode 100644 index 00000000..c0b334a5 --- /dev/null +++ b/desktop/storage/engine/sql.ts @@ -0,0 +1,66 @@ +import type { SqliteDriver, SqlRow } from "./driver"; +import { storageErrorFrom } from "./driver"; + +/** + * Runs `body` in one `BEGIN IMMEDIATE … COMMIT`: one transaction, one fsync. + * Any throw rolls back, and a driver failure becomes a `StorageError`. + */ +export function transaction(db: SqliteDriver, body: () => T): T { + try { + db.exec("BEGIN IMMEDIATE"); + } catch (error) { + throw storageErrorFrom(error); + } + try { + const result = body(); + db.exec("COMMIT"); + return result; + } catch (error) { + try { + db.exec("ROLLBACK"); + } catch { + // The failed statement may already have rolled the transaction back. + } + throw storageErrorFrom(error); + } +} + +/** Reads one column of one row; undefined when there is no row. */ +export function scalar( + db: SqliteDriver, + sql: string, + ...params: (string | number | null)[] +): SqlRow[string] | undefined { + const row = db.prepare(sql).get(...params); + return row ? Object.values(row)[0] : undefined; +} + +export function readMeta(db: SqliteDriver, key: string): string | null { + const value = scalar(db, "SELECT value FROM meta WHERE key = ?", key); + return typeof value === "string" ? value : null; +} + +export function writeMeta(db: SqliteDriver, key: string, value: string) { + db.prepare( + "INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT (key) DO UPDATE SET value = excluded.value", + ).run(key, value); +} + +/** `meta.rev`: moves on every change the files sidebar shows. */ +export function readRev(db: SqliteDriver): number { + return Number.parseInt(readMeta(db, "rev") ?? "0", 10); +} + +/** Bumps `meta.rev`; call inside the write's transaction. */ +export function bumpRev(db: SqliteDriver): number { + const rev = readRev(db) + 1; + writeMeta(db, "rev", String(rev)); + return rev; +} + +/** 32 lowercase hex characters from the platform CSPRNG (`link_key`). */ +export function randomHex32(): string { + const bytes = new Uint8Array(16); + crypto.getRandomValues(bytes); + return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join(""); +} diff --git a/desktop/storage/node-host.ts b/desktop/storage/node-host.ts new file mode 100644 index 00000000..dd38fb3c --- /dev/null +++ b/desktop/storage/node-host.ts @@ -0,0 +1,88 @@ +/** + * The `StorageHost` the worker runs with: `node:fs`, `node:os` and + * `node:path`. Bun implements these modules too, so unit tests use the same + * host (with `now` or `isPidAlive` overridden where a test needs to). + */ +import { + existsSync, + mkdirSync, + readdirSync, + readFileSync, + realpathSync, + renameSync, + rmSync, + writeFileSync, +} from "node:fs"; +import { homedir } from "node:os"; +import { join } from "node:path"; + +import type { StorageHost } from "./engine/host"; + +function isErrno(error: unknown, code: string): boolean { + return ( + typeof error === "object" && + error !== null && + (error as { code?: unknown }).code === code + ); +} + +export function createNodeHost( + overrides: Partial = {}, +): StorageHost { + return { + platform: process.platform, + pid: process.pid, + homeDir: homedir(), + env: process.env, + now: () => Date.now(), + join: (...parts) => join(...parts), + realpath(path) { + try { + return realpathSync.native(path); + } catch { + return path; + } + }, + mkdirp: (path) => { + mkdirSync(path, { recursive: true }); + }, + exists: (path) => existsSync(path), + list(dir) { + try { + return readdirSync(dir); + } catch (error) { + if (isErrno(error, "ENOENT")) return []; + throw error; + } + }, + remove: (path) => rmSync(path, { force: true }), + rename: (from, to) => renameSync(from, to), + readText(path) { + try { + return readFileSync(path, "utf8"); + } catch (error) { + if (isErrno(error, "ENOENT")) return null; + throw error; + } + }, + createExclusive(path, text) { + try { + writeFileSync(path, text, { flag: "wx" }); + return true; + } catch (error) { + if (isErrno(error, "EEXIST")) return false; + throw error; + } + }, + isPidAlive(pid) { + try { + process.kill(pid, 0); + return true; + } catch (error) { + // EPERM: it exists but belongs to someone else. + return isErrno(error, "EPERM"); + } + }, + ...overrides, + }; +} diff --git a/desktop/storage/protocol.ts b/desktop/storage/protocol.ts new file mode 100644 index 00000000..1d238cd3 --- /dev/null +++ b/desktop/storage/protocol.ts @@ -0,0 +1,208 @@ +/** + * Messages between main and the storage worker (desktop architecture §2, + * §7). Main has already validated every request at the IPC boundary; the + * worker trusts main. Nothing is thrown across: every reply is a `Result`. + * + * Bytes travel as `Uint8Array`s whose buffers are transferred, so a + * document crosses each boundary with one copy. + * + * No `node:*` imports: the dev storage host and unit tests dispatch to a + * `Library` in-process with {@link dispatch}. + */ +import type { AppendInput, LoadedContent } from "./engine/content"; +import { type StorageErrorEnvelope, toEnvelope } from "./engine/errors"; +import type { Library, LibraryStatus } from "./engine/library"; +import type { + CreateDocInput, + CreatedDoc, + IndexChange, + SidebarSnapshot, +} from "./engine/library-index"; + +/** Every request: `args` in, `result` out. */ +export type StorageOps = { + load: { args: { docId: string; owner: string }; result: LoadedContent }; + append: { + args: { docId: string; owner: string; record: AppendInput }; + result: number; + }; + compact: { + args: { docId: string; owner: string; state: Uint8Array; upToSeq: number }; + result: { foreign: null }; + }; + /** Main replays its registry after a worker restart. */ + claim: { args: { docId: string; owner: string }; result: null }; + close: { args: { docId: string; owner: string }; result: null }; + releaseOwner: { args: { owner: string }; result: null }; + /** Resolves after every request sent before it: all durable. */ + flush: { args: Record; result: null }; + stateOf: { args: { docId: string }; result: Uint8Array }; + sidebar: { args: Record; result: SidebarSnapshot }; + createDoc: { args: CreateDocInput; result: CreatedDoc }; + createFolder: { + args: { name: string; parentId?: string | null }; + result: { id: string; rev: number }; + }; + renameDoc: { args: { id: string; title: string }; result: number }; + moveDoc: { + args: { id: string; folderId: string | null; sortKey: string }; + result: number; + }; + trashDoc: { args: { id: string; group?: string | null }; result: number }; + restoreDoc: { args: { id: string }; result: number }; + purgeDoc: { args: { id: string }; result: number }; + createTag: { + args: { name: string; color?: string | null }; + result: { id: string; rev: number }; + }; + setDocTags: { args: { docId: string; tagIds: string[] }; result: number }; + listDailyCopies: { args: Record; result: string[] }; + readDailyCopy: { args: { date: string; docId: string }; result: Uint8Array }; + runIdleJobs: { + args: Record; + result: ReturnType; + }; + stats: { args: Record; result: ReturnType }; + /** The quit flush's last step: checkpoint, clear `unclean_exit`, close. */ + shutdown: { args: Record; result: null }; +}; + +export type StorageOp = keyof StorageOps; + +export type Result = + | { ok: true; value: T } + | { ok: false; error: StorageErrorEnvelope }; + +/** main → worker */ +export type StorageRequest = { + [K in StorageOp]: { id: number; op: K; args: StorageOps[K]["args"] }; +}[StorageOp]; + +/** worker → main */ +export type StorageMessage = + | { type: "ready"; status: LibraryStatus; openMs: number } + | { type: "open_failed"; error: StorageErrorEnvelope } + | { type: "reply"; id: number; result: Result } + /** Metadata changed (architecture §7 `index.changed`); main fans it out. */ + | { type: "index_changed"; rev: number; changes: IndexChange[] }; + +/** Options the worker is started with (`workerData`). */ +export type StorageWorkerOptions = { + dir: string; + fullfsync?: boolean; + synchronous?: "FULL" | "OFF"; + /** Durability soak only: protocol lines on fd 1. */ + trace?: boolean; + /** How often the worker looks for idle work, in ms (default 5000). */ + idlePollMs?: number; +}; + +function run(library: Library, request: StorageRequest): unknown { + switch (request.op) { + case "load": + return library.load(request.args.docId, request.args.owner); + case "append": + return library.append( + request.args.docId, + request.args.owner, + request.args.record, + ); + case "compact": + return library.compact( + request.args.docId, + request.args.owner, + request.args.state, + request.args.upToSeq, + ); + case "claim": + library.claim(request.args.docId, request.args.owner); + return null; + case "close": + library.close(request.args.docId, request.args.owner); + return null; + case "releaseOwner": + library.releaseOwner(request.args.owner); + return null; + case "flush": + return null; + case "stateOf": + return library.stateOf(request.args.docId); + case "sidebar": + return library.sidebar(); + case "createDoc": + return library.createDoc(request.args); + case "createFolder": + return library.createFolder(request.args); + case "renameDoc": + return library.renameDoc(request.args.id, request.args.title); + case "moveDoc": + return library.moveDoc( + request.args.id, + request.args.folderId, + request.args.sortKey, + ); + case "trashDoc": + return library.trashDoc(request.args.id, request.args.group ?? null); + case "restoreDoc": + return library.restoreDoc(request.args.id); + case "purgeDoc": + return library.purgeDoc(request.args.id); + case "createTag": + return library.createTag(request.args.name, request.args.color ?? null); + case "setDocTags": + return library.setDocTags(request.args.docId, request.args.tagIds); + case "listDailyCopies": + return library.listDailyCopies(); + case "readDailyCopy": + return library.readDailyCopy(request.args.date, request.args.docId); + case "runIdleJobs": + return library.runIdleJobs(); + case "stats": + return library.stats(); + case "shutdown": + library.shutdown(); + return null; + } +} + +/** Runs one request; a failure becomes an envelope, never a throw. */ +export function dispatch( + library: Library, + request: StorageRequest, +): Result { + try { + return { ok: true, value: run(library, request) }; + } catch (error) { + return { ok: false, error: toEnvelope(error) }; + } +} + +/** The `ArrayBuffer`s in a value that can move instead of being copied. */ +export function transferablesOf(value: unknown): ArrayBuffer[] { + const found: ArrayBuffer[] = []; + const seen = new Set(); + const visit = (item: unknown, depth: number) => { + if (depth > 3 || item === null || typeof item !== "object") return; + if (item instanceof Uint8Array) { + const { buffer } = item; + // Only a buffer the view owns whole; never a pooled or shared one. + if ( + buffer instanceof ArrayBuffer && + item.byteOffset === 0 && + item.byteLength === buffer.byteLength && + !seen.has(buffer) + ) { + seen.add(buffer); + found.push(buffer); + } + return; + } + if (Array.isArray(item)) { + for (const entry of item) visit(entry, depth + 1); + return; + } + for (const entry of Object.values(item)) visit(entry, depth + 1); + }; + visit(value, 0); + return found; +} diff --git a/desktop/storage/worker.ts b/desktop/storage/worker.ts new file mode 100644 index 00000000..f4020017 --- /dev/null +++ b/desktop/storage/worker.ts @@ -0,0 +1,115 @@ +/** + * The storage worker's entry (D-A1): a `worker_thread` inside Electron + * main, and the only SQLite connection to the library. Main starts it in + * parallel with the first window and sends it validated requests; it + * answers them one at a time, in order, so a `flush` or `shutdown` queued + * behind appends runs after they have committed. + * + * Messages: `ready` (or `open_failed`) once, then one `reply` per request, + * and `index_changed` after a metadata write. Idle jobs (daily copy, Trash) + * run between requests after 30 s without a write. + * + * With `trace`, the engine's protocol lines (`BATCH_BEGIN`, `ACK `, + * `COMPACT_BEGIN`, `COMPACT_ACK`) go straight to fd 1 with `writeSync`, so + * the durability soak knows what committed before it kills the process. + */ +import { writeSync } from "node:fs"; +import { parentPort, workerData } from "node:worker_threads"; + +import { openNodeSqlite } from "./drivers/node-sqlite"; +import { toEnvelope } from "./engine/errors"; +import { Library } from "./engine/library"; +import { createNodeHost } from "./node-host"; +import { + dispatch, + type StorageMessage, + type StorageRequest, + type StorageWorkerOptions, + transferablesOf, +} from "./protocol"; + +const port = parentPort; +if (!port) throw new Error("the storage worker must run in a worker_thread"); + +const options = workerData as StorageWorkerOptions; + +const post = (message: StorageMessage) => { + port.postMessage( + message, + message.type === "reply" && message.result.ok + ? transferablesOf(message.result.value) + : [], + ); +}; + +const line = (text: string) => { + writeSync(1, `${text}\n`); +}; + +let library: Library | null = null; +const started = performance.now(); +try { + library = Library.open({ + dir: options.dir, + host: createNodeHost(), + openSqlite: openNodeSqlite, + fullfsync: options.fullfsync, + synchronous: options.synchronous, + trace: options.trace ? line : undefined, + }); + post({ + type: "ready", + status: library.status, + openMs: performance.now() - started, + }); +} catch (error) { + post({ type: "open_failed", error: toEnvelope(error) }); +} + +/** Sends `index_changed` if anything the sidebar shows moved. */ +function announceChanges(open: Library) { + const changed = open.takeIndexChanges(); + if (changed) post({ type: "index_changed", ...changed }); +} + +let idleTimer: ReturnType | null = null; +if (library) { + const open = library; + idleTimer = setInterval(() => { + try { + open.runIdleJobs(); + } catch { + // Only the Trash purge can throw here; it retries on the next pass. + } + announceChanges(open); + }, options.idlePollMs ?? 5000); + idleTimer.unref(); +} + +port.on("message", (request: StorageRequest) => { + if (!library) { + post({ + type: "reply", + id: request.id, + result: { + ok: false, + error: { + code: "library_unavailable", + retryable: true, + message: "the library did not open", + }, + }, + }); + return; + } + const open = library; + const result = dispatch(open, request); + post({ type: "reply", id: request.id, result }); + if (request.op === "shutdown") { + if (idleTimer) clearInterval(idleTimer); + library = null; + port.close(); + return; + } + announceChanges(open); +}); diff --git a/docs/superpowers/specs/2026-11-15-desktop-architecture.md b/docs/superpowers/specs/2026-11-15-desktop-architecture.md index 5356fb1a..5fd5f1b0 100644 --- a/docs/superpowers/specs/2026-11-15-desktop-architecture.md +++ b/docs/superpowers/specs/2026-11-15-desktop-architecture.md @@ -126,8 +126,13 @@ desktop/ main/ # index, windows, app-protocol, csp, doc-handlers, quit-coordinator, menus, # dialogs, deep-links, backups, updater (lazy), diagnostics storage/worker.ts # worker_thread entry + storage/client.ts # main's handle on the worker: start, call, shutdown (P2.6) + storage/protocol.ts # request and reply types, dispatch (no node:* imports) + storage/node-host.ts # the StorageHost (fs, pid check, clock) the engine is given storage/engine/ # SQL logic over a SqliteDriver; no electron, node:* or bun:* imports storage/drivers/node-sqlite.ts + storage-harness/ # the soak's Electron main (own lockfile), like smoke/ (P2.6) + dev/bun-sqlite-driver.ts # the SqliteDriver over bun:sqlite, for tests and the dev host dev/storage-host.ts # dev and test only: doc-handlers + engine on bun:sqlite over a localhost WebSocket build/build.ts, build/electron-builder.yml ``` @@ -176,7 +181,7 @@ These are enforced by one test, `tests/unit/ci/import-boundaries.test.ts`. It re |---|---| | `desktop/shared` | anything outside itself | | `desktop/storage/engine` | `electron`, `node:*`, `bun:*`, React, `@tiptap/*` | -| `desktop/main`, `desktop/storage` | React, `lib/**` (except `lib/security/content-security-policy.ts`, `lib/doc-migrations`, `lib/prefs/schema.ts`, and the pure `lib/comments/validate.ts` and `lib/assets` sniffers) | +| `desktop/main`, `desktop/storage` | React, `lib/**` (except `lib/security/content-security-policy.ts`, `lib/doc-migrations`, `lib/prefs/schema.ts`, the pure `lib/comments/validate.ts` and `lib/assets` sniffers, and `lib/schema-names.ts`, the name lists the import guard checks; P2.6) | | `desktop/preload` | anything except `desktop/shared` and `electron` | | `lib/platform/desktop` | `desktop/{main,preload,storage}` | | `app`, `components`, `hooks`, `lib` | `desktop/**` except `desktop/shared` types; `lib/platform/desktop/**` except from `components/desktop/desktop-root.tsx` | @@ -298,6 +303,14 @@ INDEX import_sources(source_hash) | `assets.ts` (P2.6b) | Owns `/assets/`: the temp-fsync-rename write order, dedupe, the startup `tmp/` sweep, `recount` and GC with a 30-day grace (image store §3.2–§3.4). | | `jobs.ts` | One queue. Foreground calls run first; daily copy and mirror run on idle, meaning 30 s without an append. Each job yields between documents. | +**As built in P2.6.** +- **Modules.** `index.ts` is `library-index.ts` (not a barrel). `library.ts` holds the open order, the registry and the lifecycle; `lock.ts`, `cloud-folder.ts`, `sql.ts`, `driver.ts` (the `SqliteDriver` seam and SQLite error mapping) and `host.ts` (an injected `StorageHost` for files, the pid check and the clock) keep the engine free of `node:*`. `jobs.ts` is two lines in the worker: idle jobs run between requests after 30 s without a write. `mirror.ts`, `filenames.ts` and `assets.ts` come with P2.10 and P2.6b. +- **Open order.** Cloud-folder refusal, lock, pragmas, schema creation for a new file, then `format_version` before any other write (a newer library is reopened read-only and never written), then `quick_check` if `unclean_exit` was set (a failure is read-only, `corrupt`), then migrations, then `unclean_exit` is set. +- **Registry in the worker too.** `load` claims a document for a window; another window's `load` is `already_open`; an append or compaction from any window that doesn't hold the document is `forbidden`, as §7 requires of main; purge (and P2.8's import-into and restore) is `document_open` while any window holds it. After a worker restart, main replays its registry with `claim(docId, owner)`. +- **Compaction guards.** A compaction older than the stored snapshot is a no-op. An `upToSeq` past the document's last row is `invalid_argument`. `updates.seq` is `AUTOINCREMENT`, so a seq is never reused after the newest rows are compacted away. +- **`docs.schema`** defaults to 0, which is `DOC_SCHEMA` while `lib/doc-migrations` lists no steps. +- **Import guard.** `checkImport` (`lib/doc-migrations/import-guard.ts`) refuses a state over 64 MiB (`import_too_large`), bytes that aren't Yjs (`corrupt`), and nodes, marks or roots the build doesn't know (`unknown_schema`; legacy `link` and `code` pass, the import migrations rename them; roots: `default`, plus `comments` in P4.11). P2.8 calls it before anything is written. + **Backup failures.** EPERM, EACCES and Defender blocks on the backup folder raise app health `backups_failing` (§6.1; plan §5.5 "Backups failing"). The library itself is never affected. --- @@ -584,7 +597,7 @@ sequenceDiagram | Compaction invariant | property test: random appends, compactions and reopens, with the Y.Doc state compared against a reference | engine on `bun:sqlite` | | Guards and `doc-handlers` | fuzz plus golden messages, sender and registry checks | pure modules | | Reducer and chip | a table row for each plan §5.5 state; "every refusal ends in a stop or a backoff" | pure | -| Durability T1–T8, T16–T18 | `tests/load/desktop-durability-soak.ts`: SIGKILL Electron during append, compaction, migration, asset writes and image migration; a year of image churn | packaged app | +| Durability T1–T8, T16–T18 | `tests/load/desktop-durability-soak.ts`: SIGKILL Electron during append, compaction, migration, asset writes and image migration; a year of image churn | the storage worker in a windowless Electron (`desktop/storage-harness`, P2.6), verified on `bun:sqlite`; the packaged app once P2.5 lands. T7 (slow hydrate) tests the session layer, not storage: P2.1 covers it in `local-session`'s tests, and W2 again over IndexedDB. | | `BlobStore` contract and asset GC | one suite plus a property test: GC never deletes a referenced blob | `MemoryBlobStore`; the worker on `bun:sqlite`; `IdbBlobStore` in the web library | | Preferences boot script | parity with `resolveBoot()` over about 40 stored payloads; the desktop CSP hash matches the script (build test) | happy-dom; build | | Main | `argv` filter, navigation and `window.open` denial, fuses | `_electron` | diff --git a/eslint.config.mjs b/eslint.config.mjs index b5665bd6..05f3f6b7 100644 --- a/eslint.config.mjs +++ b/eslint.config.mjs @@ -11,6 +11,7 @@ const eslintConfig = defineConfig([ "build/**", "next-env.d.ts", "desktop/smoke/dist/**", + "desktop/storage-harness/dist/**", ]), { settings: { diff --git a/lib/doc-migrations/import-guard.ts b/lib/doc-migrations/import-guard.ts new file mode 100644 index 00000000..7547a6f4 --- /dev/null +++ b/lib/doc-migrations/import-guard.ts @@ -0,0 +1,96 @@ +/** + * The import guard (plan P2.6; architecture §3.3): checks a document's + * state before it enters the library from a `.tinydoc`, a web download or a + * backup. It rejects and never repairs; the import migrations + * ({@link import("./index")}) run only on a state that passed. + * + * Refused: + * - **size:** over {@link IMPORT_MAX_BYTES} (`import_too_large`); + * - **not Yjs:** bytes that don't decode (`corrupt`); + * - **unknown names:** a node or mark this build's schema doesn't know + * (`unknown_schema`). y-prosemirror would delete such content the moment + * the editor binds. The legacy `link` and `code` marks are allowed: the + * import migrations rename them; + * - **unknown roots:** a top-level type other than the editor fragment + * (`unknown_schema`). Comments add their `comments` root here, with their + * validator, in P4.11. + * + * Pure: Yjs and the schema name lists only, so main imports it. + */ +import * as Y from "yjs"; + +import { + isAllowedMarkKey, + isAllowedNodeName, + markNameFromKey, +} from "@/lib/schema-names"; + +/** The editor fragment, `EDITOR_FRAGMENT_NAME` in `lib/yjs-mark-migration.ts`. */ +const EDITOR_ROOT = "default"; + +/** + * 64 MiB: far above any document the editor stays responsive with, and + * low enough that a hostile file can't exhaust the worker's memory while it + * is decoded. + */ +export const IMPORT_MAX_BYTES = 64 * 1024 * 1024; + +export const IMPORT_ROOTS: readonly string[] = [EDITOR_ROOT]; + +export type ImportVerdict = + | { ok: true } + | { + ok: false; + code: "import_too_large" | "corrupt" | "unknown_schema"; + /** Up to five offending names, for the dialog and the log. */ + names: string[]; + }; + +export type ImportGuardOptions = { + maxBytes?: number; + roots?: readonly string[]; +}; + +const MAX_NAMES = 5; + +export function checkImport( + state: Uint8Array, + { + maxBytes = IMPORT_MAX_BYTES, + roots = IMPORT_ROOTS, + }: ImportGuardOptions = {}, +): ImportVerdict { + if (state.byteLength > maxBytes) + return { ok: false, code: "import_too_large", names: [] }; + const names = new Set(); + const note = (name: string) => { + if (names.size < MAX_NAMES) names.add(name.slice(0, 64)); + }; + const doc = new Y.Doc(); + try { + for (const struct of Y.decodeUpdate(state).structs) { + if (!(struct instanceof Y.Item)) continue; + const { content } = struct; + if ( + content instanceof Y.ContentType && + content.type instanceof Y.XmlElement && + !isAllowedNodeName(content.type.nodeName) + ) + note(content.type.nodeName); + else if ( + content instanceof Y.ContentFormat && + !isAllowedMarkKey(content.key) + ) + note(markNameFromKey(content.key)); + } + Y.applyUpdate(doc, state); + for (const root of doc.share.keys()) if (!roots.includes(root)) note(root); + } catch { + return { ok: false, code: "corrupt", names: [] }; + } finally { + doc.destroy(); + } + return names.size === 0 + ? { ok: true } + : { ok: false, code: "unknown_schema", names: [...names] }; +} diff --git a/lib/doc-migrations/index.ts b/lib/doc-migrations/index.ts new file mode 100644 index 00000000..0d645b1d --- /dev/null +++ b/lib/doc-migrations/index.ts @@ -0,0 +1,72 @@ +/** + * Document schema migrations (desktop architecture §5.1, D-A5): pure Yjs + * forward edits that bring a stored document up to {@link DOC_SCHEMA}. + * + * - **On open**, `local-session` runs them after hydrate and before the + * editor binds (y-prosemirror deletes runs whose marks the schema doesn't + * know, `lib/yjs-mark-migration.ts`). The composition root passes + * `DOC_SCHEMA` and `DOC_MIGRATIONS`; local-session runs each step in its + * own `MIGRATION_ORIGIN` transaction, outside undo, and appends it as + * origin `migration` with `schemaTo`, which the storage engine writes to + * `docs.schema` in the same transaction. + * - **At import** (P2.8), the same functions run on `.tinydoc` and web + * downloads. + * + * The list starts empty: schema 0 is today's documents. Each entry raises + * `DOC_SCHEMA` by one. A document whose schema is above `DOC_SCHEMA` was + * written by a newer app and opens read-only (`doc_too_new`). + */ +import type * as Y from "yjs"; + +export type DocMigration = { + /** The schema this step produces: the previous step's `to` plus one. */ + to: number; + /** Edits `doc` in place. Runs inside the caller's transaction. */ + migrate(doc: Y.Doc): void; +}; + +export const DOC_MIGRATIONS: readonly DocMigration[] = []; + +/** The schema documents written by this build are at. */ +export const DOC_SCHEMA: number = DOC_MIGRATIONS.at(-1)?.to ?? 0; + +export type MigrationPlan = + | { kind: "current" } + | { kind: "too_new" } + | { kind: "migrate"; steps: readonly DocMigration[]; schemaTo: number }; + +/** What opening a document stored at `schema` needs. */ +export function planDocMigrations( + schema: number, + migrations: readonly DocMigration[] = DOC_MIGRATIONS, +): MigrationPlan { + const current = migrations.at(-1)?.to ?? 0; + if (!Number.isInteger(schema) || schema < 0) + throw new RangeError(`bad document schema ${schema}`); + if (schema > current) return { kind: "too_new" }; + if (schema === current) return { kind: "current" }; + const steps = migrations.filter((step) => step.to > schema); + steps.forEach((step, i) => { + if (step.to !== schema + i + 1) + throw new RangeError(`migration to ${step.to} is out of order`); + }); + return { kind: "migrate", steps, schemaTo: current }; +} + +/** + * Runs the steps a plan lists in one transaction under `origin`, so they + * produce one update. For import, where a file migrates all or nothing; + * the open hook in local-session runs one transaction per step instead. + * Returns the schema reached, or null when there was nothing to do. + */ +export function runDocMigrations( + doc: Y.Doc, + plan: MigrationPlan, + origin: unknown, +): number | null { + if (plan.kind !== "migrate") return null; + doc.transact(() => { + for (const step of plan.steps) step.migrate(doc); + }, origin); + return plan.schemaTo; +} diff --git a/lib/platform/desktop/rpc-doc-store.ts b/lib/platform/desktop/rpc-doc-store.ts new file mode 100644 index 00000000..5ba12925 --- /dev/null +++ b/lib/platform/desktop/rpc-doc-store.ts @@ -0,0 +1,68 @@ +/** + * A `DocStore` over request/reply calls to the storage worker. `IpcDocStore` + * (P2.5) passes a `call` that goes through `window.tinydocy`; tests and the + * Electron soak pass one that goes to a `StorageClient` (adding the window's + * owner id) or dispatches in-process. + * + * A rejection that carries a storage `code` becomes a `DocStoreError`; + * anything else (a dead transport) becomes `library_unavailable`, which is + * retryable, so the batcher keeps the batch and retries it in order. + */ +import { + type AppendRecord, + type CompactResult, + DOC_STORE_ERROR_CODES, + type DocStore, + DocStoreError, + type DocStoreErrorCode, + type LoadedDoc, +} from "@/lib/platform/doc-store"; + +export type DocStoreCalls = { + load: [{ docId: string }, LoadedDoc]; + append: [{ docId: string; record: AppendRecord }, number]; + compact: [ + { docId: string; state: Uint8Array; upToSeq: number }, + CompactResult, + ]; + close: [{ docId: string }, null]; + flush: [Record, null]; +}; + +export type DocStoreCall = ( + op: K, + args: DocStoreCalls[K][0], +) => Promise; + +const CODES: ReadonlySet = new Set(DOC_STORE_ERROR_CODES); + +export function toDocStoreError(error: unknown): DocStoreError { + if (error instanceof DocStoreError) return error; + const code = (error as { code?: unknown } | null)?.code; + const message = error instanceof Error ? error.message : String(error); + if (typeof code === "string" && CODES.has(code)) + return new DocStoreError(code as DocStoreErrorCode, message); + return new DocStoreError("library_unavailable", message); +} + +export function createRpcDocStore(call: DocStoreCall): DocStore { + const wrap = async (run: () => Promise): Promise => { + try { + return await run(); + } catch (error) { + throw toDocStoreError(error); + } + }; + return { + load: (docId) => wrap(() => call("load", { docId })), + append: (docId, record) => wrap(() => call("append", { docId, record })), + compact: (docId, state, upToSeq) => + wrap(() => call("compact", { docId, state, upToSeq })), + close: async (docId) => { + await wrap(() => call("close", { docId })); + }, + flush: async () => { + await wrap(() => call("flush", {})); + }, + }; +} diff --git a/package.json b/package.json index 6b4393af..630944f6 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,7 @@ "build": "next build", "start": "next start", "lint": "eslint .", - "typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.sw.json", + "typecheck": "tsc --noEmit && tsc --noEmit -p tsconfig.sw.json && tsc --noEmit -p tsconfig.desktop-node.json", "format": "biome format --write .", "format:check": "biome format .", "check": "biome check .", diff --git a/tests/helpers/engine-doc-store.ts b/tests/helpers/engine-doc-store.ts new file mode 100644 index 00000000..3dd5c287 --- /dev/null +++ b/tests/helpers/engine-doc-store.ts @@ -0,0 +1,48 @@ +/** + * `DocStore`s over a real storage engine, in-process on `bun:sqlite`: what + * the shared contract suite runs against on every PR (architecture §11). + * Each store is one window (`owner`); two stores over one library model two + * windows, for `already_open`. + * + * Requests go through the worker's own `dispatch`, so the in-process path + * and the Electron worker share every line but the transport. Bytes are + * copied on the way in, as IPC would. + */ +import type { Library } from "@/desktop/storage/engine/library"; +import { dispatch, type StorageRequest } from "@/desktop/storage/protocol"; +import { + createRpcDocStore, + type DocStoreCall, +} from "@/lib/platform/desktop/rpc-doc-store"; +import type { DocStore } from "@/lib/platform/doc-store"; + +function copyBytes(value: T): T { + if (value instanceof Uint8Array) return value.slice() as T; + if (Array.isArray(value)) return value.map(copyBytes) as T; + if (value && typeof value === "object") + return Object.fromEntries( + Object.entries(value).map(([k, v]) => [k, copyBytes(v)]), + ) as T; + return value; +} + +let nextId = 1; + +export function createEngineDocStore( + library: Library, + owner: string, +): DocStore { + const call: DocStoreCall = async (op, args) => { + const request = { + id: nextId++, + op, + args: op === "flush" ? {} : { ...copyBytes(args), owner }, + } as StorageRequest; + // A microtask boundary, as a real transport has. + await Promise.resolve(); + const result = dispatch(library, request); + if (!result.ok) throw result.error; + return copyBytes(result.value) as never; + }; + return createRpcDocStore(call); +} diff --git a/tests/helpers/storage-library.ts b/tests/helpers/storage-library.ts new file mode 100644 index 00000000..dc5d8f0a --- /dev/null +++ b/tests/helpers/storage-library.ts @@ -0,0 +1,68 @@ +/** + * Test helpers for the desktop storage engine on `bun:sqlite`: a temp + * library folder, a controllable clock, and an opener with test defaults. + */ +import { mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { openBunSqlite } from "@/desktop/dev/bun-sqlite-driver"; +import type { StorageHost } from "@/desktop/storage/engine/host"; +import { + Library, + type OpenLibraryOptions, +} from "@/desktop/storage/engine/library"; +import { createNodeHost } from "@/desktop/storage/node-host"; + +export type TestClock = { now: number; advance(ms: number): void }; + +export function createClock( + start = Date.UTC(2026, 10, 17, 9, 0, 0), +): TestClock { + return { + now: start, + advance(ms) { + this.now += ms; + }, + }; +} + +export function createTempDir(prefix = "tinydocy-lib-"): { + dir: string; + cleanUp(): void; +} { + const dir = mkdtempSync(join(tmpdir(), prefix)); + return { dir, cleanUp: () => rmSync(dir, { recursive: true, force: true }) }; +} + +export function createTestHost( + clock: TestClock, + overrides: Partial = {}, +): StorageHost { + return createNodeHost({ now: () => clock.now, ...overrides }); +} + +export function openTestLibrary( + dir: string, + host: StorageHost, + options: Partial = {}, +): Library { + return Library.open({ + dir, + host, + openSqlite: openBunSqlite, + // Tests don't need F_FULLFSYNC's 4 ms per commit. + fullfsync: false, + ...options, + }); +} + +/** + * Simulates a crash: the library is dropped without `shutdown`, so + * `unclean_exit` stays set and the lock file stays behind. The lock's pid is + * this test process, which the next open treats as stale. + */ +export function crash(library: Library) { + (library as unknown as { db: { close(): void } | null }).db?.close(); + (library as unknown as { db: unknown }).db = null; +} diff --git a/tests/load/desktop-durability-soak.ts b/tests/load/desktop-durability-soak.ts new file mode 100644 index 00000000..5fa349ee --- /dev/null +++ b/tests/load/desktop-durability-soak.ts @@ -0,0 +1,876 @@ +/** + * Desktop durability and growth soak (plan §9 T1–T8, P2.6's exit row). + * + * Launches the real storage worker inside Electron (the harness in + * `desktop/storage-harness`, built first), SIGKILLs it at chosen moments + * (`TerminateProcess` on Windows), waits until the pid is gone, then + * reopens the library with the engine on `bun:sqlite`, which also proves + * the two drivers read each other's files. The shape follows spike S2's + * kill harness. + * + * bun tests/load/desktop-durability-soak.ts --case=all|contract|T1|T2|T3|T4|T5|T6|T8 + * [--kills=200] [--out=soak-out] [--verbose] + * + * T7 (slow-hydrate overwrite) is a web-library case: it tests the session + * layer, not storage, and runs with W2. + * + * Writes `/.json` and a Markdown table to `$GITHUB_STEP_SUMMARY`. + * Exits 1 if any case fails. Run it only on CI runners (it starts + * Electron); on a desktop machine the windows stay hidden but it still + * takes the CPU for minutes. + */ + +import { Database } from "bun:sqlite"; +import { createHash } from "node:crypto"; +import { + appendFileSync, + existsSync, + mkdirSync, + mkdtempSync, + readFileSync, + rmSync, + statSync, + writeFileSync, +} from "node:fs"; +import { createRequire } from "node:module"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import * as Y from "yjs"; + +import { openBunSqlite } from "@/desktop/dev/bun-sqlite-driver"; +import { DAILY_KEEP } from "@/desktop/storage/engine/daily-copy"; +import { + IDLE_MS, + LIBRARY_FILE, + Library, + type OpenLibraryOptions, +} from "@/desktop/storage/engine/library"; +import { LOCK_FILE } from "@/desktop/storage/engine/lock"; +import { createNodeHost } from "@/desktop/storage/node-host"; +import { + STEP_ROWS, + T5_STEPS, + T5_TARGET, +} from "@/desktop/storage-harness/format-steps"; +import { + editCount, + PAD, + padUpdate, + parseArgs, + seededRandom, + TEXT, +} from "@/desktop/storage-harness/gen"; + +const ROOT = join(import.meta.dir, "../.."); +const HARNESS = join(ROOT, "desktop/storage-harness"); +const args = parseArgs(process.argv.slice(2)); +const OUT = args.out ?? "soak-out"; +const KILLS = Number(args.kills ?? 200); +const isWindows = process.platform === "win32"; + +mkdirSync(OUT, { recursive: true }); + +const harnessMain = join(HARNESS, "dist", "main.cjs"); +/** The harness's own Electron (its package has the dependency, not the root). */ +function electronBinary(): string { + if (!existsSync(harnessMain)) + throw new Error( + "build the harness first: cd desktop/storage-harness && bun run build", + ); + return createRequire(join(HARNESS, "package.json"))("electron") as string; +} + +type CaseResult = { + name: string; + pass: boolean; + summary: string; + details: Record; + failures: string[]; +}; + +const note = (text: string) => { + if (args.verbose) console.log(text); +}; + +/** + * Removes a temp library. On Windows, Defender or the indexer may still hold + * a file for a moment; a cleanup failure is logged, never a failed case. + */ +function removeDir(dir: string) { + try { + rmSync(dir, { + recursive: true, + force: true, + maxRetries: 10, + retryDelay: 200, + }); + } catch (error) { + console.warn(`could not remove ${dir}: ${String(error)}`); + } +} + +// ── Engine access (bun:sqlite) ────────────────────────────────────────── + +function openEngine( + dir: string, + options: Partial = {}, +): Library { + return Library.open({ + dir, + host: createNodeHost(), + openSqlite: openBunSqlite, + fullfsync: false, + ...options, + }); +} + +function textOf(library: Library, docId: string, owner = "verify") { + const loaded = library.load(docId, owner); + const doc = new Y.Doc(); + if (loaded.snapshot) Y.applyUpdate(doc, loaded.snapshot); + for (const update of loaded.updates) Y.applyUpdate(doc, update); + library.close(docId, owner); + return { + text: doc.getText(TEXT).toString(), + padLength: doc.getText(PAD).length, + maxSeq: loaded.maxSeq, + }; +} + +function seedLibrary(dir: string, padBytes: number): string { + const library = openEngine(dir); + const { id } = library.createDoc({ title: "Soak" }); + if (padBytes > 0) { + library.load(id, "seed"); + const seq = library.append(id, "seed", { + origin: "local", + update: padUpdate(padBytes), + }); + const doc = new Y.Doc(); + Y.applyUpdate(doc, padUpdate(padBytes)); + library.compact(id, "seed", Y.encodeStateAsUpdate(doc), seq); + library.close(id, "seed"); + } + library.shutdown(); + return id; +} + +// ── Electron runs ─────────────────────────────────────────────────────── + +type Run = { + lines: string[]; + exitCode: number | null; + stderr: string; + pid: number; +}; + +function pidGone(pid: number): boolean { + if (isWindows) { + const out = Bun.spawnSync(["tasklist", "/FI", `PID eq ${pid}`, "/NH"]); + return !out.stdout.toString().includes(` ${pid} `); + } + try { + process.kill(pid, 0); + return false; + } catch (error) { + return (error as { code?: string }).code === "ESRCH"; + } +} + +/** + * Starts the harness and feeds each stdout line to `onLine`, which may + * return "kill" to SIGKILL the process. Resolves once the process exited + * and its pid is gone. + */ +async function runHarness( + harnessArgs: string[], + onLine: (line: string, kill: () => void) => void = () => {}, + timeoutMs = 120_000, +): Promise { + const proc = Bun.spawn([electronBinary(), harnessMain, ...harnessArgs], { + stdout: "pipe", + stderr: "pipe", + env: { ...process.env, ELECTRON_ENABLE_STACK_DUMPING: "1" }, + }); + let killed = false; + const kill = () => { + if (killed) return; + killed = true; + proc.kill("SIGKILL"); + }; + const timer = setTimeout(kill, timeoutMs); + const lines: string[] = []; + const decoder = new TextDecoder(); + let buffer = ""; + for await (const chunk of proc.stdout) { + buffer += decoder.decode(chunk, { stream: true }); + let newline = buffer.indexOf("\n"); + while (newline >= 0) { + const line = buffer.slice(0, newline).trimEnd(); + buffer = buffer.slice(newline + 1); + lines.push(line); + if (!killed) onLine(line, kill); + newline = buffer.indexOf("\n"); + } + } + const exitCode = await proc.exited; + clearTimeout(timer); + const stderr = await new Response(proc.stderr).text(); + for (let i = 0; i < 50 && !pidGone(proc.pid); i += 1) await Bun.sleep(100); + if (!pidGone(proc.pid)) throw new Error(`pid ${proc.pid} still running`); + return { lines, exitCode, stderr, pid: proc.pid, killed }; +} + +// ── contract: the engine DocStore cases inside the Electron worker ────── + +async function caseContract(): Promise { + const run = await runHarness(["--mode=contract"]); + const ok = run.lines.filter((l) => l.startsWith("CASE_OK")).length; + const failed = run.lines.filter((l) => l.startsWith("CASE_FAIL")); + return { + name: "contract", + pass: run.exitCode === 0 && failed.length === 0 && ok > 0, + summary: `P2.1's DocStore contract: ${ok} cases passed on node:sqlite in the Electron worker`, + details: { ok, exitCode: run.exitCode }, + failures: [ + ...failed, + ...(run.exitCode === 0 ? [] : [run.stderr.slice(0, 2000)]), + ], + }; +} + +// ── T1, T2, T3 control: kill during append or compaction ─────────────── + +type KillTarget = "append" | "compaction"; + +async function killLoop( + name: string, + target: KillTarget, + kills: number, + options: { dense?: boolean; synchronous?: "FULL" | "OFF" } = {}, +): Promise { + const dir = mkdtempSync(join(tmpdir(), `soak-${name}-`)); + const padBytes = target === "compaction" ? 5 * 1024 * 1024 : 64 * 1024; + const docId = seedLibrary(dir, padBytes); + const random = seededRandom(kills * 7919 + target.length); + const failures: string[] = []; + let inside = 0; + let afterFirstAck = 0; + let maxK = 0; + for (let i = 1; i <= kills; i += 1) { + let ackSeq = 0; + let ackedK = 0; + let last = ""; + let sawAck = false; + const delay = target === "append" ? 50 + random() * 1950 : random() * 20; + const run = await runHarness( + [ + "--mode=churn", + `--dir=${dir}`, + `--doc=${docId}`, + `--seed=${i}`, + `--target=${target}`, + `--synchronous=${options.synchronous ?? "FULL"}`, + ...(options.dense ? ["--dense"] : []), + ], + (line, kill) => { + const [word, a, b] = line.split(" "); + if (word === "ACK") { + ackSeq = Math.max(ackSeq, Number(a)); + sawAck = true; + } else if (word === "EDITS") ackedK = Math.max(ackedK, Number(b)); + if (/^(BATCH_BEGIN|ACK|COMPACT_BEGIN|COMPACT_ACK)$/.test(word ?? "")) + last = word ?? ""; + if (target === "append" && word === "READY") setTimeout(kill, delay); + if (target === "compaction" && word === "COMPACT_BEGIN") + setTimeout(kill, delay); + if (target === "compaction" && word === "READY") + setTimeout(kill, 15_000); // never compacted: still kill + }, + 60_000, + ); + if (!run.killed) { + failures.push( + `kill ${i}: exited by itself (${run.exitCode}) ${run.stderr.slice(0, 500)}`, + ); + continue; + } + if (last === "BATCH_BEGIN" || last === "COMPACT_BEGIN") inside += 1; + if (sawAck) afterFirstAck += 1; + try { + const library = openEngine(dir); + const { status } = library; + if (status.quickCheck !== null && status.quickCheck !== "ok") + failures.push(`kill ${i}: quick_check ${status.quickCheck}`); + const stored = textOf(library, docId); + library.shutdown(); + const k = editCount(stored.text); + if (k === null) failures.push(`kill ${i}: text has a gap or a torn edit`); + else { + maxK = k; + if (k < ackedK) + failures.push(`kill ${i}: lost acked edits (${k} < ${ackedK})`); + } + if (stored.maxSeq < ackSeq) + failures.push( + `kill ${i}: lost acked seq (${stored.maxSeq} < ${ackSeq})`, + ); + if (stored.padLength !== padBytes) + failures.push(`kill ${i}: pad is ${stored.padLength}, not ${padBytes}`); + } catch (error) { + failures.push(`kill ${i}: reopen failed: ${String(error)}`); + } + note( + `${name} ${i}/${kills} last=${last} acked=${ackedK} failures=${failures.length}`, + ); + } + removeDir(dir); + return { + name, + pass: failures.length === 0, + summary: `${kills} kills, ${afterFirstAck} after the first ack, ${inside} inside a transaction; ${failures.length} failures; final K ${maxK}`, + details: { kills, afterFirstAck, inside, maxK, target, ...options }, + failures: failures.slice(0, 50), + }; +} + +// ── T3: fullfsync on vs off, plus the synchronous=OFF control ───────── + +async function caseT3(): Promise { + const failures: string[] = []; + const timings: Record[] = []; + for (const fullfsync of ["1", "0"]) { + const dir = mkdtempSync(join(tmpdir(), "soak-t3-")); + const run = await runHarness([ + "--mode=commit-timing", + `--dir=${dir}`, + `--fullfsync=${fullfsync}`, + "--count=300", + ]); + removeDir(dir); + const line = run.lines.find((l) => l.startsWith("TIMING ")); + if (!line) { + failures.push( + `timing fullfsync=${fullfsync}: ${run.stderr.slice(0, 500)}`, + ); + continue; + } + const timing = JSON.parse(line.slice(7)) as { + pragmas: Record; + } & Record; + timings.push(timing); + const expectedFull = + process.platform === "darwin" && fullfsync === "1" ? 1 : 0; + if (timing.pragmas.journalMode !== "wal") + failures.push(`journal_mode ${timing.pragmas.journalMode}`); + if (timing.pragmas.synchronous !== 2) + failures.push(`synchronous ${timing.pragmas.synchronous}`); + if ( + process.platform === "darwin" && + timing.pragmas.fullfsync !== expectedFull + ) + failures.push( + `fullfsync ${timing.pragmas.fullfsync}, wanted ${expectedFull}`, + ); + } + const control = await killLoop("T3-control", "append", Math.min(KILLS, 50), { + synchronous: "OFF", + }); + failures.push(...control.failures); + return { + name: "T3", + pass: failures.length === 0, + summary: `pragmas read back as set; commit p50 ${timings + .map( + (t) => + `${Number(t.p50).toFixed(2)} ms (fullfsync ${t.fullfsync ? "on" : "off"})`, + ) + .join( + " vs ", + )}; synchronous=OFF control: ${control.summary}. Power loss itself needs owner hardware.`, + details: { timings, control: control.details }, + failures, + }; +} + +// ── T4: quit with writes in flight ───────────────────────────────────── + +async function caseT4(runs: number): Promise { + const dir = mkdtempSync(join(tmpdir(), "soak-t4-")); + const docId = seedLibrary(dir, 64 * 1024); + const random = seededRandom(4); + const failures: string[] = []; + let clean = 0; + for (let i = 1; i <= runs; i += 1) { + const quitAfter = Math.round(300 + random() * 2700); + const run = await runHarness([ + "--mode=churn", + `--dir=${dir}`, + `--doc=${docId}`, + `--seed=${100 + i}`, + `--quit-after-ms=${quitAfter}`, + ]); + const quit = run.lines.find((l) => l.startsWith("QUIT ")); + if (run.exitCode !== 0 || !quit) { + failures.push( + `run ${i}: exit ${run.exitCode} ${run.stderr.slice(0, 500)}`, + ); + continue; + } + const [, outcome, kText] = quit.split(" "); + if (outcome === "clean") clean += 1; + else failures.push(`run ${i}: shutdown ${outcome}`); + if (existsSync(join(dir, LOCK_FILE))) + failures.push(`run ${i}: lock left behind`); + const wal = join(dir, `${LIBRARY_FILE}-wal`); + if (existsSync(wal) && statSync(wal).size > 0) + failures.push(`run ${i}: WAL not checkpointed (${statSync(wal).size} B)`); + const library = openEngine(dir); + if (library.status.uncleanExit) + failures.push(`run ${i}: unclean_exit still set`); + const k = editCount(textOf(library, docId).text); + library.shutdown(); + if (k !== Number(kText)) + failures.push(`run ${i}: ${k} edits stored, ${kText} typed before quit`); + } + removeDir(dir); + return { + name: "T4", + pass: failures.length === 0, + summary: `${runs} quits with edits in flight: ${clean} clean shutdowns, every typed edit stored`, + details: { runs, clean }, + failures, + }; +} + +// ── T5: migration interrupted ────────────────────────────────────────── + +function stepTables(dir: string) { + const db = new Database(join(dir, LIBRARY_FILE)); + const counts: Record = {}; + for (const step of T5_STEPS) { + const exists = db + .query("SELECT 1 FROM sqlite_master WHERE name = ?") + .get(`soak_step_${step.to}`); + if (exists) + counts[step.to] = ( + db.query(`SELECT COUNT(*) AS n FROM soak_step_${step.to}`).get() as { + n: number; + } + ).n; + } + const check = ( + db.query("PRAGMA quick_check").get() as { quick_check: string } + ).quick_check; + db.close(); + return { counts, check }; +} + +async function caseT5(kills: number): Promise { + const failures: string[] = []; + const reached: Record = {}; + // Calibrate: how long a full migration takes on this runner. + const calibrationDir = mkdtempSync(join(tmpdir(), "soak-t5-")); + seedLibrary(calibrationDir, 0); + const t0 = performance.now(); + const calibration = await runHarness([ + "--mode=migrate", + `--dir=${calibrationDir}`, + ]); + const fullMs = performance.now() - t0; + if (!calibration.lines.some((l) => l === `MIGRATED ${T5_TARGET} null`)) + failures.push( + `calibration: ${calibration.lines.join(" | ")} ${calibration.stderr.slice(0, 500)}`, + ); + removeDir(calibrationDir); + const random = seededRandom(5); + for (let i = 1; i <= kills; i += 1) { + const dir = mkdtempSync(join(tmpdir(), "soak-t5-")); + const docId = seedLibrary(dir, 4096); + const delay = random() * fullMs; + await runHarness(["--mode=migrate", `--dir=${dir}`], (line, kill) => { + if (line === "READY") setTimeout(kill, delay); + }); + // As an app at format 1 sees it: a newer library opens read-only. + const probe = openEngine(dir); + const version = probe.status.formatVersion; + probe.shutdown(); + reached[version] = (reached[version] ?? 0) + 1; + const { counts, check } = stepTables(dir); + if (check !== "ok") failures.push(`kill ${i}: quick_check ${check}`); + for (const step of T5_STEPS) { + const rows = counts[step.to]; + if (step.to <= version && rows !== STEP_ROWS) + failures.push( + `kill ${i}: v${version} but step ${step.to} has ${rows} rows`, + ); + if (step.to > version && rows !== undefined) + failures.push( + `kill ${i}: v${version} but step ${step.to}'s table exists`, + ); + } + // The migrating app finishes the job. + const library = openEngine(dir, { + formatVersion: T5_TARGET, + formatSteps: T5_STEPS, + }); + if (library.status.formatVersion !== T5_TARGET || library.status.readOnly) + failures.push( + `kill ${i}: resume reached v${library.status.formatVersion} ${library.status.readOnly}`, + ); + const pad = textOf(library, docId).padLength; + library.shutdown(); + if (pad !== 4096) failures.push(`kill ${i}: document damaged (${pad})`); + removeDir(dir); + } + return { + name: "T5", + pass: failures.length === 0, + summary: `${kills} kills during a 3-step migration (${Math.round(fullMs)} ms); stopped at v1–v4: ${JSON.stringify(reached)}; each step all-or-nothing, every resume reached v${T5_TARGET}`, + details: { kills, fullMs, reached }, + failures, + }; +} + +// ── T6: a newer library is read-only ─────────────────────────────────── + +async function caseT6(): Promise { + const dir = mkdtempSync(join(tmpdir(), "soak-t6-")); + seedLibrary(dir, 1024); + const db = new Database(join(dir, LIBRARY_FILE)); + db.exec("UPDATE meta SET value = '9' WHERE key = 'format_version'"); + db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); + db.close(); + const hash = () => + createHash("sha256") + .update(readFileSync(join(dir, LIBRARY_FILE))) + .digest("hex"); + const before = hash(); + const run = await runHarness(["--mode=readonly", `--dir=${dir}`]); + const after = hash(); + removeDir(dir); + const failures: string[] = []; + const status = run.lines.find((l) => l.startsWith("STATUS ")); + if (!status || JSON.parse(status.slice(7)).readOnly !== "library_too_new") + failures.push(`status: ${status}`); + if (!run.lines.includes("WRITE refused library_too_new")) + failures.push(`write: ${run.lines.find((l) => l.startsWith("WRITE"))}`); + if (!run.lines.includes("SIDEBAR 1")) + failures.push("the newer library's list didn't load"); + if (before !== after) failures.push("library.sqlite changed"); + if (run.exitCode !== 0) + failures.push(`exit ${run.exitCode} ${run.stderr.slice(0, 500)}`); + return { + name: "T6", + pass: failures.length === 0, + summary: + "a format-9 library opened read-only in the Electron worker: list shown, writes refused, file byte-identical", + details: { before, after }, + failures, + }; +} + +// ── T8: a year of editing, then startup with 1,000 documents ────────── + +/** + * Architecture §10 targets, from process start: "Worker ready" and + * "Sidebar populated" (Mac means an M1, Win a mid-range laptop; Linux uses + * the Windows row). CI applies §10's 1.5× slack to the sidebar gate. + */ +function budgets() { + return process.platform === "darwin" + ? { workerReadyMs: 150, sidebarMs: 600 } + : { workerReadyMs: 300, sidebarMs: 1000 }; +} + +async function caseT8(): Promise { + const docs = Number(args.docs ?? 1000); + const docBytes = Number(args["doc-bytes"] ?? 200 * 1024); + const days = Number(args.days ?? 365); + const dir = mkdtempSync(join(tmpdir(), "soak-t8-")); + const failures: string[] = []; + const clock = { now: Date.UTC(2026, 0, 1, 9) }; + const host = createNodeHost({ now: () => clock.now }); + const open = () => + Library.open({ dir, host, openSqlite: openBunSqlite, fullfsync: false }); + let library = open(); + const random = seededRandom(8); + const started = performance.now(); + + // Seed: 1,000 documents of about 200 KB in 30 folders, with tags. + const folders = Array.from( + { length: 30 }, + (_, i) => library.createFolder({ name: `Folder ${i}` }).id, + ); + const tags = Array.from( + { length: 20 }, + (_, i) => library.createTag(`tag-${i}`).id, + ); + const template = new Y.Doc(); + template.getText(PAD).insert(0, "p".repeat(docBytes)); + const templateState = Y.encodeStateAsUpdate(template); + const ids: string[] = []; + const createSeeded = () => { + const { id } = library.createDoc({ + title: `Document ${ids.length}`, + folderId: folders[Math.floor(random() * folders.length)] ?? null, + }); + library.load(id, "seed"); + library.compact(id, "seed", templateState, 0); + library.close(id, "seed"); + const picked = tags.filter(() => random() < 0.08); + if (picked.length) library.setDocTags(id, picked); + ids.push(id); + return id; + }; + for (let i = 0; i < docs; i += 1) createSeeded(); + const seededBytes = library.stats().bytes; + + // References for a few documents, checked at the end. + const watched = new Map( + ids.slice(0, 5).map((id) => [id, ""]), + ); + const monthly: Record[] = []; + // How long each daily VACUUM INTO blocked the worker: a quit flush queues + // behind it (D-A2's "a clean quit loses nothing" needs it under ~3 s). + const copyMs: number[] = []; + let appends = 0; + let compactions = 0; + let trashed = 0; + + for (let day = 0; day < days; day += 1) { + const today = new Set( + [...watched.keys()].filter(() => random() < 0.3), + ); + while (today.size < 20) + today.add(ids[Math.floor(random() * ids.length)] as string); + for (const id of today) { + // An editing session: hydrate, type in batches, compact on close. + const loaded = library.load(id, "w"); + const doc = new Y.Doc(); + if (loaded.snapshot) Y.applyUpdate(doc, loaded.snapshot); + for (const u of loaded.updates) Y.applyUpdate(doc, u); + let acked = loaded.maxSeq; + let rowsSinceSnapshot = loaded.updates.length; + const batches = 10 + Math.floor(random() * 30); + for (let b = 0; b < batches; b += 1) { + let update: Uint8Array = new Uint8Array(); + doc.once("update", (u: Uint8Array) => { + update = u; + }); + const words = `w${day}-${b} `.repeat(30); + doc.getText(TEXT).insert(doc.getText(TEXT).length, words); + acked = library.append(id, "w", { + origin: "local", + update, + title: `Document ${id.slice(0, 4)}`, + // A list-view snippet as long as the real one, so the sidebar + // payload is realistic. + snippet: words.repeat(2).slice(0, 200), + }); + appends += 1; + rowsSinceSnapshot += 1; + clock.now += 1000; + if (rowsSinceSnapshot > 500) { + library.compact(id, "w", Y.encodeStateAsUpdate(doc), acked); + compactions += 1; + rowsSinceSnapshot = 0; + } + } + library.compact(id, "w", Y.encodeStateAsUpdate(doc), acked); + compactions += 1; + library.close(id, "w"); + if (watched.has(id)) watched.set(id, doc.getText(TEXT).toString()); + } + // One new document a day, and one old one trashed (purged after 30 days). + createSeeded(); + const victim = ids.splice( + 5 + Math.floor(random() * (ids.length - 6)), + 1, + )[0]; + if (victim) { + library.trashDoc(victim); + trashed += 1; + } + // Idle: the daily copy and the Trash purge. + clock.now += IDLE_MS; + const idle = library.runIdleJobs(); + if (idle.dailyCopyMs !== null) copyMs.push(idle.dailyCopyMs); + if (idle.dailyCopy === null && idle.dailyCopyMs !== null) + failures.push( + `day ${day}: daily copy failed (${library.stats().lastDailyCopyError})`, + ); + // Restart now and then, sometimes uncleanly. + if (day % 30 === 29) { + if (random() < 0.5) library.shutdown(); + else (library as unknown as { db: { close(): void } }).db.close(); + library = open(); + if ( + library.status.quickCheck !== null && + library.status.quickCheck !== "ok" + ) + failures.push(`day ${day}: quick_check ${library.status.quickCheck}`); + } + if (day % 30 === 0 || day === days - 1) { + const stats = library.stats(); + monthly.push({ + day, + mb: +(stats.bytes / 1048576).toFixed(1), + docs: stats.docs, + pendingRows: stats.pendingRows, + dailyCopies: library.listDailyCopies().length, + }); + } + clock.now = Date.UTC(2026, 0, 2 + day, 9); + } + + const stats = library.stats(); + const dailies = library.listDailyCopies(); + if (dailies.length !== Math.min(DAILY_KEEP, days)) + failures.push(`${dailies.length} daily copies kept, wanted ${DAILY_KEEP}`); + if (stats.pendingRows !== 0) + failures.push(`${stats.pendingRows} rows left uncompacted`); + for (const [id, expected] of watched) { + if (!ids.includes(id)) continue; + const { text } = textOf(library, id); + if (text !== expected) failures.push(`watched document ${id} lost text`); + } + // Growth: the year's text is small next to the seed; the library must not + // keep history it no longer needs. + const growth = stats.bytes / seededBytes; + if (growth > 1.5) + failures.push(`library grew ${growth.toFixed(2)}× in a year`); + const liveDocs = library.sidebar().docs.length; + library.shutdown(); + const simMs = performance.now() - started; + + // Startup with the year-end library: worker ready and sidebar rows. + // `--no-electron` checks the simulation alone (local runs). + const startups: Record[] = []; + for (let i = 0; i < (args["no-electron"] ? 0 : 5); i += 1) { + const run = await runHarness(["--mode=startup", `--dir=${dir}`]); + const line = run.lines.find((l) => l.startsWith("STARTUP ")); + if (!line) { + failures.push(`startup ${i}: ${run.stderr.slice(0, 500)}`); + continue; + } + startups.push(JSON.parse(line.slice(8))); + } + const budget = budgets(); + const median = (key: string) => { + const values = startups.map((s) => Number(s[key])).sort((a, b) => a - b); + return values[Math.floor(values.length / 2)] ?? Number.NaN; + }; + const toSidebar = median("processToSidebarMs"); + const toWorkerReady = median("processToReadyMs"); + if (!args["no-electron"] && !(toSidebar <= budget.sidebarMs * 1.5)) + failures.push( + `sidebar rows ${toSidebar.toFixed(0)} ms after process start, budget ${budget.sidebarMs} ms (× 1.5 CI slack)`, + ); + if (startups.some((s) => s.docs !== liveDocs)) + failures.push( + `startup listed ${startups.map((s) => s.docs)} documents, wanted ${liveDocs}`, + ); + // §10 "After an unclean exit": quick_check runs inside the open. + let unclean: Record | null = null; + if (!args["no-electron"]) { + const db = new Database(join(dir, LIBRARY_FILE)); + db.exec("UPDATE meta SET value = '1' WHERE key = 'unclean_exit'"); + db.close(); + const run = await runHarness(["--mode=startup", `--dir=${dir}`]); + const line = run.lines.find((l) => l.startsWith("STARTUP ")); + if (line) unclean = JSON.parse(line.slice(8)); + if (!unclean?.uncleanExit) + failures.push(`unclean startup: ${line ?? run.stderr.slice(0, 500)}`); + else if (unclean.openMs > 2000) + failures.push( + `quick_check open took ${unclean.openMs.toFixed(0)} ms, budget 2 s`, + ); + } + const sortedCopies = [...copyMs].sort((a, b) => a - b); + const copyMedian = sortedCopies[Math.floor(sortedCopies.length / 2)] ?? 0; + const copyMax = sortedCopies.at(-1) ?? 0; + removeDir(dir); + return { + name: "T8", + pass: failures.length === 0, + summary: `${days} days, ${appends} appends, ${compactions} compactions, ${trashed} trashed; ${(seededBytes / 1048576).toFixed(0)} → ${(stats.bytes / 1048576).toFixed(0)} MB; ${dailies.length} daily copies (VACUUM INTO median ${copyMedian.toFixed(0)} ms, max ${copyMax.toFixed(0)} ms); ${liveDocs} documents: worker ready ${toWorkerReady.toFixed(0)} ms and sidebar rows ${toSidebar.toFixed(0)} ms after process start (median of 5; §10 ${budget.workerReadyMs} / ${budget.sidebarMs} ms; excludes renderer IPC and paint); unclean-exit open with quick_check ${unclean ? unclean.openMs.toFixed(0) : "n/a"} ms`, + details: { + days, + docs: liveDocs, + appends, + compactions, + trashed, + seededBytes, + finalBytes: stats.bytes, + monthly, + startups, + unclean, + budget, + dailyCopyMs: { median: copyMedian, max: copyMax, count: copyMs.length }, + simMs, + }, + failures, + }; +} + +// ── Main ──────────────────────────────────────────────────────────────── + +const CASES: Record Promise> = { + contract: caseContract, + T1: () => killLoop("T1", "append", KILLS), + "T1-dense": () => + killLoop("T1-dense", "append", Math.ceil(KILLS / 2), { dense: true }), + T2: () => killLoop("T2", "compaction", KILLS), + T3: caseT3, + T4: () => caseT4(Number(args.quits ?? 20)), + T5: () => caseT5(Number(args["migration-kills"] ?? 40)), + T6: caseT6, + T8: caseT8, +}; + +const selected = + !args.case || args.case === "all" + ? Object.keys(CASES) + : args.case.split(",").map((name) => name.trim()); + +const results: CaseResult[] = []; +for (const name of selected) { + const run = CASES[name]; + if (!run) throw new Error(`unknown case ${name}`); + const t0 = performance.now(); + let result: CaseResult; + try { + result = await run(); + } catch (error) { + result = { + name, + pass: false, + summary: "crashed", + details: {}, + failures: [String(error instanceof Error ? error.stack : error)], + }; + } + result.details.seconds = Math.round((performance.now() - t0) / 1000); + results.push(result); + writeFileSync(join(OUT, `${name}.json`), JSON.stringify(result, null, 2)); + console.log(`${result.pass ? "PASS" : "FAIL"} ${name}: ${result.summary}`); + for (const failure of result.failures.slice(0, 10)) + console.log(` - ${failure}`); +} + +const table = [ + `### Desktop storage soak (${process.platform}-${process.arch})`, + "", + "| Case | Result | Summary |", + "|---|---|---|", + ...results.map( + (r) => + `| ${r.name} | ${r.pass ? "pass" : "**fail**"} | ${r.summary.replace(/\|/g, "\\|")} |`, + ), + "", +].join("\n"); +if (process.env.GITHUB_STEP_SUMMARY) + appendFileSync(process.env.GITHUB_STEP_SUMMARY, table); +process.exit(results.every((r) => r.pass) ? 0 : 1); diff --git a/tests/unit/ci/import-boundaries.test.ts b/tests/unit/ci/import-boundaries.test.ts new file mode 100644 index 00000000..63e28f0d --- /dev/null +++ b/tests/unit/ci/import-boundaries.test.ts @@ -0,0 +1,129 @@ +/** + * Import boundaries (desktop architecture §3.3). Imports are read with + * `Bun.Transpiler().scanImports()` (type-only imports are dropped, as at + * runtime) and followed transitively through relative and `@/` paths. + * + * Rows checked so far: the storage engine, the storage worker, and + * `lib/platform/desktop`. P2.5 adds the main, preload and renderer rows. + */ +import { describe, expect, test } from "bun:test"; +import { existsSync, readdirSync, readFileSync, statSync } from "node:fs"; +import { dirname, join, relative, resolve } from "node:path"; + +const ROOT = resolve(import.meta.dir, "../../.."); +const transpilers = { + ts: new Bun.Transpiler({ loader: "ts" }), + tsx: new Bun.Transpiler({ loader: "tsx" }), +}; + +function filesUnder(dir: string): string[] { + const out: string[] = []; + for (const entry of readdirSync(dir)) { + const path = join(dir, entry); + if (statSync(path).isDirectory()) out.push(...filesUnder(path)); + else if (/\.tsx?$/.test(entry) && !entry.endsWith(".d.ts")) out.push(path); + } + return out; +} + +function resolveLocal(from: string, specifier: string): string | null { + let base: string; + if (specifier.startsWith("@/")) base = join(ROOT, specifier.slice(2)); + else if (specifier.startsWith(".")) base = resolve(dirname(from), specifier); + else return null; + for (const candidate of [ + base, + `${base}.ts`, + `${base}.tsx`, + join(base, "index.ts"), + join(base, "index.tsx"), + ]) + if (existsSync(candidate) && statSync(candidate).isFile()) return candidate; + throw new Error(`cannot resolve ${specifier} from ${relative(ROOT, from)}`); +} + +/** Every module `entry` reaches at runtime: local files and bare specifiers. */ +function reach(entries: string[]): { + files: Set; + bare: Map; +} { + const files = new Set(); + const bare = new Map(); + const queue = [...entries]; + while (queue.length > 0) { + const file = queue.pop() as string; + if (files.has(file)) continue; + files.add(file); + const transpiler = file.endsWith(".tsx") ? transpilers.tsx : transpilers.ts; + for (const { path } of transpiler.scanImports(readFileSync(file, "utf8"))) { + const local = resolveLocal(file, path); + if (local) queue.push(local); + else if (!bare.has(path)) bare.set(path, relative(ROOT, file)); + } + } + return { files, bare }; +} + +const rel = (path: string) => relative(ROOT, path).split("\\").join("/"); + +describe("desktop/storage/engine", () => { + const { files, bare } = reach( + filesUnder(join(ROOT, "desktop/storage/engine")), + ); + + test("imports no electron, node:*, bun:*, React or Tiptap", () => { + const banned = [...bare.entries()].filter( + ([spec]) => + /^(electron|node:|bun:|react|react-dom|@tiptap\/)/.test(spec) || + /^(fs|path|os|worker_threads|child_process)$/.test(spec), + ); + expect(banned).toEqual([]); + }); + + test("stays inside desktop/storage/engine", () => { + const outside = [...files] + .map(rel) + .filter((file) => !file.startsWith("desktop/storage/engine/")); + expect(outside).toEqual([]); + }); +}); + +describe("desktop/storage", () => { + // Architecture §3.3: main and storage may reach only these parts of lib/. + const LIB_ALLOWED = [ + /^lib\/security\/content-security-policy\.ts$/, + /^lib\/doc-migrations\//, + /^lib\/prefs\/schema\.ts$/, + /^lib\/comments\/validate\.ts$/, + /^lib\/assets\//, + // Pure name lists the import guard checks against (amended in §3.3). + /^lib\/schema-names\.ts$/, + ]; + + test("reaches lib/ only through the allowlist, and never React", () => { + const { files, bare } = reach(filesUnder(join(ROOT, "desktop/storage"))); + const libFiles = [...files] + .map(rel) + .filter((file) => file.startsWith("lib/")) + .filter((file) => !LIB_ALLOWED.some((pattern) => pattern.test(file))); + expect(libFiles).toEqual([]); + expect([...bare.keys()].filter((spec) => /^react/.test(spec))).toEqual([]); + }); + + test("never imports bun:* (it runs inside Electron)", () => { + const { bare } = reach(filesUnder(join(ROOT, "desktop/storage"))); + expect([...bare.keys()].filter((spec) => spec.startsWith("bun:"))).toEqual( + [], + ); + }); +}); + +describe("lib/platform/desktop", () => { + test("never imports desktop/main, preload or storage", () => { + const { files } = reach(filesUnder(join(ROOT, "lib/platform/desktop"))); + const crossing = [...files] + .map(rel) + .filter((file) => /^desktop\/(main|preload|storage)\//.test(file)); + expect(crossing).toEqual([]); + }); +}); diff --git a/tests/unit/desktop/storage/cloud-folder-and-codes.test.ts b/tests/unit/desktop/storage/cloud-folder-and-codes.test.ts new file mode 100644 index 00000000..7b522ba0 --- /dev/null +++ b/tests/unit/desktop/storage/cloud-folder-and-codes.test.ts @@ -0,0 +1,117 @@ +import { describe, expect, test } from "bun:test"; + +import { cloudFolderOf } from "@/desktop/storage/engine/cloud-folder"; +import { STORED_ORIGINS } from "@/desktop/storage/engine/content"; +import { storageErrorFrom } from "@/desktop/storage/engine/driver"; +import { STORAGE_ERROR_CODES } from "@/desktop/storage/engine/errors"; +import { + DOC_STORE_ERROR_CODES, + UPDATE_ORIGINS, +} from "@/lib/platform/doc-store"; + +const host = ( + platform: string, + env: Record = {}, + homeDir = "", +) => ({ + platform, + env, + homeDir, + realpath: (path: string) => path, +}); + +describe("cloudFolderOf", () => { + test("a user name that looks like a provider is not a sync folder", () => { + expect( + cloudFolderOf( + "/home/box/.config/TinyDocy", + host("linux", {}, "/home/box"), + ), + ).toBeNull(); + expect( + cloudFolderOf( + "/Users/dropbox/Library/Application Support/TinyDocy", + host("darwin", {}, "/Users/dropbox"), + ), + ).toBeNull(); + expect( + cloudFolderOf("/home/box/Dropbox/T", host("linux", {}, "/home/box")), + ).toBe("dropbox"); + expect( + cloudFolderOf("D:\\Dropbox\\TinyDocy", host("win32", {}, "C:\\Users\\a")), + ).toBe("dropbox"); + }); + + const cases: [string, string, Record, string | null][] = [ + ["darwin", "/Users/a/Library/Application Support/TinyDocy", {}, null], + [ + "darwin", + "/Users/a/Library/Mobile Documents/com~apple~CloudDocs/T", + {}, + "icloud", + ], + [ + "darwin", + "/Users/a/Library/CloudStorage/OneDrive-Contoso/T", + {}, + "onedrive", + ], + [ + "darwin", + "/Users/a/Library/CloudStorage/GoogleDrive-a@b.c/My Drive/T", + {}, + "google_drive", + ], + ["darwin", "/Users/a/Library/CloudStorage/SomethingNew/T", {}, "other"], + ["darwin", "/Users/a/Dropbox/T", {}, "dropbox"], + ["linux", "/home/a/.config/TinyDocy", {}, null], + ["linux", "/home/a/Dropbox (Personal)/T", {}, "dropbox"], + ["win32", "C:\\Users\\a\\AppData\\Local\\TinyDocy", {}, null], + [ + "win32", + "C:\\Users\\a\\Work Files\\TinyDocy", + { OneDriveCommercial: "c:\\users\\a\\work files" }, + "onedrive", + ], + ["win32", "C:\\Users\\a\\OneDrive - Contoso\\T", {}, "onedrive"], + ["win32", "C:\\Users\\a\\iCloudDrive\\T", {}, "icloud"], + ["win32", "D:\\OneDriveBackup\\T", {}, null], + ]; + for (const [platform, path, env, expected] of cases) + test(`${platform} ${path} → ${expected}`, () => { + expect(cloudFolderOf(path, host(platform, env))).toBe(expected as never); + }); +}); + +describe("codes and origins stay in step with the DocStore port", () => { + test("every DocStore code but the web library's is a storage code", () => { + const storage = new Set(STORAGE_ERROR_CODES); + const missing = DOC_STORE_ERROR_CODES.filter( + (code) => + !storage.has(code) && + code !== "quota_exceeded" && + code !== "build_mismatch", + ); + expect(missing).toEqual([]); + }); + + test("stored origins match UPDATE_ORIGINS", () => { + expect([...STORED_ORIGINS]).toEqual([...UPDATE_ORIGINS]); + }); +}); + +describe("storageErrorFrom", () => { + const nodeError = (errcode: number) => + Object.assign(new Error("x"), { errcode }); + test.each([ + [13, "disk_full"], + [11, "corrupt"], + [26, "corrupt"], + [19 | (8 << 8), "invalid_argument"], + [5, "locked_elsewhere"], + [8, "library_read_only"], + [10 | (12 << 8), "io_error"], + ])("SQLite %i → %s", (code, expected) => { + expect(storageErrorFrom(nodeError(code)).code).toBe(expected as never); + }); +}); diff --git a/tests/unit/desktop/storage/content.test.ts b/tests/unit/desktop/storage/content.test.ts new file mode 100644 index 00000000..8295eac7 --- /dev/null +++ b/tests/unit/desktop/storage/content.test.ts @@ -0,0 +1,416 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; + +import * as Y from "yjs"; + +import { StorageError } from "@/desktop/storage/engine/errors"; +import type { Library } from "@/desktop/storage/engine/library"; +import { + crash, + createClock, + createTempDir, + createTestHost, + openTestLibrary, + type TestClock, +} from "@/tests/helpers/storage-library"; + +let temp: ReturnType; +let clock: TestClock; +let library: Library; +let docId: string; + +beforeEach(() => { + temp = createTempDir(); + clock = createClock(); + library = openTestLibrary(temp.dir, createTestHost(clock)); + docId = library.createDoc({ title: "Untitled" }).id; + library.load(docId, "w1"); +}); +afterEach(() => { + library.shutdown(); + temp.cleanUp(); +}); + +function edit(doc: Y.Doc, text: string): Uint8Array { + let update: Uint8Array = new Uint8Array(); + const capture = (u: Uint8Array) => { + update = u; + }; + doc.once("update", capture); + doc.getText("t").insert(doc.getText("t").length, text); + return update; +} + +function textOf(loaded: { + snapshot: Uint8Array | null; + updates: Uint8Array[]; +}) { + const doc = new Y.Doc(); + if (loaded.snapshot) Y.applyUpdate(doc, loaded.snapshot); + for (const update of loaded.updates) Y.applyUpdate(doc, update); + return doc.getText("t").toString(); +} + +function codeOf(run: () => unknown): string | undefined { + try { + run(); + } catch (error) { + if (error instanceof StorageError) return error.code; + throw error; + } + return undefined; +} + +describe("load and append", () => { + test("a new document loads empty at schema 0", () => { + expect(library.load(docId, "w1")).toEqual({ + snapshot: null, + updates: [], + maxSeq: 0, + schema: 0, + }); + }); + + test("appends come back in order with increasing seqs", () => { + const doc = new Y.Doc(); + library.load(docId, "w1"); + const a = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "a"), + }); + const b = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "b"), + }); + expect(b).toBeGreaterThan(a); + const loaded = library.load(docId, "w1"); + expect(loaded.updates).toHaveLength(2); + expect(loaded.maxSeq).toBe(b); + expect(textOf(loaded)).toBe("ab"); + }); + + test("seqs are global across documents", () => { + const other = library.createDoc().id; + library.load(other, "w2"); + const doc = new Y.Doc(); + const a = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "a"), + }); + const b = library.append(other, "w2", { + origin: "local", + update: edit(doc, "b"), + }); + expect(b).toBe(a + 1); + }); + + test("an append updates the title, snippet, schema, size and modified time", () => { + clock.advance(5000); + const update = edit(new Y.Doc(), "hello"); + library.append(docId, "w1", { + origin: "migration", + update, + title: "Hello", + snippet: "hello world", + schemaTo: 2, + }); + const row = library.sidebar().docs[0]; + expect(row?.title).toBe("Hello"); + expect(row?.snippet).toBe("hello world"); + expect(row?.modifiedAt).toBe(clock.now); + expect(row?.sizeBytes).toBe(update.byteLength); + expect(library.load(docId, "w1").schema).toBe(2); + }); + + test("an append without a title keeps the old one; titles are capped", () => { + library.append(docId, "w1", { + origin: "local", + update: edit(new Y.Doc(), "x"), + title: "T".repeat(500), + }); + library.append(docId, "w1", { + origin: "local", + update: edit(new Y.Doc(), "y"), + }); + expect(library.sidebar().docs[0]?.title).toHaveLength(200); + }); + + test("appending the same bytes twice (a resend) changes nothing", () => { + const doc = new Y.Doc(); + const update = edit(doc, "once"); + library.append(docId, "w1", { origin: "local", update }); + library.append(docId, "w1", { origin: "local", update }); + expect(textOf(library.load(docId, "w1"))).toBe("once"); + }); + + test("refuses bad input and unknown documents", () => { + const update = edit(new Y.Doc(), "x"); + expect( + codeOf(() => + library.append(docId, "w1", { origin: "bogus" as "local", update }), + ), + ).toBe("invalid_argument"); + expect( + codeOf(() => + library.append(docId, "w1", { + origin: "local", + update: new Uint8Array(), + }), + ), + ).toBe("invalid_argument"); + expect( + codeOf(() => + library.append(crypto.randomUUID(), "w1", { origin: "local", update }), + ), + ).toBe("forbidden"); + expect(codeOf(() => library.load(crypto.randomUUID(), "w1"))).toBe( + "not_found", + ); + }); + + test("a failed append writes nothing", () => { + const update = edit(new Y.Doc(), "x"); + codeOf(() => + library.append(crypto.randomUUID(), "w1", { origin: "local", update }), + ); + expect(library.stats().pendingRows).toBe(0); + }); +}); + +describe("compaction (D-A3)", () => { + test("stores the snapshot and deletes the rows it covers", () => { + const doc = new Y.Doc(); + library.load(docId, "w1"); + library.append(docId, "w1", { origin: "local", update: edit(doc, "a") }); + const covered = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "b"), + }); + const state = Y.encodeStateAsUpdate(doc); + const late = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "c"), + }); + expect(library.compact(docId, "w1", state, covered)).toEqual({ + foreign: null, + }); + const loaded = library.load(docId, "w1"); + expect(loaded.updates).toHaveLength(1); + expect(loaded.maxSeq).toBe(late); + expect(textOf(loaded)).toBe("abc"); + expect(library.sidebar().docs[0]?.sizeBytes).toBe( + state.byteLength + (loaded.updates[0]?.byteLength ?? 0), + ); + }); + + test("maxSeq is the snapshot's when every row is folded in", () => { + const doc = new Y.Doc(); + const seq = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "a"), + }); + library.compact(docId, "w1", Y.encodeStateAsUpdate(doc), seq); + expect(library.load(docId, "w1")).toMatchObject({ + updates: [], + maxSeq: seq, + }); + }); + + test("a seq is never reused after compaction deletes the newest rows", () => { + const doc = new Y.Doc(); + const seq = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "a"), + }); + library.compact(docId, "w1", Y.encodeStateAsUpdate(doc), seq); + library.shutdown(); + library = openTestLibrary(temp.dir, createTestHost(clock)); + library.load(docId, "w1"); + const next = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "b"), + }); + expect(next).toBeGreaterThan(seq); + expect(textOf(library.load(docId, "w1"))).toBe("ab"); + }); + + test("a stale compaction never moves the snapshot back", () => { + const doc = new Y.Doc(); + const first = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "a"), + }); + const oldState = Y.encodeStateAsUpdate(doc); + const second = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "b"), + }); + library.compact(docId, "w1", Y.encodeStateAsUpdate(doc), second); + library.compact(docId, "w1", oldState, first); + expect(textOf(library.load(docId, "w1"))).toBe("ab"); + }); + + test("refuses an upToSeq past the document's last row", () => { + const doc = new Y.Doc(); + const seq = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "a"), + }); + expect( + codeOf(() => + library.compact(docId, "w1", Y.encodeStateAsUpdate(doc), seq + 1), + ), + ).toBe("invalid_argument"); + }); + + test("stateOf merges the snapshot and rows", () => { + const doc = new Y.Doc(); + const seq = library.append(docId, "w1", { + origin: "local", + update: edit(doc, "a"), + }); + library.compact(docId, "w1", Y.encodeStateAsUpdate(doc), seq); + library.append(docId, "w1", { origin: "local", update: edit(doc, "b") }); + const copy = new Y.Doc(); + Y.applyUpdate(copy, library.stateOf(docId)); + expect(copy.getText("t").toString()).toBe("ab"); + }); +}); + +describe("one window per document (registry)", () => { + test("a second window gets already_open; the first keeps working", () => { + library.load(docId, "w1"); + expect(codeOf(() => library.load(docId, "w2"))).toBe("already_open"); + expect(library.load(docId, "w1").schema).toBe(0); + }); + + test("another window can't append or compact an open document", () => { + library.load(docId, "w1"); + const update = edit(new Y.Doc(), "x"); + expect( + codeOf(() => library.append(docId, "w2", { origin: "local", update })), + ).toBe("forbidden"); + expect(codeOf(() => library.compact(docId, "w2", update, 0))).toBe( + "forbidden", + ); + }); + + test("close and a closed window release the document", () => { + library.load(docId, "w1"); + library.close(docId, "w1"); + expect(library.load(docId, "w2").schema).toBe(0); + library.releaseOwner("w2"); + expect(library.isOpen(docId)).toBe(false); + }); + + test("closing from another window does nothing", () => { + library.load(docId, "w1"); + library.close(docId, "w2"); + expect(library.isOpen(docId)).toBe(true); + }); + + test("main's content writes refuse an open document", () => { + library.load(docId, "w1"); + expect(codeOf(() => library.purgeDoc(docId))).toBe("document_open"); + library.close(docId, "w1"); + library.purgeDoc(docId); + expect(library.sidebar().docs).toEqual([]); + }); +}); + +describe("compaction property (architecture §11)", () => { + // A seeded PRNG keeps failures reproducible. + function mulberry32(seed: number) { + let a = seed; + return () => { + a = (a + 0x6d2b79f5) | 0; + let t = Math.imul(a ^ (a >>> 15), 1 | a); + t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t; + return ((t ^ (t >>> 14)) >>> 0) / 4294967296; + }; + } + + for (const seed of [1, 2, 3, 4, 5]) { + test(`random appends, compactions, crashes and reopens (seed ${seed})`, () => { + const random = mulberry32(seed); + const reference = new Y.Doc(); + // The renderer's copy: rebuilt from storage on every reopen. + let session = new Y.Doc(); + let ackedSeq = 0; + const hydrate = () => { + const loaded = library.load(docId, "w1"); + session = new Y.Doc(); + if (loaded.snapshot) Y.applyUpdate(session, loaded.snapshot); + for (const u of loaded.updates) Y.applyUpdate(session, u); + ackedSeq = loaded.maxSeq; + }; + hydrate(); + for (let step = 0; step < 300; step += 1) { + const roll = random(); + if (roll < 0.7) { + const text = session.getText("t"); + const pos = Math.floor(random() * (text.length + 1)); + let update: Uint8Array = new Uint8Array(); + session.once("update", (u: Uint8Array) => { + update = u; + }); + if (text.length > 0 && random() < 0.25) + text.delete(Math.min(pos, text.length - 1), 1); + else + text.insert( + pos, + String.fromCharCode(97 + Math.floor(random() * 26)), + ); + Y.applyUpdate(reference, update); + ackedSeq = library.append(docId, "w1", { origin: "local", update }); + } else if (roll < 0.85) { + library.compact( + docId, + "w1", + Y.encodeStateAsUpdate(session), + ackedSeq, + ); + } else if (roll < 0.95) { + library.close(docId, "w1"); + library.shutdown(); + library = openTestLibrary(temp.dir, createTestHost(clock)); + hydrate(); + } else { + crash(library); + library = openTestLibrary(temp.dir, createTestHost(clock)); + hydrate(); + } + if (step % 25 === 0 || roll >= 0.85) { + const stored = library.load(docId, "w1"); + const rebuilt = new Y.Doc(); + if (stored.snapshot) Y.applyUpdate(rebuilt, stored.snapshot); + for (const u of stored.updates) Y.applyUpdate(rebuilt, u); + expect(rebuilt.getText("t").toString()).toBe( + reference.getText("t").toString(), + ); + expect(Y.encodeStateVector(rebuilt)).toEqual( + Y.encodeStateVector(reference), + ); + } + } + }); + } +}); + +describe("claim (registry replay after a worker restart)", () => { + test("lets the holding window write again without a load", () => { + library.shutdown(); + library = openTestLibrary(temp.dir, createTestHost(clock)); + const update = edit(new Y.Doc(), "x"); + expect( + codeOf(() => library.append(docId, "w1", { origin: "local", update })), + ).toBe("forbidden"); + library.claim(docId, "w1"); + expect( + library.append(docId, "w1", { origin: "local", update }), + ).toBeGreaterThan(0); + expect(codeOf(() => library.claim(docId, "w2"))).toBe("already_open"); + expect(codeOf(() => library.claim(crypto.randomUUID(), "w2"))).toBe( + "not_found", + ); + }); +}); diff --git a/tests/unit/desktop/storage/daily-copy.test.ts b/tests/unit/desktop/storage/daily-copy.test.ts new file mode 100644 index 00000000..fc4ba1a3 --- /dev/null +++ b/tests/unit/desktop/storage/daily-copy.test.ts @@ -0,0 +1,98 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; + +import * as Y from "yjs"; + +import { DAILY_KEEP, localDate } from "@/desktop/storage/engine/daily-copy"; +import { IDLE_MS, type Library } from "@/desktop/storage/engine/library"; +import { + createClock, + createTempDir, + createTestHost, + openTestLibrary, + type TestClock, +} from "@/tests/helpers/storage-library"; + +const DAY = 24 * 60 * 60 * 1000; + +let temp: ReturnType; +let clock: TestClock; +let library: Library; + +beforeEach(() => { + temp = createTempDir(); + clock = createClock(); + library = openTestLibrary(temp.dir, createTestHost(clock)); +}); +afterEach(() => { + library.shutdown(); + temp.cleanUp(); +}); + +function write(docId: string, text: string) { + if (!library.isOpen(docId)) library.load(docId, "w1"); + const doc = new Y.Doc(); + doc.getText("t").insert(0, text); + library.append(docId, "w1", { + origin: "local", + update: Y.encodeStateAsUpdate(doc), + }); +} + +describe("daily copies (D-A6)", () => { + test("waits for 30 s without a write", () => { + write(library.createDoc().id, "x"); + clock.advance(IDLE_MS - 1); + expect(library.runIdleJobs().dailyCopy).toBeNull(); + clock.advance(1); + expect(library.runIdleJobs().dailyCopy).toBe( + `library-${localDate(clock.now)}.sqlite`, + ); + }); + + test("takes one copy a day and keeps the newest 7", () => { + const docId = library.createDoc().id; + const taken: string[] = []; + for (let day = 0; day < 10; day += 1) { + write(docId, `day ${day}`); + clock.advance(IDLE_MS); + const first = library.runIdleJobs().dailyCopy; + expect(first).not.toBeNull(); + taken.push(localDate(clock.now)); + expect(library.runIdleJobs().dailyCopy).toBeNull(); + clock.advance(DAY); + } + expect(library.listDailyCopies()).toEqual(taken.slice(-DAILY_KEEP)); + }); + + test("a copy restores one document's state as of that day", () => { + const docId = library.createDoc().id; + write(docId, "before"); + clock.advance(IDLE_MS); + library.runIdleJobs(); + const day = localDate(clock.now); + write(docId, "after"); + const restored = new Y.Doc(); + Y.applyUpdate(restored, library.readDailyCopy(day, docId)); + expect(restored.getText("t").toString()).toBe("before"); + }); + + test("survives a leftover partial copy", () => { + const host = createTestHost(clock); + host.mkdirp(host.join(temp.dir, "daily")); + const partial = host.join( + temp.dir, + "daily", + `library-${localDate(clock.now)}.sqlite.partial`, + ); + host.createExclusive(partial, "junk"); + clock.advance(IDLE_MS); + expect(library.runIdleJobs().dailyCopy).not.toBeNull(); + expect(host.exists(partial)).toBe(false); + }); + + test("refuses a malformed date", () => { + expect(() => + library.readDailyCopy("../../etc", crypto.randomUUID()), + ).toThrow(); + }); +}); diff --git a/tests/unit/desktop/storage/doc-store-contract.test.ts b/tests/unit/desktop/storage/doc-store-contract.test.ts new file mode 100644 index 00000000..89f4142c --- /dev/null +++ b/tests/unit/desktop/storage/doc-store-contract.test.ts @@ -0,0 +1,43 @@ +/** + * P2.1's shared `DocStore` contract (architecture §11) on the storage + * engine over `bun:sqlite`, through the same dispatch the worker uses. The + * Electron soak's `contract` case runs it again on `node:sqlite` inside the + * real worker. + */ +import { describe, it } from "bun:test"; + +import type { Library } from "@/desktop/storage/engine/library"; +import { defineDocStoreContract } from "@/tests/helpers/doc-store-contract"; +import { createEngineDocStore } from "@/tests/helpers/engine-doc-store"; +import { + createClock, + createTempDir, + createTestHost, + openTestLibrary, +} from "@/tests/helpers/storage-library"; + +defineDocStoreContract( + { describe, it }, + "DocStore contract on the engine (bun:sqlite)", + { + exclusiveOpen: true, + async createLibrary() { + const temp = createTempDir(); + const library: Library = openTestLibrary( + temp.dir, + createTestHost(createClock()), + ); + let clients = 0; + return { + connect: () => createEngineDocStore(library, `client-${++clients}`), + async createDoc(docId, schema) { + library.createDoc({ id: docId, schema }); + }, + async dispose() { + library.shutdown(); + temp.cleanUp(); + }, + }; + }, + }, +); diff --git a/tests/unit/desktop/storage/doc-store-engine.test.ts b/tests/unit/desktop/storage/doc-store-engine.test.ts new file mode 100644 index 00000000..5c1840b6 --- /dev/null +++ b/tests/unit/desktop/storage/doc-store-engine.test.ts @@ -0,0 +1,86 @@ +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; + +import * as Y from "yjs"; + +import { IDLE_MS, type Library } from "@/desktop/storage/engine/library"; +import { TRASH_RETENTION_MS } from "@/desktop/storage/engine/library-index"; +import { dispatch } from "@/desktop/storage/protocol"; +import { + createClock, + createTempDir, + createTestHost, + openTestLibrary, +} from "@/tests/helpers/storage-library"; + +let temp: ReturnType; +let library: Library; + +beforeEach(() => { + temp = createTempDir(); + library = openTestLibrary(temp.dir, createTestHost(createClock())); +}); +afterEach(() => { + library.shutdown(); + temp.cleanUp(); +}); + +describe("dispatch", () => { + test("a failure comes back as an envelope, never a throw", () => { + expect( + dispatch(library, { + id: 2, + op: "load", + args: { docId: "nope", owner: "w1" }, + }), + ).toMatchObject({ + ok: false, + error: { code: "not_found", retryable: false }, + }); + }); +}); + +describe("the index change feed", () => { + test("an append that changes the title moves rev and names the document", () => { + const { id } = library.createDoc(); + library.load(id, "w1"); + library.takeIndexChanges(); + const before = library.sidebar().rev; + const doc = new Y.Doc(); + doc.getText("t").insert(0, "x"); + library.append(id, "w1", { + origin: "local", + update: Y.encodeStateAsUpdate(doc), + title: "New title", + }); + expect(library.takeIndexChanges()).toEqual({ + rev: before + 1, + changes: [{ kind: "doc", id }], + }); + expect(library.takeIndexChanges()).toBeNull(); + }); + + test("index writes report each changed row once", () => { + const folder = library.createFolder({ name: "A" }); + const { id } = library.createDoc({ folderId: folder.id }); + library.renameDoc(id, "x"); + library.moveDoc(id, null, "a"); + const tag = library.createTag("t"); + expect(library.takeIndexChanges()?.changes).toEqual([ + { kind: "folder", id: folder.id }, + { kind: "doc", id }, + { kind: "tag", id: tag.id }, + ]); + }); + + test("the idle Trash purge is announced", () => { + const clock = createClock(); + library.shutdown(); + library = openTestLibrary(temp.dir, createTestHost(clock)); + const { id } = library.createDoc(); + library.trashDoc(id); + library.takeIndexChanges(); + clock.advance(TRASH_RETENTION_MS + IDLE_MS); + expect(library.runIdleJobs().purged).toBe(1); + expect(library.takeIndexChanges()?.changes).toEqual([{ kind: "doc", id }]); + }); +}); diff --git a/tests/unit/desktop/storage/library-index.test.ts b/tests/unit/desktop/storage/library-index.test.ts new file mode 100644 index 00000000..0947972d --- /dev/null +++ b/tests/unit/desktop/storage/library-index.test.ts @@ -0,0 +1,219 @@ +import { Database } from "bun:sqlite"; +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { join } from "node:path"; + +import * as Y from "yjs"; + +import { StorageError } from "@/desktop/storage/engine/errors"; +import { + IDLE_MS, + LIBRARY_FILE, + type Library, +} from "@/desktop/storage/engine/library"; +import { TRASH_RETENTION_MS } from "@/desktop/storage/engine/library-index"; +import { + createClock, + createTempDir, + createTestHost, + openTestLibrary, + type TestClock, +} from "@/tests/helpers/storage-library"; + +let temp: ReturnType; +let clock: TestClock; +let library: Library; + +beforeEach(() => { + temp = createTempDir(); + clock = createClock(); + library = openTestLibrary(temp.dir, createTestHost(clock)); +}); +afterEach(() => { + library.shutdown(); + temp.cleanUp(); +}); + +function codeOf(run: () => unknown) { + try { + run(); + } catch (error) { + if (error instanceof StorageError) return error.code; + throw error; + } + return undefined; +} + +describe("creating documents", () => { + test("mints a UUID, a lineage and a 32-hex link key at insert", () => { + const a = library.createDoc(); + const b = library.createDoc(); + expect(a.id).toMatch(/^[0-9a-f-]{36}$/); + expect(a.lineage).toMatch(/^[0-9a-f-]{36}$/); + expect(a.linkKey).toMatch(/^[0-9a-f]{32}$/); + expect(a.lineage).not.toBe(b.lineage); + expect(a.linkKey).not.toBe(b.linkKey); + }); + + test("keeps a lineage and link key only when given (backup restore)", () => { + const lineage = crypto.randomUUID(); + const linkKey = "a".repeat(32); + expect(library.createDoc({ lineage, linkKey })).toMatchObject({ + lineage, + linkKey, + }); + expect(codeOf(() => library.createDoc({ linkKey }))).toBe( + "invalid_argument", + ); + }); + + test("refuses malformed ids, link keys and kinds", () => { + expect(codeOf(() => library.createDoc({ id: "../x" }))).toBe( + "invalid_argument", + ); + expect(codeOf(() => library.createDoc({ linkKey: "XYZ" }))).toBe( + "invalid_argument", + ); + expect(codeOf(() => library.createDoc({ kind: "page" as "doc" }))).toBe( + "invalid_argument", + ); + }); + + test("templates are stored with kind 'template'", () => { + library.createDoc({ kind: "template", title: "Weekly" }); + expect(library.sidebar().docs[0]?.kind).toBe("template"); + }); +}); + +describe("the sidebar snapshot", () => { + test("lists live folders, documents and tags; rev moves on every write", () => { + const start = library.sidebar().rev; + const folder = library.createFolder({ name: "Work" }); + const doc = library.createDoc({ title: "Plan", folderId: folder.id }); + const tag = library.createTag("urgent", "red"); + library.setDocTags(doc.id, [tag.id]); + const snapshot = library.sidebar(); + expect(snapshot.rev).toBe(start + 4); + expect(snapshot.folders.map((f) => f.name)).toEqual(["Work"]); + expect(snapshot.docs[0]).toMatchObject({ + id: doc.id, + title: "Plan", + folderId: folder.id, + tagIds: [tag.id], + pinned: false, + kind: "doc", + }); + expect(snapshot.tags).toEqual([ + { id: tag.id, name: "urgent", color: "red" }, + ]); + }); + + test("tag names are unique regardless of case", () => { + library.createTag("Urgent"); + expect(codeOf(() => library.createTag("urgent"))).toBe("invalid_argument"); + }); + + test("opening a document records opened_at", () => { + const doc = library.createDoc(); + clock.advance(1000); + library.load(doc.id, "w1"); + expect(library.sidebar().docs[0]?.openedAt).toBe(clock.now); + }); + + test("rename and move", () => { + const folder = library.createFolder({ name: "A" }); + const doc = library.createDoc({ title: "x" }); + library.renameDoc(doc.id, " Renamed "); + library.moveDoc(doc.id, folder.id, "m"); + expect(library.sidebar().docs[0]).toMatchObject({ + title: "Renamed", + folderId: folder.id, + sortKey: "m", + }); + expect(codeOf(() => library.renameDoc(crypto.randomUUID(), "y"))).toBe( + "not_found", + ); + }); +}); + +describe("Trash and purge", () => { + test("trash hides a document and restore brings it back to its folder", () => { + const folder = library.createFolder({ name: "A" }); + const doc = library.createDoc({ folderId: folder.id }); + library.trashDoc(doc.id); + expect(library.sidebar().docs).toEqual([]); + library.restoreDoc(doc.id); + expect(library.sidebar().docs[0]?.folderId).toBe(folder.id); + }); + + test("purge cascades to rows, snapshot, tags, view and import source", () => { + const doc = library.createDoc(); + const tag = library.createTag("t"); + library.setDocTags(doc.id, [tag.id]); + const ydoc = new Y.Doc(); + ydoc.getText("t").insert(0, "x"); + const update = Y.encodeStateAsUpdate(ydoc); + library.load(doc.id, "w1"); + const seq = library.append(doc.id, "w1", { origin: "local", update }); + library.compact(doc.id, "w1", update, seq); + library.append(doc.id, "w1", { origin: "local", update }); + library.shutdown(); + const raw = new Database(join(temp.dir, LIBRARY_FILE)); + raw.exec("PRAGMA foreign_keys = ON"); + raw + .prepare("INSERT INTO doc_view (doc_id, width) VALUES (?, 'wide')") + .run(doc.id); + raw + .prepare( + "INSERT INTO import_sources (doc_id, source_hash) VALUES (?, 'h')", + ) + .run(doc.id); + raw.close(); + library = openTestLibrary(temp.dir, createTestHost(clock)); + library.purgeDoc(doc.id); + library.shutdown(); + const check = new Database(join(temp.dir, LIBRARY_FILE)); + for (const table of [ + "updates", + "snapshots", + "doc_tags", + "doc_view", + "import_sources", + ]) { + const row = check.query(`SELECT COUNT(*) AS n FROM ${table}`).get() as { + n: number; + }; + expect(`${table}=${row.n}`).toBe(`${table}=0`); + } + check.close(); + library = openTestLibrary(temp.dir, createTestHost(clock)); + }); + + test("idle jobs empty Trash after 30 days, skipping open documents", () => { + const old = library.createDoc(); + const open = library.createDoc(); + const recent = library.createDoc(); + library.trashDoc(old.id); + library.trashDoc(open.id); + clock.advance(TRASH_RETENTION_MS - 1000); + library.trashDoc(recent.id); + library.load(open.id, "w1"); + clock.advance(IDLE_MS + 1000); + expect(library.runIdleJobs().purged).toBe(1); + expect(library.stats().docs).toBe(2); + }); +}); + +describe("ids and schema at creation", () => { + test("accepts any safe id and stores the given schema", () => { + const { id } = library.createDoc({ id: "doc-a_1", schema: 3 }); + expect(library.load(id, "w1").schema).toBe(3); + }); + + test("refuses ids that aren't safe in SQL, paths or logs", () => { + for (const id of ["", "../x", "a b", "a/b", "-x", "x".repeat(65)]) + expect(codeOf(() => library.createDoc({ id }))).toBe("invalid_argument"); + expect(codeOf(() => library.createDoc({ schema: -1 }))).toBe( + "invalid_argument", + ); + }); +}); diff --git a/tests/unit/desktop/storage/library-open.test.ts b/tests/unit/desktop/storage/library-open.test.ts new file mode 100644 index 00000000..42366f65 --- /dev/null +++ b/tests/unit/desktop/storage/library-open.test.ts @@ -0,0 +1,365 @@ +import { Database } from "bun:sqlite"; +import { afterEach, beforeEach, describe, expect, test } from "bun:test"; +import { readFileSync, writeFileSync } from "node:fs"; +import { join } from "node:path"; + +import { StorageError } from "@/desktop/storage/engine/errors"; +import { LIBRARY_FILE } from "@/desktop/storage/engine/library"; +import { LOCK_FILE } from "@/desktop/storage/engine/lock"; +import type { FormatStep } from "@/desktop/storage/engine/migrations"; +import { TABLES_V1 } from "@/desktop/storage/engine/schema"; +import { + crash, + createClock, + createTempDir, + createTestHost, + openTestLibrary, + type TestClock, +} from "@/tests/helpers/storage-library"; + +let temp: ReturnType; +let clock: TestClock; + +beforeEach(() => { + temp = createTempDir(); + clock = createClock(); +}); +afterEach(() => temp.cleanUp()); + +function raw(dir = temp.dir) { + return new Database(join(dir, LIBRARY_FILE)); +} + +function rejectsWith(code: string, run: () => unknown) { + let caught: unknown; + try { + run(); + } catch (error) { + caught = error; + } + expect(caught).toBeInstanceOf(StorageError); + expect((caught as StorageError).code).toBe(code as StorageError["code"]); +} + +describe("creating a library", () => { + test("creates every format-1 table and the meta rows", () => { + const library = openTestLibrary(temp.dir, createTestHost(clock)); + library.shutdown(); + const db = raw(); + const tables = db + .query( + "SELECT name FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%' ORDER BY name", + ) + .all() + .map((row) => (row as { name: string }).name); + expect(tables).toEqual([...TABLES_V1]); + const meta = Object.fromEntries( + db + .query("SELECT key, value FROM meta") + .all() + .map((row) => { + const { key, value } = row as { key: string; value: string }; + return [key, value]; + }), + ); + expect(meta.format_version).toBe("1"); + expect(meta.yjs_encoding).toBe("v1"); + expect(meta.library_id).toMatch(/^[0-9a-f-]{36}$/); + expect(meta.unclean_exit).toBe("0"); + db.close(); + }); + + test("the docs columns include the v7/v8 seams and the §6.3 list columns", () => { + openTestLibrary(temp.dir, createTestHost(clock)).shutdown(); + const db = raw(); + const columns = db + .query("SELECT name, \"notnull\" FROM pragma_table_info('docs')") + .all() as { name: string; notnull: number }[]; + const byName = new Map(columns.map((c) => [c.name, c.notnull])); + for (const name of [ + "lineage", + "link_key", + "kind", + "pin_key", + "size_bytes", + "snippet", + "trash_group", + "trashed_from", + "mirrored_at", + "schema", + ]) + expect(byName.has(name)).toBe(true); + expect(byName.get("link_key")).toBe(1); + expect(byName.get("lineage")).toBe(1); + const view = db + .query("SELECT name FROM pragma_table_info('doc_view')") + .all() + .map((row) => (row as { name: string }).name); + expect(view).toEqual(["doc_id", "width", "place", "remote_images"]); + db.close(); + }); + + test("sets WAL, synchronous=FULL and foreign keys", () => { + const library = openTestLibrary(temp.dir, createTestHost(clock)); + const db = ( + library as unknown as { + db: { + prepare(s: string): { get(): Record | undefined }; + }; + } + ).db; + expect(db.prepare("PRAGMA journal_mode").get()).toEqual({ + journal_mode: "wal", + }); + expect(db.prepare("PRAGMA synchronous").get()).toEqual({ synchronous: 2 }); + expect(db.prepare("PRAGMA foreign_keys").get()).toEqual({ + foreign_keys: 1, + }); + library.shutdown(); + }); + + test("turns on fullfsync on macOS", () => { + const library = openTestLibrary( + temp.dir, + createTestHost(clock, { platform: "darwin" }), + { fullfsync: true }, + ); + const db = ( + library as unknown as { + db: { + prepare(s: string): { get(): Record | undefined }; + }; + } + ).db; + expect(db.prepare("PRAGMA fullfsync").get()).toEqual({ fullfsync: 1 }); + expect(db.prepare("PRAGMA checkpoint_fullfsync").get()).toEqual({ + checkpoint_fullfsync: 1, + }); + library.shutdown(); + }); + + test("refuses a file that isn't a library", () => { + const db = raw(); + db.exec("CREATE TABLE other (x)"); + db.close(); + rejectsWith("corrupt", () => + openTestLibrary(temp.dir, createTestHost(clock)), + ); + // The lock is released on failure. + expect(() => readFileSync(join(temp.dir, LOCK_FILE))).toThrow(); + }); +}); + +describe("cloud folders", () => { + test("refuses a library inside a synced folder", () => { + const dir = join(temp.dir, "Dropbox", "TinyDocy"); + rejectsWith("cloud_folder", () => + openTestLibrary(dir, createTestHost(clock)), + ); + }); +}); + +describe("library.lock", () => { + test("refuses a library another running process holds", () => { + writeFileSync(join(temp.dir, LOCK_FILE), "424242"); + const host = createTestHost(clock, { isPidAlive: (pid) => pid === 424242 }); + rejectsWith("locked_elsewhere", () => openTestLibrary(temp.dir, host)); + }); + + test("takes over a stale lock", () => { + writeFileSync(join(temp.dir, LOCK_FILE), "424242"); + const host = createTestHost(clock, { isPidAlive: () => false }); + const library = openTestLibrary(temp.dir, host); + expect(readFileSync(join(temp.dir, LOCK_FILE), "utf8")).toBe( + String(process.pid), + ); + library.shutdown(); + expect(() => readFileSync(join(temp.dir, LOCK_FILE))).toThrow(); + }); +}); + +describe("unclean exit", () => { + test("a clean shutdown needs no quick_check", () => { + openTestLibrary(temp.dir, createTestHost(clock)).shutdown(); + const library = openTestLibrary(temp.dir, createTestHost(clock)); + expect(library.status.uncleanExit).toBe(false); + expect(library.status.quickCheck).toBeNull(); + library.shutdown(); + }); + + test("a crash runs quick_check on the next open", () => { + const first = openTestLibrary(temp.dir, createTestHost(clock)); + first.createDoc(); + crash(first); + const library = openTestLibrary(temp.dir, createTestHost(clock)); + expect(library.status.uncleanExit).toBe(true); + expect(library.status.quickCheck).toBe("ok"); + expect(library.status.readOnly).toBeNull(); + library.shutdown(); + }); + + test("shutdown checkpoints the WAL into the main file", () => { + const library = openTestLibrary(temp.dir, createTestHost(clock)); + library.createDoc({ title: "kept" }); + library.shutdown(); + const wal = join(temp.dir, `${LIBRARY_FILE}-wal`); + let walBytes = 0; + try { + walBytes = readFileSync(wal).byteLength; + } catch { + walBytes = 0; + } + expect(walBytes).toBe(0); + }); + + test("shutdown is idempotent", () => { + const library = openTestLibrary(temp.dir, createTestHost(clock)); + library.shutdown(); + expect(() => library.shutdown()).not.toThrow(); + }); +}); + +describe("format_version guard (T6)", () => { + function makeNewerLibrary() { + const library = openTestLibrary(temp.dir, createTestHost(clock)); + library.createDoc({ title: "from the future" }); + library.shutdown(); + const db = raw(); + db.exec("UPDATE meta SET value = '7' WHERE key = 'format_version'"); + db.exec("PRAGMA wal_checkpoint(TRUNCATE)"); + db.close(); + } + + test("a newer library opens read-only and is never written", () => { + makeNewerLibrary(); + const before = readFileSync(join(temp.dir, LIBRARY_FILE)); + const library = openTestLibrary(temp.dir, createTestHost(clock)); + expect(library.status.readOnly).toBe("library_too_new"); + expect(library.status.formatVersion).toBe(7); + expect(library.sidebar().docs.map((d) => d.title)).toEqual([ + "from the future", + ]); + rejectsWith("library_too_new", () => library.createDoc()); + const id = library.sidebar().docs[0]?.id ?? ""; + expect(library.load(id, "w1").updates).toEqual([]); + rejectsWith("library_too_new", () => + library.append(id, "w1", { + origin: "local", + update: new Uint8Array([1]), + }), + ); + library.shutdown(); + expect(readFileSync(join(temp.dir, LIBRARY_FILE)).equals(before)).toBe( + true, + ); + }); +}); + +describe("format migrations (T5)", () => { + const addColumn: FormatStep = { + to: 2, + up: (db) => db.exec("ALTER TABLE docs ADD COLUMN extra TEXT"), + }; + const addTable: FormatStep = { + to: 3, + up: (db) => db.exec("CREATE TABLE later (x INTEGER) STRICT"), + }; + + test("takes a pre-migration copy, then runs each step", () => { + openTestLibrary(temp.dir, createTestHost(clock)).shutdown(); + const library = openTestLibrary(temp.dir, createTestHost(clock), { + formatVersion: 3, + formatSteps: [addColumn, addTable], + }); + expect(library.status.formatVersion).toBe(3); + expect(library.status.readOnly).toBeNull(); + library.shutdown(); + const copies = createTestHost(clock) + .list(temp.dir) + .filter((f) => f.startsWith("library.pre-v1-")); + expect(copies).toHaveLength(1); + const copy = new Database(join(temp.dir, copies[0] ?? "")); + expect( + ( + copy + .query("SELECT value FROM meta WHERE key = 'format_version'") + .get() as { value: string } + ).value, + ).toBe("1"); + copy.close(); + }); + + test("a failed step leaves the library read-only at the last good version", () => { + openTestLibrary(temp.dir, createTestHost(clock)).shutdown(); + const library = openTestLibrary(temp.dir, createTestHost(clock), { + formatVersion: 3, + formatSteps: [ + addColumn, + { + to: 3, + up: (db) => { + db.exec("CREATE TABLE half (x)"); + throw new Error("boom"); + }, + }, + ], + }); + expect(library.status.readOnly).toBe("migration_failed"); + expect(library.status.formatVersion).toBe(2); + rejectsWith("library_read_only", () => library.createDoc()); + library.shutdown(); + const db = raw(); + expect( + db.query("SELECT 1 FROM sqlite_master WHERE name = 'half'").get(), + ).toBeNull(); + db.close(); + }); + + test("keeps the newest two pre-migration copies", () => { + openTestLibrary(temp.dir, createTestHost(clock)).shutdown(); + for (const to of [2, 3, 4]) { + clock.advance(60_000); + openTestLibrary(temp.dir, createTestHost(clock), { + formatVersion: to, + formatSteps: [ + { to: 2, up: () => {} }, + { to: 3, up: () => {} }, + { to: 4, up: () => {} }, + ], + }).shutdown(); + } + const copies = createTestHost(clock) + .list(temp.dir) + .filter((f) => f.startsWith("library.pre-v")) + .sort(); + expect(copies).toHaveLength(2); + expect(copies[0]).toStartWith("library.pre-v2-"); + expect(copies[1]).toStartWith("library.pre-v3-"); + }); +}); + +test("stats reads the pragmas back as set", () => { + const library = openTestLibrary(temp.dir, createTestHost(clock)); + // checkpoint_fullfsync's default differs between SQLite builds. + expect(library.stats().pragmas).toMatchObject({ + journalMode: "wal", + synchronous: 2, + fullfsync: 0, + foreignKeys: 1, + }); + library.shutdown(); +}); + +test("shutdown releases every file handle (Windows can delete the folder)", () => { + const lsof = Bun.which("lsof"); + if (!lsof) return; + const library = openTestLibrary(temp.dir, createTestHost(clock)); + library.createDoc({ title: "x" }); + library.sidebar(); + library.shutdown(); + const open = Bun.spawnSync([lsof, "-p", String(process.pid)]) + .stdout.toString() + .split("\n") + .filter((line) => line.includes(temp.dir)); + expect(open).toEqual([]); +}); diff --git a/tests/unit/lib/doc-migrations/doc-migrations.test.ts b/tests/unit/lib/doc-migrations/doc-migrations.test.ts new file mode 100644 index 00000000..e4bcc1bc --- /dev/null +++ b/tests/unit/lib/doc-migrations/doc-migrations.test.ts @@ -0,0 +1,51 @@ +import { describe, expect, test } from "bun:test"; + +import * as Y from "yjs"; + +import { + DOC_MIGRATIONS, + DOC_SCHEMA, + type DocMigration, + planDocMigrations, + runDocMigrations, +} from "@/lib/doc-migrations"; + +const steps: DocMigration[] = [ + { to: 1, migrate: (doc) => doc.getMap("m").set("one", true) }, + { to: 2, migrate: (doc) => doc.getMap("m").set("two", true) }, +]; + +describe("document migrations", () => { + test("the list starts empty, so today's documents are current", () => { + expect(DOC_MIGRATIONS).toEqual([]); + expect(DOC_SCHEMA).toBe(0); + expect(planDocMigrations(0)).toEqual({ kind: "current" }); + }); + + test("a newer document is too new", () => { + expect(planDocMigrations(1)).toEqual({ kind: "too_new" }); + }); + + test("plans only the missing steps", () => { + const plan = planDocMigrations(1, steps); + expect(plan).toMatchObject({ kind: "migrate", schemaTo: 2 }); + expect(plan.kind === "migrate" && plan.steps.map((s) => s.to)).toEqual([2]); + }); + + test("refuses a gap in the list", () => { + expect(() => + planDocMigrations(0, [{ to: 2, migrate: () => {} }]), + ).toThrow(); + }); + + test("runs every step in one transaction under the given origin", () => { + const doc = new Y.Doc(); + const origin = Symbol("migration"); + const seen: unknown[] = []; + doc.on("update", (_u: Uint8Array, o: unknown) => seen.push(o)); + expect(runDocMigrations(doc, planDocMigrations(0, steps), origin)).toBe(2); + expect(seen).toEqual([origin]); + expect(doc.getMap("m").toJSON()).toEqual({ one: true, two: true }); + expect(runDocMigrations(doc, { kind: "current" }, origin)).toBeNull(); + }); +}); diff --git a/tests/unit/lib/doc-migrations/import-guard.test.ts b/tests/unit/lib/doc-migrations/import-guard.test.ts new file mode 100644 index 00000000..82678a37 --- /dev/null +++ b/tests/unit/lib/doc-migrations/import-guard.test.ts @@ -0,0 +1,92 @@ +import { describe, expect, test } from "bun:test"; + +import * as Y from "yjs"; + +import { checkImport } from "@/lib/doc-migrations/import-guard"; + +function docWith(build: (doc: Y.Doc) => void): Uint8Array { + const doc = new Y.Doc(); + build(doc); + return Y.encodeStateAsUpdate(doc); +} + +function paragraph(text: string, attrs?: Record) { + const p = new Y.XmlElement("paragraph"); + const t = new Y.XmlText(); + t.insert(0, text, attrs); + p.insert(0, [t]); + return p; +} + +describe("checkImport", () => { + test("accepts a document the editor can open", () => { + const state = docWith((doc) => { + const heading = new Y.XmlElement("heading"); + heading.setAttribute("level", "1"); + doc + .getXmlFragment("default") + .insert(0, [heading, paragraph("hi", { bold: true })]); + }); + expect(checkImport(state)).toEqual({ ok: true }); + }); + + test("accepts the legacy link and code marks (the import migrations rename them)", () => { + const state = docWith((doc) => { + doc + .getXmlFragment("default") + .insert(0, [ + paragraph("x", { link: { href: "https://a.b" } }), + paragraph("y", { code: true }), + ]); + }); + expect(checkImport(state)).toEqual({ ok: true }); + }); + + test("refuses unknown nodes and marks, naming them", () => { + const state = docWith((doc) => { + doc + .getXmlFragment("default") + .insert(0, [ + new Y.XmlElement("marquee"), + paragraph("x", { blink: true }), + ]); + }); + expect(checkImport(state)).toEqual({ + ok: false, + code: "unknown_schema", + names: ["marquee", "blink"], + }); + }); + + test("refuses unknown roots", () => { + const state = docWith((doc) => { + doc.getXmlFragment("default").insert(0, [paragraph("x")]); + doc.getMap("payload").set("a", 1); + }); + expect(checkImport(state)).toMatchObject({ + ok: false, + code: "unknown_schema", + names: ["payload"], + }); + }); + + test("refuses oversized states before decoding them", () => { + const state = docWith((doc) => { + doc.getXmlFragment("default").insert(0, [paragraph("x".repeat(100))]); + }); + expect(checkImport(state, { maxBytes: 10 })).toEqual({ + ok: false, + code: "import_too_large", + names: [], + }); + }); + + test("refuses bytes that aren't a Yjs update", () => { + expect( + checkImport(new Uint8Array([255, 255, 255, 255, 1, 2, 3])), + ).toMatchObject({ + ok: false, + code: "corrupt", + }); + }); +}); diff --git a/tsconfig.desktop-node.json b/tsconfig.desktop-node.json new file mode 100644 index 00000000..d77f129c --- /dev/null +++ b/tsconfig.desktop-node.json @@ -0,0 +1,19 @@ +{ + "compilerOptions": { + "target": "ES2023", + "lib": ["es2023"], + "module": "esnext", + "moduleResolution": "bundler", + "strict": true, + "noEmit": true, + "skipLibCheck": true, + "isolatedModules": true, + "resolveJsonModule": true, + "types": ["node"] + }, + "include": [ + "desktop/storage/**/*.ts", + "desktop/main/**/*.ts", + "desktop/build/**/*.ts" + ] +} diff --git a/tsconfig.json b/tsconfig.json index 99c6bb37..9ea694ab 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -31,5 +31,13 @@ ".next/dev/types/**/*.ts", "**/*.mts" ], - "exclude": ["node_modules", "lib/pwa/service-worker.ts", "desktop/smoke"] + "exclude": [ + "node_modules", + "lib/pwa/service-worker.ts", + "desktop/smoke", + "desktop/storage-harness", + "desktop/storage", + "desktop/main", + "desktop/build" + ] }