Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
c49d549
build(desktop): typecheck the storage worker in a Node-only project
HMarzban Nov 17, 2026
f45c536
feat(desktop): add the library storage engine
HMarzban Nov 17, 2026
4b944d8
feat(platform): add the document migration hook and the import guard
HMarzban Nov 17, 2026
6897139
fix(desktop): only the window holding a document may write it
HMarzban Nov 17, 2026
57ed7ba
feat(desktop): add the storage worker entry and its main-side client
HMarzban Nov 17, 2026
02af525
test(ci): enforce the storage engine and worker import boundaries
HMarzban Nov 17, 2026
ba33787
feat(desktop): report the library pragmas and SQLite version in stats
HMarzban Nov 17, 2026
4cc17fe
test(desktop): add the storage durability soak (T1–T8)
HMarzban Nov 17, 2026
0ba711a
ci(desktop): run the full storage soak on a labelled pull request
HMarzban Nov 17, 2026
3a6cb91
docs(desktop): record the P2.6 storage worker in the changelog and spec
HMarzban Nov 17, 2026
207a5e7
fix(desktop): don't take a home folder named like a sync client for one
HMarzban Nov 17, 2026
26b3bfd
fix(desktop): announce every sidebar change in the index feed
HMarzban Nov 17, 2026
8cb74ea
test(desktop): time the soak's startup from process start
HMarzban Nov 17, 2026
d0d82ed
fix(desktop): finalize bun:sqlite statements before closing
HMarzban Nov 17, 2026
e6aadf3
Merge remote-tracking branch 'origin/claude/feat-p2-1-local-session' …
HMarzban Nov 17, 2026
f6e5306
feat(desktop): create documents at a given schema with any safe id
HMarzban Nov 17, 2026
7a3f09f
test(desktop): run P2.1's DocStore contract on both storage drivers
HMarzban Nov 17, 2026
1390edb
docs(desktop): note the shared contract and T7's home for P2.6
HMarzban Nov 17, 2026
de7a82c
refactor(platform): name the migration step migrate, as local-session…
HMarzban Nov 17, 2026
c5e2613
docs(platform): rewrap the runDocMigrations comment
HMarzban Nov 17, 2026
ca8ddbe
Merge remote-tracking branch 'origin/claude/feat-p2-1-local-session' …
HMarzban Nov 17, 2026
5c55dbf
Merge remote-tracking branch 'origin/main' into claude/feat-p2-6-stor…
HMarzban Nov 17, 2026
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
152 changes: 152 additions & 0 deletions .github/workflows/desktop-storage-soak.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,152 @@
name: Desktop storage soak

# Plan T1–T8 (P2.6): the storage worker on node:sqlite inside Electron,
# SIGKILLed during appends, compactions and migrations, quit with writes in
# flight, opened by an older app, and run through a simulated year with
# 1,000 documents. The verifier reopens every library with the engine on
# bun:sqlite. See tests/load/desktop-durability-soak.ts.
#
# Pull requests that touch storage run a short profile (20 kills per case);
# the nightly and manual runs use the plan's 200, and so does a pull request
# labelled `storage-soak-full` (workflow_dispatch needs the file on main).
# Not a required check.
# Read-only token and no secrets, like CI.

on:
pull_request:
types: [opened, synchronize, reopened, labeled]
paths:
- desktop/storage/**
- desktop/storage-harness/**
- desktop/dev/bun-sqlite-driver.ts
- lib/platform/desktop/**
- lib/platform/doc-store.ts
- tests/helpers/doc-store-contract.ts
- tests/load/desktop-durability-soak.ts
- .github/workflows/desktop-storage-soak.yml
workflow_call:
inputs:
kills:
type: number
default: 200
workflow_dispatch:
inputs:
kills:
description: Kills per kill case (T1, T2; T1-dense runs half)
type: number
default: 200

permissions:
contents: read

concurrency:
group: desktop-storage-soak-${{ github.event.pull_request.number || github.ref }}-${{ github.event_name }}
cancel-in-progress: true

jobs:
soak:
name: ${{ matrix.group.name }} (${{ matrix.os }})
runs-on: ${{ matrix.os }}
timeout-minutes: 90
permissions:
contents: read
strategy:
fail-fast: false
matrix:
os: [macos-26, windows-latest, ubuntu-24.04]
group:
- name: contract, T3–T6
cases: contract,T3,T4,T5,T6
- name: T1 kill during append
cases: T1,T1-dense
- name: T2 kill during compaction
cases: T2
- name: T8 year and startup
cases: T8
defaults:
run:
shell: bash
env:
KILLS: ${{ github.event_name != 'pull_request' && (inputs.kills || 200) || contains(github.event.pull_request.labels.*.name, 'storage-soak-full') && 200 || 20 }}
steps:
- name: Checkout
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false

- name: Set up Bun
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
with:
bun-version-file: .tool-versions

# The root install brings Yjs and the engine's dependencies; the
# harness has its own lockfile with Electron, whose binary downloads
# on first use.
- name: Install
run: |
bun install --frozen-lockfile
cd desktop/storage-harness
bun install --frozen-lockfile
bun -e 'require("electron")'

# Everything after install runs on Bun. The runner image ships Node,
# so a stub makes any call to it fail loudly. Install comes first:
# lifecycle scripts (husky) call `node`.
- name: Block Node.js
run: |
mkdir -p "$RUNNER_TEMP/no-node"
cat > "$RUNNER_TEMP/no-node/node" <<'EOF'
#!/bin/sh
echo "::error::node was called: $*" >&2
exit 1
EOF
chmod +x "$RUNNER_TEMP/no-node/node"
echo "$RUNNER_TEMP/no-node" >> "$GITHUB_PATH"

# Windows process spawns look for node.exe and never run the sh stub
# above, so put a compiled stub beside it (found in spike S3).
- name: Block Node.js (node.exe on Windows)
if: runner.os == 'Windows'
run: |
cat > "$RUNNER_TEMP/node-stub.ts" <<'EOF'
console.error(`::error::node was called: ${process.argv.slice(2).join(" ")}`);
process.exit(1);
EOF
bun build --compile "$RUNNER_TEMP/node-stub.ts" --outfile "$RUNNER_TEMP/no-node/node.exe"
if cmd.exe //d //c "node --version" > /dev/null 2>&1; then
echo "::error::the node.exe stub is not first on PATH"
exit 1
fi

- name: Audit the harness dependencies
working-directory: desktop/storage-harness
run: bun audit --audit-level=high

- name: Typecheck and build the harness
working-directory: desktop/storage-harness
run: |
bun run typecheck
bun run build

- name: Allow the Chromium sandbox (electron/electron#42510)
if: runner.os == 'Linux'
run: sudo sysctl -w kernel.apparmor_restrict_unprivileged_userns=0

- name: Soak
env:
CASES: ${{ matrix.group.cases }}
run: |
if [ "$RUNNER_OS" = Linux ]; then
xvfb-run -a bun tests/load/desktop-durability-soak.ts --case="$CASES" --kills="$KILLS" --out=soak-out
else
bun tests/load/desktop-durability-soak.ts --case="$CASES" --kills="$KILLS" --out=soak-out
fi

- name: Upload results
if: always()
uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: storage-soak-${{ matrix.os }}-${{ strategy.job-index }}
path: soak-out
if-no-files-found: ignore
retention-days: 30
10 changes: 10 additions & 0 deletions .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,8 @@ name: Nightly
# the gate env at its default (spec §3.1 N+1 ship gate)
# desktop-smoke the D-A12 Electron smoke (desktop-smoke.yml) on macOS,
# Windows and Linux
# desktop-storage-soak plan T1–T8 against the storage worker in Electron
# (desktop-storage-soak.yml), 200 kills per case, 3 OSes
#
# Read-only token and no secrets, like CI. A red run means a regression in
# offline editing, long sessions, concurrency or schema rollout safety that
Expand Down Expand Up @@ -361,3 +363,11 @@ jobs:
permissions:
contents: read
uses: ./.github/workflows/desktop-smoke.yml

desktop-storage-soak:
name: Desktop storage soak (T1–T8)
permissions:
contents: read
uses: ./.github/workflows/desktop-storage-soak.yml
with:
kills: 200
61 changes: 61 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,67 @@ since the previous tag.

## [Unreleased]

### Desktop P2.6: the library storage worker

The SQLite engine behind the desktop library, where documents, folders and
tags live, and the `worker_thread` that runs it inside Electron main (D-A1).
Nothing in the web app or its build uses it yet; P2.5's shell starts the
worker and P2.7 builds the files sidebar on it.

#### Added

- **Desktop:** `desktop/storage/engine/`, with no `node:`, `bun:` or
Electron imports, behind a `SqliteDriver` seam (`node:sqlite` in the
worker, `bun:sqlite` in tests and the dev host):
- the format-1 schema, so the Local release needs no later migration:
`docs` (lineage and a `NOT NULL` `link_key` minted at insert, `kind`, the
list columns), `folders`, `tags`, `doc_tags`, `doc_view(doc_id, width,
place, remote_images)`, `import_sources`, `snapshots` and `updates` with
an `AUTOINCREMENT` seq;
- the open order: cloud-synced folders refused, `library.lock` with a pid
check, WAL with `synchronous=FULL` (`fullfsync` on macOS), and the
`format_version` guard before any write, so a newer library opens
read-only and untouched; `quick_check` after an unclean exit;
- format migrations with a `VACUUM INTO` copy first (2 kept), one
transaction per step, and read-only on failure;
- appends and renderer-driven compaction in one transaction each, with
one window per document (`already_open`, `forbidden`, `document_open`);
- documents, folders, tags, Trash (emptied after 30 days) and the sidebar
snapshot, with a change feed: every change the sidebar shows, appends
included, moves `rev`, and the worker sends `index_changed` with the
changed rows;
- a daily `VACUUM INTO` copy on the first idle moment, keeping 7, and a
reader for restoring one document from a day;
- shutdown for the quit flush: WAL checkpoint, `unclean_exit` cleared,
lock released.
- **Desktop:** `desktop/storage/worker.ts` and `client.ts`: the worker
answers main one request at a time with `Result` envelopes and transferred
bytes; the client rejects pending calls with `worker_restarting` if the
worker dies and ends with a 2 s shutdown.
- **Platform:** `lib/doc-migrations/` (the open-time migration hook list,
empty so far, and `checkImport`, which refuses an import that is too
large, isn't Yjs, or carries a node, mark or root this build doesn't
know) and `lib/platform/desktop/rpc-doc-store.ts` (`DocStore` over worker
replies, for `IpcDocStore`).
- **Tests:** P2.1's shared `DocStore` contract on the engine, on
`bun:sqlite` under `bun test` and on `node:sqlite` inside the Electron
worker; engine suites (open, content, index and its change feed, daily
copies, codes, and a property test of random appends, compactions,
crashes and reopens); the import-boundary test's storage rows; and
`tests/load/desktop-durability-soak.ts` with its Electron harness in
`desktop/storage-harness/` for soak cases T1–T6 and T8.
- **CI:** `desktop-storage-soak.yml` on macOS, Windows and Linux: 20 kills
per case on pull requests that touch storage, 200 nightly or with the
`storage-soak-full` label. `bun run typecheck` also checks the new
`tsconfig.desktop-node.json`.

#### Changed

- **Docs:** the desktop architecture spec records where P2.6 put the engine
and the harness, the worker-side registry check and `claim`, the
import-boundary allowance for `lib/schema-names.ts`, and that T7 is covered
by `local-session`'s tests, not the storage soak.

### Desktop P2.1b: `local-session`, the batcher and `MemoryDocStore`

The second half of desktop plan P2.1: the platform-neutral storage path that
Expand Down
56 changes: 56 additions & 0 deletions desktop/dev/bun-sqlite-driver.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
/**
* `SqliteDriver` over `bun:sqlite`, for unit tests and the dev storage host
* (architecture §11). Normalizes Bun's differences from `node:sqlite`: a
* missing row is `null` in Bun and `undefined` here, and integers stay
* numbers (`safeIntegers` off).
*
* `close` finalizes every cached statement first: Bun's `close()` leaves a
* connection with live statements open as a zombie that keeps the file
* handles, which on Windows blocks deleting or replacing the library.
*/
import { Database } from "bun:sqlite";

import type {
OpenSqlite,
SqliteDriver,
SqliteStatement,
SqlRow,
SqlValue,
} from "@/desktop/storage/engine/driver";

export const openBunSqlite: OpenSqlite = (path, { readOnly }) => {
const db = readOnly
? new Database(path, { readonly: true })
: new Database(path, { create: true });
const cache = new Map<string, SqliteStatement>();
const statements: { finalize(): void }[] = [];
const driver: SqliteDriver = {
exec: (sql) => db.exec(sql),
prepare(sql) {
const cached = cache.get(sql);
if (cached) return cached;
const statement = db.prepare<SqlRow, SqlValue[]>(sql);
statements.push(statement);
const wrapped: SqliteStatement = {
run: (...params) => {
const result = statement.run(...params);
return {
changes: result.changes,
lastInsertRowid: Number(result.lastInsertRowid),
};
},
get: (...params) => statement.get(...params) ?? undefined,
all: (...params) => statement.all(...params),
};
cache.set(sql, wrapped);
return wrapped;
},
close: () => {
for (const statement of statements) statement.finalize();
statements.length = 0;
cache.clear();
db.close();
},
};
return driver;
};
2 changes: 2 additions & 0 deletions desktop/storage-harness/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
/node_modules
/dist
25 changes: 25 additions & 0 deletions desktop/storage-harness/build.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
/**
* Bundles the harness main and the real storage worker as CJS for
* Electron, the way `desktop/build/build.ts` (P2.5) will: `node:*` and
* `electron` stay external, everything else (Yjs, the engine) is inlined.
*/
import path from "node:path";

const root = import.meta.dir;
const result = await Bun.build({
entrypoints: [
path.join(root, "main.ts"),
path.join(root, "../storage/worker.ts"),
],
outdir: path.join(root, "dist"),
target: "node",
format: "cjs",
external: ["electron"],
naming: "[name].cjs",
});
if (!result.success) {
for (const log of result.logs) console.error(log);
process.exit(1);
}
for (const output of result.outputs)
console.log(`${path.relative(root, output.path)} ${output.size} B`);
46 changes: 46 additions & 0 deletions desktop/storage-harness/bun.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 15 additions & 0 deletions desktop/storage-harness/bunfig.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# Supply-chain guardrail: only resolve package versions that have been
# public for at least 3 days (value in seconds). Gives the registry and the
# community time to catch and unpublish compromised releases before
# `bun add` / `bun update` (local or in CI) can pick them up.
[install]
minimumReleaseAge = 259200

# Run package bins (next, eslint, playwright, ...) on the Bun runtime
# instead of Node, for `bun run` scripts and `bun <bin>`. Plain `bunx`
# ignores this setting; use `bunx --bun` for packages that are not installed.
[run]
bun = true

# `bun test`: unit tests only (Playwright specs live in tests/e2e), with a
# happy-dom window preloaded.
Loading
Loading